From 0e2a5ece68be38e123fc7fa85a39a59cd7caff6f Mon Sep 17 00:00:00 2001 From: lvyufeng Date: Sat, 16 Aug 2025 18:04:16 +0800 Subject: [PATCH] add core.testing for torch ut --- .gitignore | 3 +- .../inference/gpt-oss/gpt_oss_standalone.py | 1 - .../peft/lora/Qwen2.5-7B-Instruct-Lora.ipynb | 23662 +++++++++++++--- mindnlp/core/_C/__init__.py | 84 +- mindnlp/core/_C/_nn.py | 3 +- mindnlp/core/__init__.py | 8 +- mindnlp/core/_bind.py | 2 +- mindnlp/core/_dtype.py | 5 + mindnlp/core/_dynamo/__init__.py | 1 + mindnlp/core/_dynamo/config.py | 677 + mindnlp/core/_dynamo/decorators.py | 14 + mindnlp/core/_dynamo/trace_rules.py | 8 + mindnlp/core/_environment.py | 2 + mindnlp/core/_guards.py | 1169 + mindnlp/core/_logging/__init__.py | 20 + mindnlp/core/_logging/_internal.py | 1433 + mindnlp/core/_logging/_registrations.py | 249 + mindnlp/core/_logging/scribe.py | 63 + mindnlp/core/_logging/structured.py | 109 + mindnlp/core/_prims/numpy.py | 11 + mindnlp/core/_tensor.py | 9 +- mindnlp/core/_utils_internal.py | 358 + mindnlp/core/amp/grad_scaler.py | 14 - mindnlp/core/backends/__init__.py | 11 + mindnlp/core/cuda/__init__.py | 5 +- mindnlp/core/dispatcher.py | 5 +- mindnlp/core/npu/__init__.py | 3 + mindnlp/core/testing/__init__.py | 1 + mindnlp/core/testing/_comparison.py | 15 + mindnlp/core/testing/_creation.py | 276 + mindnlp/core/testing/_internal/common_cuda.py | 362 + .../testing/_internal/common_device_type.py | 1980 ++ .../core/testing/_internal/common_dtype.py | 226 + mindnlp/core/testing/_internal/common_nn.py | 3991 +++ mindnlp/core/testing/_internal/common_npu.py | 362 + .../core/testing/_internal/common_utils.py | 5634 ++++ mindnlp/core/types.py | 78 +- mindnlp/core/utils/_backport_slots.py | 116 + mindnlp/core/utils/_config_module.py | 804 + mindnlp/core/utils/_import_utils.py | 38 + mindnlp/core/utils/_traceback.py | 260 + mindnlp/core/utils/weak.py | 363 + mindnlp/utils/torch_proxy.py | 2 +- 43 files changed, 38623 insertions(+), 3814 deletions(-) create mode 100644 mindnlp/core/_dynamo/config.py create mode 100644 mindnlp/core/_dynamo/trace_rules.py create mode 100644 mindnlp/core/_environment.py create mode 100644 mindnlp/core/_guards.py create mode 100644 mindnlp/core/_logging/__init__.py create mode 100644 mindnlp/core/_logging/_internal.py create mode 100644 mindnlp/core/_logging/_registrations.py create mode 100644 mindnlp/core/_logging/scribe.py create mode 100644 mindnlp/core/_logging/structured.py create mode 100644 mindnlp/core/_utils_internal.py create mode 100644 mindnlp/core/testing/_creation.py create mode 100644 mindnlp/core/testing/_internal/common_cuda.py create mode 100644 mindnlp/core/testing/_internal/common_device_type.py create mode 100644 mindnlp/core/testing/_internal/common_dtype.py create mode 100644 mindnlp/core/testing/_internal/common_nn.py create mode 100644 mindnlp/core/testing/_internal/common_npu.py create mode 100644 mindnlp/core/testing/_internal/common_utils.py create mode 100644 mindnlp/core/utils/_backport_slots.py create mode 100644 mindnlp/core/utils/_config_module.py create mode 100644 mindnlp/core/utils/_import_utils.py create mode 100644 mindnlp/core/utils/_traceback.py create mode 100644 mindnlp/core/utils/weak.py diff --git a/.gitignore b/.gitignore index e18da48b0..cde8b6f6b 100644 --- a/.gitignore +++ b/.gitignore @@ -172,4 +172,5 @@ tests/transformers/ tests/huggingface_transformers/ .gradio/ -huanhuan.json \ No newline at end of file +huanhuan.json +pytorch/ \ No newline at end of file diff --git a/examples/transformers/inference/gpt-oss/gpt_oss_standalone.py b/examples/transformers/inference/gpt-oss/gpt_oss_standalone.py index ee618ea12..f500d3185 100644 --- a/examples/transformers/inference/gpt-oss/gpt_oss_standalone.py +++ b/examples/transformers/inference/gpt-oss/gpt_oss_standalone.py @@ -1,5 +1,4 @@ import mindnlp -from mindnlp.core import distributed as dist from transformers import AutoModelForCausalLM, AutoTokenizer model_id = "openai/gpt-oss-20b" diff --git a/examples/transformers/peft/lora/Qwen2.5-7B-Instruct-Lora.ipynb b/examples/transformers/peft/lora/Qwen2.5-7B-Instruct-Lora.ipynb index 50ceab397..95b86641d 100644 --- a/examples/transformers/peft/lora/Qwen2.5-7B-Instruct-Lora.ipynb +++ b/examples/transformers/peft/lora/Qwen2.5-7B-Instruct-Lora.ipynb @@ -20,8 +20,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] DEVICE(957136,ffffb13e8020,python):2025-08-16-12:18:37.758.703 [mindspore/ccsrc/utils/dlopen_macro.h:165] DlsymAscend] Dynamically load symbol aclprofGetSupportedFeaturesV2 failed, result = /usr/local/Ascend/ascend-toolkit/latest/lib64/libmsprofiler.so: undefined symbol: aclprofGetSupportedFeaturesV2\n", - "[WARNING] DEVICE(957136,ffffb13e8020,python):2025-08-16-12:18:37.758.835 [mindspore/ccsrc/utils/dlopen_macro.h:165] DlsymAscend] Dynamically load symbol aclrtEventGetTimestamp failed, result = /usr/local/Ascend/ascend-toolkit/latest/lib64/libascendcl.so: undefined symbol: aclrtEventGetTimestamp\n", + "[WARNING] DEVICE(1065413,ffff968db020,python):2025-08-16-13:12:49.008.926 [mindspore/ccsrc/utils/dlopen_macro.h:165] DlsymAscend] Dynamically load symbol aclprofGetSupportedFeaturesV2 failed, result = /usr/local/Ascend/ascend-toolkit/latest/lib64/libmsprofiler.so: undefined symbol: aclprofGetSupportedFeaturesV2\n", + "[WARNING] DEVICE(1065413,ffff968db020,python):2025-08-16-13:12:49.009.049 [mindspore/ccsrc/utils/dlopen_macro.h:165] DlsymAscend] Dynamically load symbol aclrtEventGetTimestamp failed, result = /usr/local/Ascend/ascend-toolkit/latest/lib64/libascendcl.so: undefined symbol: aclrtEventGetTimestamp\n", "/home/lvyufeng/miniconda3/envs/mindspore/lib/python3.11/site-packages/numpy/core/getlimits.py:549: UserWarning: The value of the smallest subnormal for type is zero.\n", " setattr(self, word, getattr(machar, word).flat[0])\n", "/home/lvyufeng/miniconda3/envs/mindspore/lib/python3.11/site-packages/numpy/core/getlimits.py:89: UserWarning: The value of the smallest subnormal for type is zero.\n", @@ -59,16 +59,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "--2025-08-16 12:18:51-- https://openi.pcl.ac.cn/mindnlp/self-llm/raw/branch/master/dataset/huanhuan.json\n", - "Resolving openi.pcl.ac.cn (openi.pcl.ac.cn)... 210.22.22.132, 120.237.18.54, 218.17.115.247\n", - "Connecting to openi.pcl.ac.cn (openi.pcl.ac.cn)|210.22.22.132|:443... connected.\n", + "--2025-08-16 13:13:02-- https://openi.pcl.ac.cn/mindnlp/self-llm/raw/branch/master/dataset/huanhuan.json\n", + "Resolving openi.pcl.ac.cn (openi.pcl.ac.cn)... 218.17.115.247, 120.237.18.54, 210.22.22.132\n", + "Connecting to openi.pcl.ac.cn (openi.pcl.ac.cn)|218.17.115.247|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: unspecified [text/plain]\n", "Saving to: ‘huanhuan.json’\n", "\n", - "huanhuan.json [ <=> ] 914.84K 4.41MB/s in 0.2s \n", + "huanhuan.json [ <=> ] 914.84K 454KB/s in 2.0s \n", "\n", - "2025-08-16 12:18:52 (4.41 MB/s) - ‘huanhuan.json’ saved [936793]\n", + "2025-08-16 13:13:05 (454 KB/s) - ‘huanhuan.json’ saved [936793]\n", "\n" ] } @@ -214,7 +214,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Map: 100%|██████████| 3729/3729 [00:13<00:00, 284.85 examples/s]\n" + "Map: 100%|██████████| 3729/3729 [00:13<00:00, 279.28 examples/s]\n" ] }, { @@ -292,7 +292,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "170764e5-d899-4ef4-8c53-36f6dec0d198", "metadata": { "tags": [] @@ -302,7 +302,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Loading checkpoint shards: 100%|██████████| 4/4 [00:34<00:00, 8.54s/it]\n" + "Loading checkpoint shards: 100%|██████████| 4/4 [00:32<00:00, 8.19s/it]\n" ] }, { @@ -322,7 +322,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "2323eac7-37d5-4288-8bc5-79fac7113402", "metadata": { "tags": [] @@ -334,7 +334,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "f808b05c-f2cb-48cf-a80d-0c42be6051c7", "metadata": { "tags": [] @@ -365,7 +365,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "2d304ae2-ab60-4080-a80d-19cac2e3ade3", "metadata": { "tags": [] @@ -374,7 +374,7 @@ { "data": { "text/plain": [ - "LoraConfig(task_type=, peft_type=, auto_mapping=None, base_model_name_or_path=None, revision=None, inference_mode=False, r=8, target_modules={'v_proj', 'o_proj', 'up_proj', 'q_proj', 'k_proj', 'down_proj', 'gate_proj'}, exclude_modules=None, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', trainable_token_indices=None, loftq_config={}, eva_config=None, corda_config=None, use_dora=False, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False), lora_bias=False)" + "LoraConfig(task_type=, peft_type=, auto_mapping=None, base_model_name_or_path=None, revision=None, inference_mode=False, r=8, target_modules={'gate_proj', 'v_proj', 'o_proj', 'up_proj', 'q_proj', 'down_proj', 'k_proj'}, exclude_modules=None, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', trainable_token_indices=None, loftq_config={}, eva_config=None, corda_config=None, use_dora=False, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False), lora_bias=False)" ] }, "execution_count": 13, @@ -398,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "2c2489c5-eaab-4e1f-b06a-c3f914b4bf8e", "metadata": { "tags": [] @@ -407,7 +407,7 @@ { "data": { "text/plain": [ - "LoraConfig(task_type=, peft_type=, auto_mapping=None, base_model_name_or_path='Qwen/Qwen2.5-7B-Instruct', revision=None, inference_mode=False, r=8, target_modules={'v_proj', 'o_proj', 'up_proj', 'q_proj', 'k_proj', 'down_proj', 'gate_proj'}, exclude_modules=None, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', trainable_token_indices=None, loftq_config={}, eva_config=None, corda_config=None, use_dora=False, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False), lora_bias=False)" + "LoraConfig(task_type=, peft_type=, auto_mapping=None, base_model_name_or_path='Qwen/Qwen2.5-7B-Instruct', revision=None, inference_mode=False, r=8, target_modules={'gate_proj', 'v_proj', 'o_proj', 'up_proj', 'q_proj', 'down_proj', 'k_proj'}, exclude_modules=None, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', trainable_token_indices=None, loftq_config={}, eva_config=None, corda_config=None, use_dora=False, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False), lora_bias=False)" ] }, "execution_count": 14, @@ -422,7 +422,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "ebf5482b-fab9-4eb3-ad88-c116def4be12", "metadata": { "tags": [] @@ -450,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "7e76bbff-15fd-4995-a61d-8364dc5e9ea0", "metadata": { "tags": [] @@ -466,14 +466,14 @@ " save_steps=100, \n", " learning_rate=1e-4,\n", " save_on_each_node=True,\n", - " fp16=True,\n", + " # fp16=True,\n", " # gradient_checkpointing=True\n", ")" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "f142cb9c-ad99-48e6-ba86-6df198f9ed96", "metadata": { "tags": [] @@ -483,17 +483,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/lvyufeng/miniconda3/envs/mindspore/lib/python3.11/site-packages/accelerate/utils/modeling.py:2086: FutureWarning: `torch.npu.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('npu', args...)` instead.\n", - " return torch.npu.amp.GradScaler(**kwargs)\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "get_grad_scaler {}\n" - ] } ], "source": [ @@ -507,7 +498,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "3d948594", "metadata": {}, "outputs": [ @@ -520,7 +511,7 @@ "Local process index: 0\n", "Device: device(type=npu, index=0)\n", "\n", - "Mixed precision type: fp16" + "Mixed precision type: no" ] }, "execution_count": 18, @@ -540,122 +531,404 @@ "tags": [] }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:37.656.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.106.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "get_mixed_precision_context_manager True\n", - "fp16\n", - "torch.autocast\n", - "\n", - "\n", - "\n", - "起始行号: 55, 源代码: def __init__(\n", - " self,\n", - " device_type: str,\n", - " dtype: Optional[_dtype] = None,\n", - " enabled: bool = True,\n", - " cache_enabled: Optional[bool] = None,\n", - " ):\n", - " if not isinstance(device_type, str):\n", - " raise ValueError(\n", - " f\"Expected `device_type` of type `str`, got: `{type(device_type)}`\"\n", - " )\n", - " self.device_type = device_type\n", - " if dtype is None:\n", - " dtype = float32\n", - " self.dtype = dtype\n", - " self.amp_level = AmpLevel.AmpAuto if enabled else AmpLevel.AmpO0\n", - "\n" + "embed_tokens True\n", + "position_embeddings True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:05.938.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.310.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.412.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "54910.402 0.83786625\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] PRE_ACT(957136,fffd8bfff120,python):2025-08-16-12:20:06.449.385 [mindspore/ccsrc/memory/mem_pool/mem_dynamic_allocator.cc:700] FreeIdleMemsByEagerFree] Eager free count : 2, free memory : 12837284864, real free : 1186988032, not free size: 11650296832.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.481.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.532.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.654.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.671.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:09.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.687.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.705.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.743.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.791.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "112496.695 1.7165633\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:10.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:40.910.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "81901.17 1.2497127\n", - "all_finite forward False\n" + "True\n", + "0.8376558 0.8376558\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:11.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:41.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.800.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.960.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.185.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.256.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "71451.305 1.0902604\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [-1018540541 0 0 0 0 0\n", - " 0 0]\n", - "found_inf [-1018540541 0 0 0 0 0\n", - " 0 0]\n", - "found_inf [ True False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 1.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "1.0\n", - "scaler update\n", - "scaler update False\n", - "65536.0\n", - "32768.0\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.369.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.382.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.394.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.459.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.7167852 1.7167852\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:42.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.127.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.139.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.175.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.187.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.198.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.222.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.234.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.246.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.258.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.411.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.424.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.427.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.437.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.441.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.2495626 1.2495626\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:43.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.330.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.530.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.104.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.130.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.143.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.181.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.219.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.258.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.284.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.310.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.336.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.387.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.401.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.414.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:44.418.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0894517 1.0894517\n", + "all_finite forward True\n" ] }, { @@ -664,8 +937,8 @@ "\n", "
\n", " \n", - " \n", - " [145/702 09:43 < 37:52, 0.25 it/s, Epoch 0.62/3]\n", + " \n", + " [ 70/702 05:00 < 46:28, 0.23 it/s, Epoch 0.30/3]\n", "
\n", " \n", " \n", @@ -677,59 +950,27 @@ " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", " \n", " \n", "
104.3729003.862200
203.1472003.101300
302.9642002.953100
403.1936003.186700
503.0626003.065700
602.995400
702.982700
803.067700
902.939100
1003.005000
1103.044500
1203.052100
1303.225000
1403.4697003.014500

" @@ -745,10456 +986,26461 @@ "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:12.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:45.772.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:45.961.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:45.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:45.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.350.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.470.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.600.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.980.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.129.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "30739.01 0.93808013\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:13.394.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.181.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.220.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.246.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.285.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.342.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.352.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "88414.24 2.6981885\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8636833 0.8636833\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:14.234.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.954.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:46.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.450.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.590.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.740.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.890.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.104.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "31042.045 0.94732803\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:15.440.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.164.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.179.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.224.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "28081.932 0.85699254\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [-803458813 0 0 0 0 0\n", - " 0 0]\n", - "found_inf [-803458813 0 0 0 0 0\n", - " 0 0]\n", - "found_inf [ True False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 1.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "1.0\n", - "scaler update\n", - "scaler update False\n", - "32768.0\n", - "16384.0\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:16.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.328.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.362.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "17063.844 1.0414944\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "2.4955323 2.4955323\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:16.936.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.970.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:47.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.350.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.740.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.100.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.126.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.139.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "20659.578 1.2609606\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:17.764.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.179.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.205.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.258.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.284.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.329.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "18347.51 1.1198431\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8483153 0.8483153\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:18.581.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.742.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.934.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.965.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.978.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:48.991.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.500.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.310.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.450.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.710.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.124.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "18720.67 1.142619\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [-1164986109 0 0 0 0 0\n", - " 0 0]\n", - "found_inf [-1164986109 0 0 0 0 0\n", - " 0 0]\n", - "found_inf [ True False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 1.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "1.0\n", - "scaler update\n", - "scaler update False\n", - "16384.0\n", - "8192.0\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:19.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.137.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.164.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.204.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.217.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.244.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.312.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.315.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:49.328.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "12079.898 1.474597\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.775309 0.775309\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:20.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.615.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.704.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.718.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.733.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "14997.002 1.8306887\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:21.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.820.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.835.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.849.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.864.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.893.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.907.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.936.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.965.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9899.985 1.2084943\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:22.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.981.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.995.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:50.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.140.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7154.4824 0.8733499\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.91648364 0.91648364\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:23.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.717.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.736.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.751.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.780.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.795.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.810.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.856.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.885.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9302.338 1.1355393\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:24.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.914.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:51.991.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.700.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.380.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.530.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.700.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.100.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "11410.603 1.3928958\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:25.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.137.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.156.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "13681.863 1.6701493\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1137623 1.1137623\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:26.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:52.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.700.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.270.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.590.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.750.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.124.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.140.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7389.4536 0.9020329\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:27.821.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.226.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.242.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.258.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.274.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.290.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.322.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9701.396 1.1842525\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:28.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.424.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.474.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:53.495.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4588.254 0.5600896\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0039128 1.0039128\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:29.512.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.340.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.420.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.570.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.116.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.131.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.175.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.190.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.204.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.219.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.234.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7891.192 0.96328026\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:30.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.248.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.278.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.341.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.355.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.414.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.443.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9734.306 1.1882697\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:31.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.462.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:54.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5672.57 0.6924524\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9893847 0.9893847\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:32.776.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:55.746.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:55.964.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:55.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:55.997.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.120.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.270.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.420.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.570.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.860.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.116.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.130.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5531.1675 0.67519134\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:33.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.189.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.203.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.247.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.262.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.320.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9602.322 1.1721585\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:34.572.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.397.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "11391.412 1.3905532\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1666603 1.1666603\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:36.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.891.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.915.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.944.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:56.987.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.200.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.310.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.600.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.750.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.900.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6778.675 0.82747495\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:36.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.105.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.164.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.222.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.295.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6709.415 0.8190204\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:37.830.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.310.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.328.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7294.2373 0.8904098\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.4709126 1.4709126\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:38.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:57.864.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.106.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.135.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.179.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.224.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7415.2695 0.90518427\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:40.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.312.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.341.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.356.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.414.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.443.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5251.7427 0.64108187\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:41.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.502.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:58.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "11432.913 1.3956193\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8739696 0.8739696\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:42.830.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.287.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7051.7896 0.86081415\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:42.961.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.506.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.521.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.564.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.578.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.592.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.607.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.635.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5747.045 0.70154357\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:44.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.666.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.669.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.678.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:07:59.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7764.5986 0.947827\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6429041 0.6429041\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:45.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:00.894.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.940.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.181.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.269.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6362.836 0.7767134\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:46.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.327.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.342.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.356.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.428.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.442.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.456.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.470.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7277.203 0.88833046\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:47.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.529.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:01.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7053.908 0.8610728\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8781486 0.8781486\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:48.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.242.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.261.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.369.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.431.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5251.716 0.6410786\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:49.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.462.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.538.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.554.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.585.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.633.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.648.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9778.426 1.1936555\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:50.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.682.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.693.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:02.697.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4631.264 0.56533986\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0777109 1.0777109\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:51.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.452.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.465.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.489.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.514.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.538.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.550.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.586.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.623.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6773.184 0.8268047\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:53.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.635.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.648.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.671.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.684.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.720.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.732.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.744.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.768.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.780.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.783.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.793.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:03.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2924.2185 0.35696027\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.3414818 1.3414818\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:54.127.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.428.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.452.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.546.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7816.322 0.95414084\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:55.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.594.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.606.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.618.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.641.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.654.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.666.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.677.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.703.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.706.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.714.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:04.718.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6861.905 0.83763486\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6096401 0.6096401\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:56.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:05.946.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.256.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.316.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6842.7554 0.8352973\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:57.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.538.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5346.579 0.6526586\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:57.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.568.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.586.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:06.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5900.034 0.720219\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0617661 1.0617661\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:58.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.760.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.118.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.242.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.258.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "13363.887 1.6313338\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:59.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.303.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.333.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.363.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3293.2358 0.40200633\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:00.882.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:07.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4303.8423 0.5253714\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.41638795 0.41638795\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:01.612.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.287.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.316.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "11541.967 1.4089315\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:02.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.502.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.529.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.556.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.667.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.677.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7780.0923 0.9497183\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:03.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:08.681.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4069.7551 0.49679628\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "True\n", + "0.77956146 0.77956146\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:04.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.397.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.411.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.453.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.481.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.495.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.537.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.565.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9629.118 1.1754295\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:05.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.579.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.593.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.607.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.635.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.690.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.704.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.718.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.732.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.746.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "8123.1504 0.9915955\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:06.140.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.789.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:09.793.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2239.2522 0.27334622\n", - "all_finite forward False\n" + "True\n", + "1.0538523 1.0538523\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:06.767.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:10.947.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:10.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:10.970.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:10.985.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.0. [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.290.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.440.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.590.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.740.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.880.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.131.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6092.8887 0.7437608\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:08.139.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.175.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.190.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.204.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.219.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.247.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.261.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.290.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.363.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5872.828 0.71689796\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:08.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.367.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9445.795 1.1530511\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5826431 0.5826431\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:09.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:11.901.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.222.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.287.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3538.7412 0.43197525\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:10.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.340.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.369.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.412.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.431.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.525.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7384.433 0.90142006\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:11.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.556.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.590.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.618.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4692.3076 0.57279146\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5392038 0.5392038\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:12.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:12.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.202.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.216.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.274.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.303.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6250.6396 0.76301754\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:13.290.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.387.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.401.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.486.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.529.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.585.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "8471.654 1.0341375\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:14.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:13.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3702.066 0.45191234\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "norm True\n", + "True\n", + "1.0727259 1.0727259\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:15.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.290.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.406.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3078.8574 0.3758371\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:16.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.502.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.585.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.612.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.625.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.628.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5785.0063 0.70617753\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:17.641.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.638.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:14.643.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7677.179 0.93715566\n", - "all_finite forward False\n" + "True\n", + "1.2431451 1.2431451\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:18.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.856.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.877.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.912.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.926.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.941.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:15.997.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.380.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9333.895 1.1393914\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:20.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.520.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.800.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.930.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.152.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.221.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.235.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9388.088 1.1460068\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:21.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.262.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5197.871 0.63450575\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.75203556 0.75203556\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:22.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.631.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.672.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.686.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.714.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.755.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.768.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.782.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.810.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3747.0469 0.45740318\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:23.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.838.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.893.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.907.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.949.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.963.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.977.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:16.990.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.210.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7584.0938 0.9257927\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:24.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.310.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.350.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9502.698 1.1599973\n", - "all_finite forward False\n" + "True\n", + "0.7435477 0.7435477\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:25.697.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.411.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.442.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.455.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.469.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.481.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.495.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.508.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.521.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.534.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.547.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.560.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.591.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3334.9731 0.4071012\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:26.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.618.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.644.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.657.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.670.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.709.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.721.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.747.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.789.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:17.800.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6249.941 0.76293224\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8267632 0.8267632\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:27.456.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.471.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.525.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.538.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.552.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.565.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.578.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.591.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.634.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5016.228 0.6123325\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:29.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.701.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.714.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.726.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.751.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.763.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.776.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.788.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.801.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.813.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.816.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:18.830.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "8176.815 0.99814636\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8361121 0.8361121\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:29.851.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:19.880.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:19.888.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:19.903.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:19.920.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:19.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:19.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:19.967.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:19.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:19.998.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.140.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.290.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.440.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.590.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.740.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "8464.828 1.0333042\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:30.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.900.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.106.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.212.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.242.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2668.5417 0.32574973\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:31.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.303.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.335.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6561.969 0.80102164\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.58724463 0.58724463\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:33.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.703.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.908.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.927.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.942.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.957.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:20.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.300.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.340.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.490.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.650.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.960.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2891.233 0.3529337\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:34.740.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.111.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.126.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.187.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.203.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.248.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.269.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9314.32 1.137002\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:34.916.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.342.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4721.1777 0.57631564\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.3616694 1.3616694\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:35.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.771.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.977.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:21.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.500.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.760.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.900.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.116.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.129.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6200.0293 0.7568395\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:37.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.187.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.229.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.243.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.284.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.367.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3346.6133 0.40852213\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.81388605 0.81388605\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:37.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:22.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.700.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.400.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.530.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.800.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.940.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.173.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6461.0024 0.7886966\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:38.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.212.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.255.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.295.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.322.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.335.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.362.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.388.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.401.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6798.041 0.829839\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:39.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:23.406.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5975.327 0.72941005\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "True\n", + "0.6447276 0.6447276\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:41.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.418.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.656.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.673.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.690.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.707.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.723.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.789.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5864.8228 0.71592075\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:41.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.855.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.871.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.887.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.904.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.920.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.937.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:24.985.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.200.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4894.2925 0.5974478\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:42.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.350.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.510.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.880.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.105.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3699.2966 0.4515743\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9140862 0.9140862\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:43.459.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.534.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.753.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.768.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.783.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.857.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.872.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.886.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.901.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.916.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "11721.943 1.4309013\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:44.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.931.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.945.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.960.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:25.989.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.340.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.490.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.640.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.930.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3283.5522 0.40082425\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:45.627.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.156.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.170.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "7233.9663 0.8830525\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7439656 0.7439656\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:46.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.743.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.776.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.804.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.818.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.847.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.861.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.889.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.903.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.917.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.931.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "8731.289 1.0658312\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:47.275.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.945.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.959.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:26.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.200.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.160.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.440.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.590.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.116.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.130.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.144.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4651.584 0.5678203\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:48.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "10831.513 1.3222061\n", - "all_finite forward False\n" + "norm True\n", + "True\n", + "0.8560374 0.8560374\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:49.631.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.781.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.795.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.838.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.866.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.881.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.895.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.923.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.937.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4985.6704 0.60860234\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:50.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.980.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:27.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.370.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.510.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.650.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.940.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6020.3394 0.7349047\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:51.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.164.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:28.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2673.9316 0.32640767\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "norm True\n", + "True\n", + "0.83151346 0.83151346\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:52.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.468.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.525.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.538.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.552.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.580.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.594.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9668.475 1.1802337\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:53.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.677.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.705.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.719.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.732.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.746.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.788.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.802.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.817.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.846.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.849.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5637.6772 0.688193\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:54.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.860.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:29.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5649.9814 0.689695\n", - "all_finite forward False\n" + "True\n", + "0.5997607 0.5997607\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:54.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.328.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.529.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.546.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.559.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.572.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.585.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.677.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.704.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.718.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4796.758 0.5855417\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:56.120.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.732.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.745.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.759.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.772.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.786.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.866.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.893.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.906.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.910.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.920.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:30.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3073.137 0.3751388\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1614288 1.1614288\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:56.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.443.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.450.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.504.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.584.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5740.5127 0.7007462\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:57.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.651.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.677.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.704.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.717.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.731.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.744.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.758.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.771.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.784.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.798.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.811.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.828.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:31.841.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "10597.762 1.2936721\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5219102 0.5219102\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:57.932.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.368.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.382.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.422.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.455.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.469.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.495.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5920.4272 0.7227084\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:59.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.615.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.628.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.655.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.668.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.682.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.721.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.735.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.751.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.759.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:32.763.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5089.523 0.62127966\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7992721 0.7992721\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:59.769.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:33.838.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.710.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.152.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3671.932 0.44823387\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:00.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.328.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4486.311 0.5476454\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:01.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.497.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.500.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.513.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "9323.181 1.1380836\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.31654006 0.31654006\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:02.278.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.976.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:34.992.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.700.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.370.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.530.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.830.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.990.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.114.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.129.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5019.006 0.6126716\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:02.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.176.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.256.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.333.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.363.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6086.423 0.74297154\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:03.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.394.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.422.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.427.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6050.372 0.7385708\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.93780446 0.93780446\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:04.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.925.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.933.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.947.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.960.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.972.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.985.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:35.997.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.350.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.860.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.990.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.112.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.125.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5843.4897 0.7133166\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:06.560.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.176.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.189.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.202.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.228.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.267.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.310.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "10407.959 1.2705028\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.81498957 0.81498957\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:06.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:36.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.140.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.620.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.950.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.112.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3190.8948 0.38951352\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:07.401.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.228.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.262.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.347.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.381.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5735.7183 0.7001609\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:08.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.432.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.466.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.484.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:37.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "8044.478 0.98199195\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [3 0 0 0 0 0 0 0]\n", - "found_inf [3 0 0 0 0 0 0 0]\n", - "found_inf [ True False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 1.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "1.0\n", - "scaler update\n", - "scaler update False\n", - "8192.0\n", - "4096.0\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.80549556 0.80549556\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:08.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.612.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.619.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.635.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.687.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.699.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.723.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.736.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.798.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.810.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2063.103 0.50368726\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:09.406.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.823.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.848.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.861.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.873.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.886.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.898.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.911.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.937.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.978.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:38.990.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5106.7617 1.246768\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6552199 0.6552199\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:10.160.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.646.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.677.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.702.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.715.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.753.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.779.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.804.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.817.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.829.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3206.504 0.78283787\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:10.629.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.857.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.934.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.947.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.960.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:39.998.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.140.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3586.835 0.8756921\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7148214 0.7148214\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:11.798.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.784.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.798.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.814.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.829.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.844.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.859.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.890.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.920.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3134.63 0.7652905\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:12.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.981.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:40.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.560.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.102.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3094.2312 0.75542754\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:12.974.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.198.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.219.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2821.3284 0.6888009\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.6356169 1.6356169\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:13.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.779.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:41.992.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.500.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.310.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.560.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.690.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.820.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.950.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4547.082 1.1101274\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:14.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.171.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.220.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.258.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.295.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.335.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:42.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2537.3735 0.61947596\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.3849152 0.3849152\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:15.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.442.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.450.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.465.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.480.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.495.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.556.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.585.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.615.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2220.1116 0.5420194\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:15.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.690.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.704.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.719.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.763.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.807.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.821.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5356.7944 1.3078111\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:16.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.850.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.876.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:43.880.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2157.0803 0.52663094\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5115913 0.5115913\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:18.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.454.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.469.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.484.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.513.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.528.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.603.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2994.0312 0.73096466\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:18.800.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.619.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.633.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.648.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.678.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.693.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.737.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.767.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.782.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3442.0366 0.840341\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:19.580.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.830.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:44.846.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3599.7903 0.87885505\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.3941355 1.3941355\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:20.362.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.599.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.665.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.694.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.709.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.753.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.767.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.782.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2163.684 0.5282432\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:21.880.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.811.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.869.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.912.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.926.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.941.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:45.997.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4498.332 1.0982256\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:22.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2629.19 0.6418921\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9264112 0.9264112\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:23.489.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.565.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.603.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.632.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.646.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.661.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.705.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.719.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.763.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4061.885 0.99167114\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:24.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.791.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.820.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.834.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.849.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.863.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.877.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.892.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.906.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.920.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.949.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.963.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.977.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1987.3943 0.48520368\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:25.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.980.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:46.992.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3955.8167 0.96577555\n", - "all_finite forward False\n" + "norm True\n", + "True\n", + "0.48899296 0.48899296\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:26.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.411.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.426.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.455.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.470.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.500.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.514.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.528.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3548.518 0.8663374\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:27.200.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.572.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.586.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.628.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.657.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.671.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.685.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.699.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.713.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.755.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.768.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4050.5361 0.9889004\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:27.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.782.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.795.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:48.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1569.208 0.38310742\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1476018 1.1476018\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:29.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.300.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.363.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.442.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.455.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.468.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.481.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "832.6935 0.2032943\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:29.906.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.520.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.584.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.635.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.672.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.685.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:49.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3535.163 0.8630769\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9708269 0.9708269\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:30.750.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.303.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.397.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.422.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1524.4944 0.372191\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:31.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.514.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.539.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.552.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.564.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.577.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.590.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.615.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.627.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.665.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.668.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:50.678.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6841.27 1.6702319\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.26835757 0.26835757\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:33.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.940.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.303.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.316.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.329.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.342.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.355.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.368.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.406.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.432.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.457.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.469.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4650.126 1.1352847\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:33.934.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.532.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.595.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.607.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.619.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.632.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.644.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.656.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:51.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4189.2603 1.0227686\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.72622925 0.72622925\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:34.620.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.692.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.715.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.742.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.771.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.813.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.867.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.881.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.894.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1747.7362 0.4266934\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:35.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.908.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.949.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.976.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:52.989.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.300.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.160.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.560.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.690.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.820.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.930.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2132.2605 0.5205714\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7098302 0.7098302\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:37.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.980.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4626.734 1.1295737\n", - "all_finite forward False\n" + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:37.758.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.427.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.643.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.657.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.670.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.696.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.709.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.723.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.736.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.749.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.775.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.788.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.801.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.814.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1891.8018 0.46186566\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:38.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.855.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.869.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.882.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.895.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.908.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.934.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.948.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.961.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.974.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:53.987.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.0. [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.300.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4587.7695 1.1200609\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1436845 1.1436845\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:39.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.587.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.628.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.641.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.665.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.677.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.701.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.713.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.737.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.749.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.772.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2230.4863 0.5445523\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:41.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.821.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.833.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.845.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.856.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.880.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.892.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.904.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.916.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.927.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.930.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:54.944.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6230.9077 1.5212177\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.4309411 0.4309411\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:42.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.540.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.609.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.634.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.673.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.686.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.699.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2295.3296 0.5603832\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:43.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.737.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.750.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.763.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.775.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.788.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.801.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.814.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.890.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.903.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.907.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.918.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:55.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2321.9253 0.5668763\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.89707816 0.89707816\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:44.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:56.908.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:56.916.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:56.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:56.942.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:56.957.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:56.970.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:56.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:56.995.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.210.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.330.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.710.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.111.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3213.8562 0.78463286\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:45.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.125.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.139.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.222.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2609.2563 0.6370255\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5647389 0.5647389\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:46.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.845.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.895.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.910.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.925.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.946.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.960.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:57.990.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.600.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.210.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4119.854 1.0058237\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer " ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:47.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.520.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.820.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.112.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.127.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.143.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.173.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.190.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.205.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.220.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.235.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1779.3223 0.43440485\n", - "all_finite forward False\n" + "False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:48.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.265.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.285.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2661.3545 0.64974475\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7564678 0.7564678\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:49.810.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.731.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.930.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.981.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:58.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.440.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.570.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.710.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.111.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3987.2744 0.97345567\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:50.673.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.124.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.137.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.164.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.190.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.204.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.217.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.244.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.310.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5199.431 1.2693924\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:51.534.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.327.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2221.0906 0.54225844\n", - "all_finite forward False\n" + "norm True\n", + "True\n", + "1.0348396 1.0348396\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:52.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.755.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.949.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.970.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:08:59.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.490.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.620.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.750.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.880.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.114.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.127.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.140.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1756.6427 0.42886785\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:53.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.255.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.284.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.299.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.328.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3740.6125 0.9132355\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:54.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:00.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1228.1027 0.29982975\n", - "all_finite forward False\n" + "True\n", + "0.44076905 0.44076905\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:55.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.593.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.706.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.721.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.736.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.751.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.780.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1765.4086 0.43100795\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:55.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.811.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.841.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.855.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.885.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.900.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.918.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.932.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.945.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.959.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.972.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:01.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5914.357 1.4439348\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:57.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.0. [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.290.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1924.4934 0.46984702\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.3671055 0.3671055\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:58.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.633.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.657.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.669.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.681.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.694.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.705.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.717.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.753.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.765.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.789.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.801.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3857.4397 0.94175774\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:59.143.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.813.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.849.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.861.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.873.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.885.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.932.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.944.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.956.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.959.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:02.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5267.875 1.2861023\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7022525 0.7022525\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:00.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.418.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.426.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.452.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.465.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.505.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.518.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.531.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.584.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "802.6415 0.19595739\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:01.661.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.623.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.702.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.715.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.728.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.754.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.767.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.780.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.793.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.804.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:03.808.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4382.6255 1.0699769\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9325581 0.9325581\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:02.495.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.261.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.322.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.369.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1668.9064 0.40744784\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:03.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.506.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.521.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.612.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.627.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2848.946 0.69554347\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:04.274.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.657.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.672.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.685.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:04.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4386.1807 1.0708449\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1594725 1.1594725\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:05.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:05.706.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:05.908.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:05.928.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:05.944.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:05.961.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:05.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:05.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.470.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.640.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.980.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3836.6562 0.93668365\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:06.671.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.216.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.300.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.316.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2283.4812 0.5574905\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:07.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.333.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.366.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.388.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2550.9177 0.62278265\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1340053 1.1340053\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:08.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:06.838.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.600.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.750.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.900.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.105.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.181.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.226.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3712.3071 0.906325\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:09.959.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.406.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3248.1965 0.79301673\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:10.801.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.466.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.469.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.483.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3715.835 0.90718627\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6314351 0.6314351\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:11.437.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.936.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.957.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.970.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:07.997.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.370.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.500.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.760.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.890.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.116.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.129.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1434.2343 0.35015485\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:12.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.181.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.220.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.234.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.247.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.299.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.315.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.329.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.333.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5075.508 1.2391376\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.45982212 0.45982212\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:13.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:08.867.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.780.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.119.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.131.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.189.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.212.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.224.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.247.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3918.3735 0.95663416\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:14.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.295.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.347.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:09.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4883.9463 1.1923697\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9327588 0.9327588\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:15.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.542.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.793.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.810.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.844.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.861.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.895.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.911.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.928.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2761.7478 0.67425483\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:16.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.945.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.961.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.978.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:10.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.270.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.440.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.600.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.770.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.930.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.126.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.143.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "731.8806 0.17868179\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:18.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.176.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.214.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.229.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2371.8125 0.5790558\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1540717 1.1540717\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:19.287.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.666.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.681.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.710.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.755.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.769.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.784.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.814.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.829.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.844.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3086.8704 0.75363046\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:20.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.859.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.873.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.888.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.903.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.917.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.932.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.947.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.976.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:11.991.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.600.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.200.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.350.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.490.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3252.5999 0.79409176\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:21.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.640.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.770.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2395.0474 0.58472836\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.40440667 0.40440667\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:23.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.578.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.592.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.607.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.667.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.682.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.696.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.726.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4134.273 1.009344\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:24.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.771.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.800.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.830.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.845.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.860.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.890.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.920.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3289.594 0.80312353\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:25.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.963.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:12.967.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2756.6907 0.6730202\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7644355 0.7644355\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:26.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.418.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.441.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.456.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.471.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.500.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.560.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.575.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.590.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.605.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.620.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2028.4688 0.49523163\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:28.126.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.651.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.666.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.681.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.696.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.726.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.771.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.786.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.801.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.816.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1733.1372 0.4231292\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:28.925.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.847.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.850.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.859.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:13.863.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6022.089 1.4702365\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6104776 0.6104776\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:29.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:14.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.156.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.289.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2203.7324 0.5380206\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:30.633.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.368.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.431.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.525.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2952.8098 0.72090083\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:32.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.620.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:15.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4729.455 1.1546521\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.997129 0.997129\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:32.948.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.320.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.267.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.312.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.328.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.388.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.418.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1020.94714 0.24925467\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:33.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.539.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.554.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.584.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.599.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.629.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2930.2244 0.7153868\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:34.592.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.644.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:16.678.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2463.5574 0.60145444\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0271108 1.0271108\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:36.217.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.143.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.170.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.262.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.275.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.327.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3422.4927 0.8355695\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:37.860.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.340.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.366.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.443.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.456.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.469.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.518.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4271.732 1.0429033\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.32039133 0.32039133\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:37.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:17.957.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.187.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.216.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.244.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.316.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2430.3735 0.5933529\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:38.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.387.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.418.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.465.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.481.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.525.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.540.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.554.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3444.4534 0.840931\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:40.531.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.587.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:18.592.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6385.7764 1.5590274\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7976982 0.7976982\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:41.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.579.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.586.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.599.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.626.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.667.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.712.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.751.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.764.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1240.1918 0.3027812\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:42.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.803.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.816.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.828.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.841.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.866.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.948.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.960.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.963.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:19.977.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2183.2522 0.53302056\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.3452397 0.3452397\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:43.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.382.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.395.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.459.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.537.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.575.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2284.8604 0.55782723\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:44.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.627.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.665.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.678.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.704.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.716.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.744.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.754.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:20.758.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2473.9578 0.6039936\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1316606 1.1316606\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:45.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.202.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.216.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.229.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.242.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.267.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3331.0835 0.8132528\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:46.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.411.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.423.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.462.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.502.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.514.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.528.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.579.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:21.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1416.7529 0.34588695\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5785658 0.5785658\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:47.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.380.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.510.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.760.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.890.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.102.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.141.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.219.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5735.763 1.4003328\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:48.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.258.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.322.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.335.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.387.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.389.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:22.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3121.3445 0.762047\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7461037 0.7461037\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:49.848.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.474.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.489.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.505.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.520.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.612.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.627.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.656.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3999.0747 0.9763366\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:50.744.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.672.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.685.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.697.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.710.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.735.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.747.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.772.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.834.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.847.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.859.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.862.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.869.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:23.873.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1499.0521 0.36597952\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.39682442 0.39682442\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:51.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.470.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.502.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.559.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.631.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2203.596 0.5379873\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:52.978.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.703.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.718.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.732.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.747.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.775.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.804.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.819.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.833.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.847.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.862.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.876.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4022.6245 0.98208606\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:53.631.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.881.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.891.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:24.895.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2960.8364 0.72286046\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7885564 0.7885564\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:54.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.342.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.363.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.389.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.414.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.427.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.454.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.480.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.506.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.532.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2717.0479 0.66334176\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:55.262.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.547.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.561.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.575.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.618.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.633.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.648.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.702.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.715.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.730.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:25.744.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2791.7627 0.6815827\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.83048546 0.83048546\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:56.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.412.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.453.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.466.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.505.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.518.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.531.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2090.3008 0.51032734\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:57.487.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.595.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.620.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.633.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.646.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.672.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.686.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.737.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.753.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.765.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:26.769.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1559.8855 0.38083142\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.70709467 0.70709467\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:58.395.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.807.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.881.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.895.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.948.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:27.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.100.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5144.402 1.2559575\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:59.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.140.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.270.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.540.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.800.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.930.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.106.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.171.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.213.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3353.7117 0.81877726\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7129579 0.7129579\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:00.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.623.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.688.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.701.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.714.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.753.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.779.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.805.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3815.8193 0.9315965\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:01.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.818.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.831.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.844.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.857.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.939.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.956.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.971.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.985.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:28.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.200.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.120.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.160.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6458.0757 1.5766786\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6007396 0.6007396\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:02.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.459.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.486.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.513.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.540.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.580.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.594.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.607.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.634.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "703.1942 0.17167827\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:03.665.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.661.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.688.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.701.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.715.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.728.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.742.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.755.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.769.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.782.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.849.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:29.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1366.9177 0.33372015\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.4525315 0.4525315\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:05.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.289.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.310.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.406.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.462.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.489.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3784.085 0.92384887\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:06.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.504.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.531.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.559.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.572.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.587.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.628.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.656.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.669.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.687.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:30.702.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1432.3479 0.3496943\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.4186795 1.4186795\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:07.860.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.758.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.765.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.820.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.834.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.848.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.862.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.876.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.889.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.903.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.917.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.930.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.944.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1503.9475 0.36717469\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:08.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.972.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:31.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.270.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.400.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.540.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.950.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.109.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.169.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5457.336 1.3323574\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.3993219 0.3993219\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:09.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.585.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.623.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.702.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.715.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.742.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.755.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2521.4412 0.6155862\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:10.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.768.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.781.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.794.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.807.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.820.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.833.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.846.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.860.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.873.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.886.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.912.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.925.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.938.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.941.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:32.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2520.7927 0.6154279\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.88992006 0.88992006\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:11.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.592.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.626.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.666.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.692.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.706.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.719.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.732.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.746.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.759.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.772.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4100.4893 1.001096\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:12.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.838.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.892.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.918.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.932.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.945.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.972.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.985.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3381.381 0.8255325\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:13.874.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:33.989.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1938.0549 0.47315794\n", - "all_finite forward False\n" + "1.0694686 1.0694686\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:14.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.443.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.457.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.470.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.483.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.629.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "6447.8345 1.5741783\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:15.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.656.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.669.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.682.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.735.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.749.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.775.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.789.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.802.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.818.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.828.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:34.831.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2814.3816 0.6871049\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5690588 0.5690588\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:16.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:35.882.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.770.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.112.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.144.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.176.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.224.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.255.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2760.7659 0.6740151\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:17.860.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.287.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.303.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.381.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.397.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1892.635 0.4620691\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:18.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.528.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:36.548.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2359.016 0.57593167\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.3174789 1.3174789\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:19.709.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.330.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.590.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.980.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.112.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.125.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.205.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3728.3625 0.91024476\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:20.651.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.244.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.258.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.284.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.377.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3748.4856 0.9151576\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6079407 0.6079407\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:22.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.849.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.863.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.876.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.889.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.902.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.915.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.942.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:37.995.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.210.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.350.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2375.694 0.58000344\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:23.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.740.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.114.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.127.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.140.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.219.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2720.705 0.66423464\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.73206925 0.73206925\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:24.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.682.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.696.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.709.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.736.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.749.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.775.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.789.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.802.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.828.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.855.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2644.7095 0.645681\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:25.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.882.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.895.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.908.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.934.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.948.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.961.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.974.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:38.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:39.100.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:39.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:39.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:39.420.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:39.550.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:39.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:39.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:39.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3930.5056 0.9595961\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.32264215 0.32264215\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:26.904.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.124.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.164.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.204.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.285.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2034.2415 0.49664098\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:27.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.377.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.457.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.470.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.483.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.508.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.512.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2664.4976 0.6505121\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1717907 1.1717907\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:28.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:40.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.100.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.173.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.213.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.226.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2996.8933 0.7316634\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:29.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.427.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.453.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.466.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3425.7595 0.8363671\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6995856 0.6995856\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:31.438.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.938.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.946.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.960.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:41.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.120.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.390.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.530.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.106.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.119.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3279.7349 0.8007165\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:32.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.185.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.212.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.265.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.278.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.333.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1817.4989 0.44372532\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6834339 0.6834339\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:33.295.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.787.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.800.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.813.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.866.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.880.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.893.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.907.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.920.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.933.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.946.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.959.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2916.2424 0.71197325\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:34.860.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:42.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.120.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.390.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.520.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.105.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.118.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.131.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:43.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3910.3855 0.95468396\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5783601 0.5783601\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:35.607.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.214.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.224.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.267.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.320.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.333.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.412.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2448.4377 0.5977631\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:36.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.426.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.452.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.465.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.504.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.599.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:44.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3935.6804 0.9608595\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.376577 0.376577\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:37.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.500.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.140.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.540.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.940.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.187.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3086.025 0.75342405\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:38.880.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.214.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.267.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.320.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.347.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.387.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3903.8867 0.95309734\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.69522613 0.69522613\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:39.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:45.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.340.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.540.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.800.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.940.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.204.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1581.8405 0.38619152\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:40.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.246.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.300.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.328.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.341.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.355.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.369.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.424.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.428.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4406.353 1.0757698\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:41.235.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.438.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.442.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3575.5322 0.8729327\n", - "all_finite forward False\n" + "True\n", + "1.2842464 1.2842464\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:42.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:46.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.800.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.100.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.127.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.171.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.244.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3104.3206 0.75789076\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:43.590.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "661.32446 0.16145617\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:44.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:47.513.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2175.3613 0.5310941\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7147303 0.7147303\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:45.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.581.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.643.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.656.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.670.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.684.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.765.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3228.7397 0.78826654\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:46.930.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.805.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.819.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.846.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.859.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.873.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.886.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.913.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.926.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:48.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4954.6807 1.2096388\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6254871 0.6254871\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:47.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.395.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.431.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.471.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.512.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.525.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.539.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.580.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.593.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5022.986 1.2263149\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:48.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.607.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.634.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.648.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.661.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.688.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.702.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.715.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.743.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.770.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.783.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.786.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.793.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:49.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1163.1852 0.28398076\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.44593582 0.44593582\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:49.299.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.242.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.290.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.426.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2857.671 0.69767356\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:50.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.453.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.480.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.521.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.603.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.633.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.643.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:50.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3177.867 0.7758464\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.54238087 0.54238087\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:51.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.100.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.114.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.141.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4655.253 1.1365364\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:52.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.427.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.441.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.455.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.468.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.495.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:51.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3287.647 0.8026482\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1532724 1.1532724\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:53.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.651.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.665.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.693.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.706.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.720.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.733.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.747.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2145.1516 0.52371866\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:54.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.774.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.788.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.801.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.828.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.855.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.869.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.882.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.936.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.949.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.959.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:52.963.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2959.772 0.7226006\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.612654 0.612654\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:55.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.593.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.607.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.668.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.713.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.744.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.759.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4290.3853 1.0474573\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:56.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.775.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.805.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.820.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.835.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.850.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.866.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.881.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.912.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.927.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.974.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3385.0151 0.8264197\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer " ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:57.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:53.990.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.500.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2108.6501 0.51480716\n", - "all_finite forward False\n" + "False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7472157 0.7472157\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:57.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.481.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.681.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.707.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.720.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.733.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.747.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.786.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2900.335 0.7080896\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:59.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.891.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.904.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.917.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.930.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.956.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:54.995.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.210.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.340.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.470.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.600.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.770.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4811.238 1.1746186\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.724629 0.724629\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:00.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.561.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.595.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.634.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.687.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.713.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.726.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1017.25867 0.24835417\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:01.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.765.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.805.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.817.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.831.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.844.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.857.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.938.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.947.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:55.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2914.7021 0.7115972\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7157301 0.7157301\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:02.160.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.200.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.290.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.422.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3869.4255 0.94468397\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:03.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.486.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.512.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.537.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.550.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.575.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:57.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4281.2715 1.0452323\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.2614112 1.2614112\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:04.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.222.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.267.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.320.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.333.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.411.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3111.8303 0.7597242\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:05.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.426.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.556.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.595.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.599.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.609.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:58.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2724.7866 0.6652311\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.38337803 0.38337803\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:06.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.690.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.820.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.960.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.109.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.214.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2723.32 0.66487306\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:07.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.322.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.369.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3569.3816 0.87143105\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:08.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2030.474 0.4957212\n", - "all_finite forward False\n" + "True\n", + "0.67901933 0.67901933\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:09.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.932.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.939.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.954.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:09:59.998.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.880.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2737.8909 0.6684304\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer " ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:10.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.269.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.284.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.299.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.329.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4238.767 1.0348552\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:11.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:00.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1876.1686 0.45804897\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.98313546 0.98313546\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:12.638.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.658.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.670.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.721.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.747.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.759.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.772.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.784.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2766.6577 0.67545354\n", - "all_finite forward False\n" + "embed_tokens False\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:13.450.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.821.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.834.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.846.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.858.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.871.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.893.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.939.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.980.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:01.993.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.600.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4410.1436 1.0766952\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.49452388 0.49452388\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:14.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3249.0298 0.79322016\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:15.772.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.389.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.432.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.445.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.471.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.484.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.547.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.560.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.586.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3512.2156 0.8574745\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:16.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.599.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.661.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.673.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.685.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.710.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.746.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.759.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.769.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:02.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2828.2466 0.6904899\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.2470416 1.2470416\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:17.387.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.156.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.179.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.203.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.216.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.228.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.265.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.289.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4035.949 0.9853391\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:18.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.363.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.388.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.412.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.424.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.437.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.474.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.486.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.513.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.521.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.525.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2642.2598 0.64508295\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7704746 0.7704746\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:19.887.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:03.963.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.203.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.226.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.316.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.327.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2404.0688 0.5869309\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:20.733.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.382.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.426.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.437.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.471.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:04.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3282.9634 0.80150473\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.87775475 0.87775475\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:21.632.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.412.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.654.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.666.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.678.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.690.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.702.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.714.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.726.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.750.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.774.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2404.595 0.5870593\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:22.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.821.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.833.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.844.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.856.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.891.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.902.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.914.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.926.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.937.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.949.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.960.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:05.964.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3850.3262 0.94002104\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.76167804 0.76167804\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:24.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.418.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.431.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.457.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.470.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.495.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.508.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.521.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.534.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.547.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.560.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3645.4087 0.88999236\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:24.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.586.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.599.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.702.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.714.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.743.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.753.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:06.757.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2321.8076 0.56684756\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.75592226 0.75592226\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:25.871.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.202.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.216.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.229.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.267.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3742.2166 0.9136271\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:26.769.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.422.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.486.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.512.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.525.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.538.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.567.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:07.578.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2697.6575 0.6586078\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6868435 0.6868435\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:28.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.320.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.590.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.710.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.135.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.173.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.212.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.224.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2612.6794 0.6378612\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:29.217.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.275.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.352.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.377.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.394.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:08.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1409.8065 0.34419104\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1035453 1.1035453\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:30.114.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.424.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.437.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.450.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.609.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4215.067 1.0290691\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:31.120.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.623.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.688.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.713.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.726.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.765.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.793.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.802.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:09.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2363.574 0.5770444\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.62707794 0.62707794\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:32.618.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.347.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.412.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4023.3787 0.9822702\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:33.468.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.438.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.529.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.542.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.555.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.567.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.580.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.593.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.606.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:10.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1198.9829 0.29272044\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5357292 0.5357292\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:34.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.600.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.175.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.228.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4681.8955 1.1430409\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:35.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.267.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.423.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.450.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.454.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3139.9504 0.76658946\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.3163823 1.3163823\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:36.893.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.906.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.919.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.931.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.944.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.957.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:11.995.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.700.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.200.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.330.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.710.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3898.5327 0.9517902\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:37.749.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.176.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.189.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.202.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.228.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.242.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.255.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:12.285.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4852.447 1.1846794\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5229181 0.5229181\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:38.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.418.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.431.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.457.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.470.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.483.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "775.9978 0.18945259\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:39.584.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.548.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.561.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.586.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.629.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.655.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.668.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.680.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.693.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:13.697.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3675.351 0.8973025\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.72673875 0.72673875\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:41.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.366.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.462.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4687.4033 1.1443856\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:42.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.585.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.625.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.666.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.692.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.705.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.719.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2975.3308 0.7263991\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:42.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:14.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1972.605 0.481593\n", - "all_finite forward False\n" + "True\n", + "0.8398252 0.8398252\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:43.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.190.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.224.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.289.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.315.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.327.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.340.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.366.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1634.4521 0.39903617\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:45.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.443.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.455.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.468.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.481.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.506.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.532.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.560.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:15.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4789.534 1.1693199\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8711176 0.8711176\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:46.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.210.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.290.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.102.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.118.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2980.1025 0.7275641\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:47.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.328.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.387.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2221.8416 0.5424418\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:48.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.432.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.452.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:16.471.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2832.235 0.69146365\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5297197 0.5297197\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:49.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.505.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.518.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.531.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.609.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.634.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.673.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3214.4805 0.7847853\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:50.767.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.686.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.699.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.737.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.749.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.775.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.787.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.849.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:17.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2249.228 0.54912794\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0888232 1.0888232\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:51.686.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.278.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.356.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.369.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.382.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.395.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.459.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4381.9355 1.0698085\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:52.552.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.500.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.513.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.538.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.564.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.577.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.590.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.603.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.628.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.641.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.644.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:18.657.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3384.9075 0.8263934\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.631684 0.631684\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:54.820.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.111.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.125.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.176.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.189.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.202.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.228.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2204.1877 0.5381318\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:54.928.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.356.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.369.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.382.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.395.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.459.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.489.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3218.2935 0.7857162\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.99913275 0.99913275\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:55.834.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:19.934.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.179.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.204.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.217.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.243.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.256.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.269.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1773.498 0.43298292\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:56.733.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.422.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.462.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:20.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4461.385 1.0892053\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.49044862 0.49044862\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:58.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.587.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.787.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.808.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.864.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.882.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.900.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.917.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.934.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2602.352 0.63533986\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:59.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:21.985.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.100.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.340.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.510.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3387.674 0.82706887\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer " ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:00.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.216.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.248.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.265.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.290.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "760.3062 0.18562163\n", - "all_finite forward False\n" + "False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.95372164 0.95372164\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:00.976.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.767.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.781.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.794.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.819.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.851.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.864.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.877.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.890.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.903.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.916.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.942.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5261.101 1.2844485\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:02.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.981.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:22.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.600.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.320.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.450.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.570.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.700.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.830.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.960.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.137.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2562.8271 0.6256902\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8735594 0.8735594\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:03.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.688.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.701.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.713.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.726.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.765.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2316.9653 0.56566536\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:04.303.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.803.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.816.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.828.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.841.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.866.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.892.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.918.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.930.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.971.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.981.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:23.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2486.6904 0.60710216\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.98477405 0.98477405\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:05.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.638.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.661.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.687.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.712.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.751.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.764.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.776.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.789.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.802.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.814.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3944.2283 0.96294636\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:06.813.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.866.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.891.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.904.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.916.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.942.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.954.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.967.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.980.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:24.993.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:25.600.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:25.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:25.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:25.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3088.5564 0.7540421\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.39029893 0.39029893\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:07.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.397.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.422.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2681.675 0.6547058\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:08.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.486.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.561.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.587.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.599.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.612.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.627.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:26.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4771.17 1.1648364\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.21626417 0.21626417\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:09.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.510.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.710.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.960.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.109.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.135.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.173.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.212.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "694.8997 0.16965325\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:11.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.327.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.340.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.432.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1366.021 0.33350122\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8622213 0.8622213\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:11.860.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.889.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.902.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.914.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.927.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.965.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.978.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:27.991.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.550.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3513.4495 0.85777575\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:12.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.940.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.119.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.171.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.222.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.234.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.247.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4784.0063 1.1679703\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.3788529 0.3788529\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:13.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.694.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.701.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.715.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.730.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.745.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.776.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.791.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.807.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.838.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.884.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3176.009 0.77539283\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:15.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.900.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.915.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.931.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.947.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.977.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:28.993.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.390.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.540.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.700.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2994.316 0.73103416\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:16.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.116.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.131.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.137.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.152.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:29.156.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3309.6448 0.80801874\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.6739917 1.6739917\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:17.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.459.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.474.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.518.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.548.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.577.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.592.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.606.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.621.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4509.583 1.1009724\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:18.130.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.665.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.694.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.723.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.737.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.781.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.795.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.810.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "708.0294 0.17285874\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:19.585.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.857.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:30.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3415.4304 0.8338453\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1235838 1.1235838\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:20.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.300.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.377.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.428.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.441.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.454.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.480.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3763.2283 0.9187569\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:21.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.506.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.532.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.609.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.635.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.648.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:31.677.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3140.482 0.7667192\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0205213 1.0205213\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:21.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.171.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.221.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.234.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.247.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.284.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2068.263 0.504947\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:23.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.322.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.335.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.347.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.397.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.422.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.484.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5703.475 1.39245\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.42534208 0.42534208\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:24.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.928.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.949.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:32.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.100.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.550.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.950.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2879.4458 0.7029897\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:25.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.135.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.175.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.228.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.255.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.295.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.312.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:33.327.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2471.863 0.6034822\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.51165473 0.51165473\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:25.976.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.411.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.506.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.521.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.580.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.594.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.609.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1287.2333 0.31426594\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:27.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.623.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.638.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.666.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.681.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.710.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.767.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.781.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.795.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3874.8025 0.9459967\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:28.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:34.841.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4476.9316 1.0930009\n", - "all_finite forward False\n" + "norm True\n", + "True\n", + "1.1237204 1.1237204\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:29.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.487.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2039.0051 0.497804\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:30.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.559.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.631.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.703.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.717.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.732.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.746.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2269.6277 0.5541083\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:31.786.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.764.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:35.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5291.4897 1.2918676\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.46159288 0.46159288\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:32.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.368.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.397.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.411.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.454.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.468.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.497.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.525.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2163.3503 0.5281617\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:33.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.540.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.554.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.626.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.655.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.669.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.684.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.712.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3316.262 0.80963427\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:34.595.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.745.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.753.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:36.757.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "609.1483 0.14871785\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.12779 1.12779\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:36.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.329.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.414.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.443.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.457.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.486.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.500.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3665.958 0.8950093\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:37.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.529.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.572.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.587.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.690.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.705.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4388.3657 1.0713784\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:38.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.720.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.735.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:37.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "818.99097 0.19994897\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5546738 0.5546738\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:39.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:38.974.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.261.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.341.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3164.364 0.7725498\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:40.830.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.389.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.452.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.483.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.546.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.577.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2013.4603 0.49156746\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:41.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.593.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.658.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:39.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3498.887 0.85422045\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.522349 1.522349\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:42.692.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.189.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.204.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.246.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.261.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.275.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.289.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.333.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.347.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1435.1669 0.35038254\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:43.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.462.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.505.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.534.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.548.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.577.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.591.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5345.398 1.3050288\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:45.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.595.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:40.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4083.9087 0.997048\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.56197435 0.56197435\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:46.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.295.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.310.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2068.6528 0.5050422\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:47.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.367.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.382.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.454.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.468.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.483.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.497.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.512.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.540.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.555.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4617.3994 1.1272948\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:48.198.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:41.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2951.5132 0.7205843\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.56783384 0.56783384\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:49.888.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.140.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.179.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2527.65 0.617102\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:50.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.352.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.366.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.381.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.395.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.424.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.438.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.453.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.525.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.539.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3003.5598 0.733291\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:51.747.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.568.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:42.572.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2533.02 0.6184131\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.78038806 0.78038806\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:52.697.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.804.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.850.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.880.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.894.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.939.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:43.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3836.7646 0.9367101\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:57.819.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.390.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.540.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.820.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.111.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.124.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3517.1953 0.85869026\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:58.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.222.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.226.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3907.5168 0.9539836\n", - "all_finite forward False\n" + "norm True\n", + "True\n", + "0.6403896 0.6403896\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:59.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.742.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.750.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.765.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.779.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.793.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.807.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.850.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.893.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.908.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.936.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3199.6958 0.78117573\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:00.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.965.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:44.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.370.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.510.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.800.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.940.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.109.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.137.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.152.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2244.1816 0.5478959\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:01.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.170.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3987.194 0.97343606\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0017179 1.0017179\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:02.782.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.936.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.957.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.971.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:45.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.100.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.160.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.310.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.760.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.105.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5922.999 1.4460447\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:03.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.135.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.255.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.285.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.300.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.315.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2669.573 0.6517512\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:04.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.363.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1339.6589 0.32706517\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.4287255 0.4287255\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:06.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.937.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.980.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:46.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.370.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.520.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.950.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.124.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4780.2915 1.1670634\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:06.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.139.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.340.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2808.8567 0.685756\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:07.823.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:47.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4319.5864 1.0545865\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6592822 0.6592822\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:08.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.590.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.628.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.643.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.657.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.694.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.723.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.767.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.781.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1555.7203 0.37981454\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:10.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.810.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.912.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.926.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.954.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:48.997.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3972.7566 0.9699113\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:11.109.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.310.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3148.7598 0.7687402\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9719825 0.9719825\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:11.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.547.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.791.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.808.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.867.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.882.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.912.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.926.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.941.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2567.216 0.62676173\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:12.859.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.970.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:49.985.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.0. [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.140.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.290.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.900.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.105.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3792.02 0.92578614\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:14.456.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.198.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4074.2795 0.99469715\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.2692299 1.2692299\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:15.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.771.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:50.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.600.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.200.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.340.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.490.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.780.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.135.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2182.1865 0.5327604\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:16.590.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.221.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.235.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.320.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.362.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4822.094 1.1772691\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:16.807.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.394.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1527.0446 0.3728136\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.54293776 0.54293776\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:18.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:51.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.170.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.221.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.235.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.278.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.322.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.336.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2736.775 0.66815794\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:18.916.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.521.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "929.5039 0.22692966\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:19.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.577.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.581.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.590.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:52.594.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4356.846 1.0636832\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.41961423 0.41961423\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:20.612.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.835.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.866.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.881.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.895.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.910.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.941.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:53.998.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.120.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.270.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3247.2205 0.79277843\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:22.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.560.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.700.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.980.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.112.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.127.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.141.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.169.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.243.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1996.5781 0.48744583\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:22.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.256.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2653.3093 0.6477806\n", - "all_finite forward False\n" + "norm True\n", + "True\n", + "0.8941109 0.8941109\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:23.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.770.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:54.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.600.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.200.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.340.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.490.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.780.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.106.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.135.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3320.1025 0.8105719\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:24.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.221.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.235.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4271.953 1.0429573\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:26.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.397.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.414.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2766.1836 0.6753378\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.30332795 0.30332795\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:26.813.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:55.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.185.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.222.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.265.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2076.362 0.50692433\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:27.670.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.394.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.423.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.437.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.465.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.508.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.537.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.565.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.579.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3979.745 0.97161746\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:28.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.593.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.607.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:56.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2629.8696 0.642058\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.42979375 0.42979375\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:30.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.175.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.445.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.538.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.554.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1676.544 0.4093125\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:30.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.591.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.606.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.668.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.699.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.715.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.730.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.746.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1614.4132 0.39414385\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:31.638.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.808.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.823.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.838.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.843.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.855.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:57.860.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3877.262 0.94659716\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.4548912 1.4548912\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:32.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.104.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.265.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3491.2598 0.85235834\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:33.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.322.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.336.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.505.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4200.6865 1.0255582\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:34.807.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.520.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.532.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:10:59.537.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3300.4563 0.80577546\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.4740869 0.4740869\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:35.684.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.490.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.640.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.780.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.106.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.205.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.219.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3413.9983 0.8334957\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:36.559.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.248.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.262.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.290.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.388.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1825.1127 0.44558415\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:38.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:00.462.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2158.9978 0.5270991\n", - "all_finite forward False\n" + "True\n", + "0.9441306 0.9441306\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:38.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.340.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.620.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.760.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.900.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.104.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.118.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.189.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.203.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2280.8928 0.5568586\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:39.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.217.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.287.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.316.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.414.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3151.2498 0.7693481\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:40.706.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.428.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.432.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4769.5474 1.1644403\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "True\n", + "1.2883363 1.2883363\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:42.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.970.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:01.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.0. [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.120.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.135.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1355.8026 0.3310065\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:43.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.171.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.220.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:02.335.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2831.3933 0.69125813\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.19902064 0.19902064\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:43.864.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.387.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.426.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.452.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.465.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.505.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.518.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.531.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4406.976 1.0759219\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:44.705.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.584.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.623.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.688.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.701.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.714.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.731.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.742.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:03.746.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3419.664 0.8348789\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0702639 1.0702639\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:46.256.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.156.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.169.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.243.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.255.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.342.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2348.3533 0.57332844\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:47.118.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.355.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.367.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.441.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.454.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.466.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.504.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4425.2124 1.0803741\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.40977433 0.40977433\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:47.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:04.956.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.265.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.335.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2748.4243 0.67100203\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:48.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.489.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.531.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.559.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1977.3588 0.4827536\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:50.483.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:05.580.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "5191.6143 1.267484\n", - "all_finite forward False\n" + "True\n", + "0.68828875 0.68828875\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:51.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.200.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.289.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.316.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.328.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.341.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.367.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.406.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2272.7595 0.55487293\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:52.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.432.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.445.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.471.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.484.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.537.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.550.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.592.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:06.605.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2765.5916 0.67519325\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0776465 1.0776465\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:53.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.629.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.655.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.668.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.682.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.721.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.787.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.800.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.813.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3578.8508 0.8737429\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:55.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.891.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.904.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.917.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.930.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.956.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.995.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:07.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4559.232 1.1130937\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.94401425 0.94401425\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:55.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.428.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.453.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.465.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.489.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.502.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.514.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.527.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.539.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.564.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1396.2253 0.34087533\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:56.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.626.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.638.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.651.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.687.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.712.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.737.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.750.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.765.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:08.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4579.3677 1.1180097\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5595749 0.5595749\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:57.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.214.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.300.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.362.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.387.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2098.8145 0.5124059\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:59.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.412.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.424.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.437.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.474.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.486.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.512.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.561.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.564.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:09.577.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1856.6504 0.4532838\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.62003046 0.62003046\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:00.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.860.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.980.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.111.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3838.4595 0.9371239\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:01.169.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.224.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.261.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.274.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.287.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.300.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.312.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.368.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:10.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4931.9004 1.2040772\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.91670084 0.91670084\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:02.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.438.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.513.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.538.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.575.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2437.27 0.5950366\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:03.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.625.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.686.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.699.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.736.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.751.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:11.764.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4280.9966 1.0451652\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.78961116 0.78961116\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:04.655.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.170.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.221.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.246.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.284.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.347.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1824.5895 0.44545642\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:05.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.397.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.423.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.539.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2574.7996 0.6286132\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9032245 0.9032245\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:06.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:12.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.980.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.173.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2493.048 0.6086543\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:08.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.198.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.235.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.248.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.336.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.352.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4956.0977 1.2099848\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.35319164 0.35319164\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:08.980.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.805.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.817.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.830.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.867.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.892.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.918.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.930.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.981.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1881.2456 0.45928848\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:09.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:13.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.600.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.310.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.440.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.560.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.690.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.820.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.950.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:14.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3857.2769 0.941718\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.2345585 1.2345585\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:10.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.381.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.401.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.426.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.528.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1254.2574 0.3062152\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:12.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.579.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.591.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.629.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.641.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.654.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.667.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.692.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.704.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.717.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.745.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:15.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2241.2073 0.54716974\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9581397 0.9581397\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:12.884.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.213.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.289.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.315.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.327.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.340.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2695.1318 0.6579912\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:13.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.442.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.455.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.480.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.505.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.518.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.534.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.544.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.548.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "4728.833 1.1545002\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1904072 1.1904072\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:14.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:16.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.300.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.270.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.400.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.520.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.640.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.760.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.890.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.125.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.175.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3484.7983 0.85078084\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [-1355665149 0 0 0 0 0\n", - " 0 0]\n", - "found_inf [-1355665149 0 0 0 0 0\n", - " 0 0]\n", - "found_inf [ True False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 1.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "1.0\n", - "scaler update\n", - "scaler update False\n", - "4096.0\n", - "2048.0\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:15.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.212.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.261.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.274.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.299.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.335.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.352.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2687.9365 1.312469\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6794921 0.6794921\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:16.564.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.794.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:17.990.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.350.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.470.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.590.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.710.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.109.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.171.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1284.6727 0.6272816\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:17.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.220.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.244.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.269.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.355.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:18.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1028.1296 0.5020164\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.18182413 0.18182413\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:18.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.382.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.395.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.431.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.443.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.455.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.480.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.504.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.528.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.540.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.552.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.564.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1679.874 0.820251\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:20.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.625.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.661.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.673.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.685.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.710.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:19.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1269.6825 0.61996216\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.58622515 0.58622515\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:20.906.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.173.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.185.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.220.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.269.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.329.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1280.1108 0.6250541\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:21.823.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.341.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.377.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.389.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.401.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.437.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.489.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "752.4598 0.367412\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7514929 0.7514929\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:22.750.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.917.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.937.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.981.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:20.995.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.400.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.540.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.690.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.990.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.114.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2832.8757 1.3832401\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:24.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.129.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.144.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.173.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.203.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.247.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.262.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1847.7007 0.9021976\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:25.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.336.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.341.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1878.5565 0.9172639\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7963656 0.7963656\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:26.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.789.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.810.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.835.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.847.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.859.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.872.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.884.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.933.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.946.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.971.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.983.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1835.2625 0.89612424\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:27.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:21.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.210.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.330.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.450.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.700.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.830.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.950.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.135.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.152.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:22.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1685.8992 0.82319295\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5918657 0.5918657\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:28.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.212.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.289.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.327.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.340.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.352.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.377.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1761.0078 0.8598671\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:29.575.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.428.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.453.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.465.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.528.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.540.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2128.3337 1.0392255\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0086026 1.0086026\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:30.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.954.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.978.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:23.990.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.200.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.270.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.400.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.520.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.650.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.780.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.116.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.141.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1864.431 0.9103667\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:31.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.203.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.216.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.228.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.278.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.303.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.315.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.327.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "718.7248 0.35093984\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8112537 0.8112537\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:33.440.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:24.980.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.100.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.160.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.330.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.490.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.820.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.990.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1000.2243 0.48839077\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:34.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.198.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.248.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.381.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2324.5261 1.1350225\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer " ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:34.818.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.397.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.414.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.454.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.506.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:25.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "600.19507 0.293064\n", - "all_finite forward False\n" + "False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6757405 0.6757405\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:35.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.290.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.710.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.830.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.960.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.222.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2589.2842 1.2642989\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:36.970.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.234.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.247.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.285.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.336.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.389.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:26.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1109.3081 0.54165435\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5059494 0.5059494\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:37.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.644.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.690.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.703.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.717.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.731.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.744.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.758.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.772.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.805.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.818.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.831.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1103.2681 0.5387051\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:38.537.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.845.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.858.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.871.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.884.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.911.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.937.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.964.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.977.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:27.991.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.340.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1445.8438 0.7059784\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:39.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.500.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3183.7522 1.5545665\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "True\n", + "0.43005237 0.43005237\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:40.816.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.483.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.497.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.534.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.547.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.559.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.572.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.584.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.609.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.635.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.672.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1476.5559 0.72097456\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:41.609.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.685.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.710.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.723.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.736.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.786.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.798.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.814.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.843.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:28.857.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2311.9521 1.1288829\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.4772607 1.4772607\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:42.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.316.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.336.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.423.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1559.1323 0.7612951\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:43.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.548.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.561.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.586.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.599.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.678.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.687.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:29.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "855.9128 0.41792616\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.53844887 0.53844887\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:44.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.170.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.220.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.295.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.320.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.333.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "545.6742 0.26644248\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:45.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.459.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.497.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.513.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:30.527.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1966.5497 0.96022934\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.72059286 0.72059286\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:46.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.622.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.845.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.893.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.907.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.949.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.963.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.977.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:31.992.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.600.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.330.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2017.1786 0.9849505\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:46.946.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.750.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.890.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.116.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.130.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.144.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.185.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.213.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.226.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.243.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2230.986 1.0893487\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:48.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1629.3646 0.7955882\n", - "all_finite forward False\n" + "True\n", + "1.157895 1.157895\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:49.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.779.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.786.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.801.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.828.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.856.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.869.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.970.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1805.7606 0.88171905\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:50.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.985.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:32.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.270.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.420.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.560.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.700.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.990.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.127.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.141.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2679.648 1.3084219\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:51.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "545.60693 0.26640964\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.25074092 0.25074092\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:52.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.754.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.763.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.791.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.804.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.818.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.845.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.859.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.872.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.886.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.913.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.927.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.954.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1477.9883 0.72167397\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:53.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:33.995.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.500.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.770.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.900.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.104.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.131.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.144.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.147.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.156.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1786.734 0.8724287\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.715173 0.715173\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:54.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.737.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.749.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.774.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.786.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.811.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.849.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.862.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.874.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.887.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.900.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1263.3077 0.6168495\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:55.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.913.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.925.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.938.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.963.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.976.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:34.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:35.100.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:35.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:35.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:35.380.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:35.510.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:35.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:35.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:35.760.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:35.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1791.2345 0.8746262\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6023731 0.6023731\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:56.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.185.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.214.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.265.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1689.6965 0.82504714\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:57.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.470.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.506.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.518.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.542.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.554.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:36.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1009.9577 0.4931434\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.83590895 0.83590895\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:58.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.380.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.530.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.800.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.105.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.130.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1871.7771 0.91395366\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:59.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.220.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.246.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.285.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.363.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.389.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.401.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2321.0906 1.133345\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.0536109 1.0536109\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:00.992.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.914.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.922.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.936.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.949.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.974.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:37.987.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.0. [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.390.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.520.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.650.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.780.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.104.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1083.5278 0.5290663\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:01.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.118.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.131.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.144.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.171.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.262.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.275.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1789.0507 0.8735599\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5904696 0.5904696\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:02.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.818.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.835.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.848.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.861.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.874.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.887.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.900.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.914.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.927.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:38.992.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.500.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2226.2102 1.0870167\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:03.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.320.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.450.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.710.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.176.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:39.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1832.8191 0.8949312\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.84666675 0.84666675\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:05.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.556.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.578.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.646.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.669.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.692.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.715.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3011.4976 1.4704578\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:05.914.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.783.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.829.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.851.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.874.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.919.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1144.1918 0.5586874\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:06.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.942.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.964.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:40.987.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.320.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.550.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.770.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.100.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1817.8107 0.88760287\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:07.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.204.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "950.64014 0.46417975\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.5656523 1.5656523\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:09.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:41.885.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.111.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.124.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.137.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.175.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.214.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "776.62964 0.3792137\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:09.794.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.382.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.395.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.474.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1523.0018 0.74365324\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.30679736 0.30679736\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:10.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.971.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:42.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.350.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.740.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.100.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.126.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1859.3116 0.907867\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:11.406.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.164.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.189.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.202.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.310.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.338.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2627.973 1.2831899\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5380368 0.5380368\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:12.871.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.802.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.823.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.848.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.861.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.874.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.886.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.900.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.912.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.925.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.939.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.965.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.978.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:43.991.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1453.8527 0.709889\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:13.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.290.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.420.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.550.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.930.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.106.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.119.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.176.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:44.190.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3468.0308 1.6933744\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.5652024 0.5652024\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:14.457.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.355.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.368.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1243.8833 0.6073649\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:15.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.442.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.455.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.504.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.528.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.540.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.565.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.577.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.592.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.603.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:45.606.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1032.7545 0.50427467\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6098017 0.6098017\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:16.730.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.640.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.740.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.890.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.102.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.141.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.169.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "745.72 0.36412108\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:17.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.278.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.332.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.358.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.412.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.426.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.452.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.455.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.468.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3014.853 1.4720962\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8047879 0.8047879\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:18.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.857.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.892.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.918.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.931.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.944.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.971.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:46.998.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.380.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.510.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2728.3872 1.3322203\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:19.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.650.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.930.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.187.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.213.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.226.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.242.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "835.76514 0.40808845\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.35182828 0.35182828\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:20.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.731.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.755.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.768.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.782.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.795.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.808.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.821.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.834.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.847.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.860.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.873.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.887.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.900.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.913.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.926.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2196.3145 1.0724192\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:21.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:47.991.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.560.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.940.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.106.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.110.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.119.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:48.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1899.5171 0.9274986\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.4146342 1.4146342\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:22.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.187.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.235.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.248.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.285.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.336.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1024.5664 0.50027657\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:22.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.411.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.423.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.548.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.561.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.564.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1954.4487 0.95432067\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7616383 0.7616383\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:24.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:49.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.550.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.680.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.940.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.106.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.119.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.145.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1415.517 0.6911704\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:25.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.202.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.216.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.246.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.285.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.311.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.336.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.362.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.388.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1292.7661 0.63123345\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.983957 0.983957\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:25.819.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.867.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.874.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.890.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.903.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.916.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.928.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.965.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.978.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:50.990.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.300.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.400.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.530.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2534.3506 1.2374759\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:26.643.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.104.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.116.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.129.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.205.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.217.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.243.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.247.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1985.1097 0.96929187\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.3741302 0.3741302\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:28.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.720.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.765.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.789.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.801.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.814.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.851.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.864.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.876.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.889.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.901.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.914.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1159.2529 0.56604147\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:28.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.927.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.965.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.977.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:51.990.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:52.200.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:52.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:52.270.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:52.400.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:52.530.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:52.650.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:52.780.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:52.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:52.890.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:52.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1769.1779 0.8638564\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.53968495 0.53968495\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:29.717.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.129.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.187.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.213.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.226.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "904.20416 0.44150594\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:30.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.343.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.356.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.368.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.381.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.394.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.445.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.471.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.484.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.500.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3243.373 1.5836782\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.98932827 0.98932827\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:32.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.944.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.980.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:53.992.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.500.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.420.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.550.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.104.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.129.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1253.8743 0.6122433\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:32.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.192.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.205.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.217.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.229.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.242.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.267.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.315.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2329.8455 1.1376199\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.72632253 0.72632253\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:33.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.850.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.862.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.888.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.900.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.913.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.926.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.939.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.964.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.977.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:54.990.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2122.3071 1.0362828\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:34.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.300.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.530.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.790.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.104.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.130.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.143.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1846.2656 0.9014969\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.66474444 0.66474444\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:36.198.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.634.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.641.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.654.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.666.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.704.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.716.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.728.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.753.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.779.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.791.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.804.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.816.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.829.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2907.9622 1.4199034\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:36.998.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.867.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.880.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.892.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.917.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.930.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.981.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.993.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:55.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:56.500.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:56.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1618.8727 0.7904652\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6837539 0.6837539\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:37.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.135.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.212.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.288.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.300.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1014.78235 0.4954992\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:38.681.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.362.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.411.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.423.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.473.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.497.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1420.6028 0.6936537\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.51479 0.51479\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:40.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:57.932.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.223.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.267.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.310.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2495.4941 1.218503\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer " ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:41.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.368.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.382.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.453.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.481.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.539.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1886.6587 0.92122006\n", - "all_finite forward False\n" + "False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:41.861.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:58.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1711.8558 0.8358671\n", - "all_finite forward False\n" + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.3815746 0.3815746\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:42.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.330.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.300.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.315.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.388.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.432.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2554.4795 1.2473044\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:44.244.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.504.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.548.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.577.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.591.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.605.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.620.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.635.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3104.7605 1.5159963\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:45.370.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.654.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.667.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:11:59.670.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1519.2853 0.7418385\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.2846315 1.2846315\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:45.876.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.124.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.131.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.144.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.156.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.181.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.218.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.243.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.255.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.317.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2525.38 1.2330956\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:46.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.329.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.341.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.353.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.390.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.427.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.451.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.476.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:00.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1891.6155 0.9236404\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8359874 0.8359874\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:48.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.745.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.758.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.771.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.784.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.835.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.847.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.860.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.873.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.885.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.898.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.910.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1249.3854 0.61005145\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:49.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.923.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.948.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.960.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:01.998.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.860.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.890.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.105.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3613.9075 1.7646033\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9413241 0.9413241\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:49.887.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.520.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.527.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.559.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.575.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.591.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.657.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.673.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.706.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2038.497 0.99535984\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:50.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.755.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.771.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.787.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.804.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.820.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.869.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.885.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.901.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.918.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3012.2153 1.4708083\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:52.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.934.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:02.989.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.0. [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1920.4708 0.9377299\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.5773715 1.5773715\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:53.570.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.457.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.465.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.502.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.527.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.539.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.552.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.564.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.626.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.651.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2902.804 1.4173847\n", - "all_finite forward False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:53.802.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.688.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.713.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.750.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.775.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.787.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:03.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "1897.9937 0.9267547\n", - "all_finite forward False\n" + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.17223412 0.17223412\n", + "all_finite forward True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:54.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.318.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.342.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.355.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.367.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.404.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.442.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.454.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.466.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "3229.8235 1.5770622\n", - "all_finite forward False\n", - "found_inf before 0.0\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [0 0 0 0 0 0 0 0]\n", - "found_inf [False False False False False False False False]\n", - "after clear [0 0 0 0 0 0 0 0]\n", - "found_inf after 0.0\n", - "grad scaler\n", - "False\n", - "_maybe_opt_step\n", - "0.0\n", - "patched_step\n", - "scaler update\n", - "scaler update False\n" + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.479.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.504.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.529.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.578.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.590.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.603.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.615.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.628.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.631.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:04.643.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.3356887 0.3356887\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.667.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.687.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.713.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.726.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.751.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.764.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.803.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.816.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.829.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.855.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.881.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.894.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.907.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.920.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.933.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.946.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.959.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.971.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:05.997.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.390.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.490.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.530.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.92722166 0.92722166\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.391.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.421.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.434.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.459.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.485.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.497.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.510.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.575.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.587.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.626.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.651.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.677.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.702.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.714.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:06.769.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.35183567 0.35183567\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.166.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.190.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.203.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.265.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.289.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.363.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.388.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.401.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.426.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.438.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.450.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.487.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.500.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.514.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.37801167 0.37801167\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:07.961.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.185.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.198.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.293.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.348.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.362.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.389.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.474.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.502.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.529.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:08.577.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "1.3542188 1.3542188\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.658.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.665.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.681.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.710.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.753.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.768.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.782.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.811.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.869.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.913.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.928.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.942.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.957.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.971.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:09.985.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.0. [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.140.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.290.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.570.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.750.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.890.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.62417126 0.62417126\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.492.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.502.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.547.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.626.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.655.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.669.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.712.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.726.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.754.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.768.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.783.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.811.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.912.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.915.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:10.928.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.61516726 0.61516726\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.632.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.706.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.720.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.749.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.763.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.820.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.835.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.849.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.863.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.892.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.906.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.964.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:11.993.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.700.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.190.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.230.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.015204 1.015204\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.540.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.554.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.568.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.582.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.625.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.668.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.682.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.696.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.710.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.754.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.768.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.782.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.811.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.869.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.883.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.912.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.926.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.930.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:12.944.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8376163 0.8376163\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.580.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.830.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.112.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.126.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.141.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.155.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.170.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.214.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.229.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.243.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.258.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.287.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.316.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.373.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.388.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.431.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.445.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.463.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.481.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.494.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.47947112 0.47947112\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.819.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.838.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.867.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.881.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.895.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.910.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.939.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:14.997.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.400.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.550.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.690.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.840.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.980.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.171.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.186.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.229.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.5827235 1.5827235\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.628.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.635.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.678.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.692.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.706.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.720.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.777.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.791.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.821.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.851.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.880.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.894.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.923.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.938.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:15.998.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.430.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.570.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.69956195 0.69956195\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.585.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.795.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.819.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.833.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.849.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.864.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.880.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.912.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.928.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.959.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:16.991.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.600.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.220.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.380.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.540.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.690.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.117.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.164.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.212.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.243.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:17.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6831159 0.6831159\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.422.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.430.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.447.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.489.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.532.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.546.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.560.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.644.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.658.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.672.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.686.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.714.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.728.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.742.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.770.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.784.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.811.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.828.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:18.841.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "0.47875917 0.47875917\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.312.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.427.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.441.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.456.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.470.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.484.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.513.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.527.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.555.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.597.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.611.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.625.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.667.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.681.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.709.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.736.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:19.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "0.57107794 0.57107794\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.269.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.319.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.333.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.347.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.388.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.431.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.505.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.546.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.560.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.615.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.629.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.643.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.657.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.670.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.673.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.684.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:20.688.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "0.92264843 0.92264843\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.205.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.213.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.255.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.323.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.405.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.435.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.449.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.478.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.507.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.581.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.595.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.623.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:21.627.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "0.93381655 0.93381655\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.854.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.884.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.914.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.944.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.974.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:22.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.300.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.330.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.770.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.165.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.180.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.194.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.224.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.238.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.256.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.58622193 0.58622193\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.748.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.764.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.793.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.808.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.823.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.852.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.867.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.881.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.911.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.927.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:23.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.300.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.330.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.480.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.780.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.930.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.109.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.124.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.139.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.170.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.173.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.189.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6643892 0.6643892\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.735.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.751.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.766.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.781.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.796.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.842.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.857.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.872.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.888.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.903.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.919.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.965.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.980.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:24.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.110.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.560.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.103.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.118.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.164.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.182.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.6637927 0.6637927\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.721.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.730.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.746.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.794.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.810.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.841.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.857.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.873.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.889.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.937.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:25.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.300.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.770.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.920.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.123.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.169.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.181.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:26.185.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.97988355 0.97988355\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.423.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.438.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.452.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.497.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.512.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.527.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.542.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.572.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.587.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.631.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.645.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.674.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.688.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.703.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.717.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.732.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.747.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.776.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.794.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.804.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:27.808.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.49330634 0.49330634\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.304.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.315.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.346.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.422.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.437.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.452.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.467.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.497.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.513.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.528.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.542.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.572.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.587.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.632.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.662.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.707.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.737.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.741.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.756.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:28.760.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.67321664 0.67321664\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.301.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.309.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.341.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.355.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.400.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.415.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.431.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.445.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.460.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.505.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.521.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.550.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.565.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.580.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.595.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.655.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.670.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.685.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.716.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.731.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.743.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:29.747.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.745478 0.745478\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.287.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.294.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.310.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.340.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.369.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.428.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.443.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.457.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.488.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.532.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.546.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.561.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.590.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.605.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.620.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.635.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.650.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.666.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.682.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.713.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.717.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:30.732.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8523119 0.8523119\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:31.941.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:31.949.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:31.965.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:31.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:31.995.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.400.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.550.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.700.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.850.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.116.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.131.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.176.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.190.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.205.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.220.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.234.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.248.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.278.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.322.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.336.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.351.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.365.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.369.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.383.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.8207974 0.8207974\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.814.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.830.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.846.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.862.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.894.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.911.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.927.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.943.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.959.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.976.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:32.992.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.570.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.890.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.105.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.152.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.183.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.214.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.246.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.46716288 0.46716288\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.807.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.816.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.847.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.863.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.879.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.894.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.910.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.925.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.941.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.957.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.972.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:33.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.200.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.510.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.670.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.830.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.980.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.114.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.130.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.162.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.209.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.274.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7383423 0.7383423\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.820.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.884.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.915.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.931.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.947.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.962.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.978.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:34.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.410.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.570.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.880.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.104.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.120.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.136.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.152.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.168.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.216.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.232.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.248.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.276.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:35.281.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9718822 0.9718822\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.538.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.547.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.578.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.595.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.610.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.626.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.641.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.657.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.673.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.705.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.721.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.737.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.768.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.784.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.800.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.831.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.847.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.862.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.893.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.908.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.924.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.940.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.955.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.971.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:36.990.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.0. [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.400.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.61867744 0.61867744\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.496.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.508.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.591.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.641.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.658.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.675.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.725.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.742.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.759.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.775.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.808.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.825.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.841.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.858.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.892.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.908.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer " + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.925.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.942.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.958.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.992.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:37.996.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.97974914 0.97974914\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.554.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.586.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.632.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.648.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.664.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.680.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.742.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.758.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.774.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.822.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.837.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.886.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.904.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.919.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.935.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.984.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:38.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.300.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.78401196 0.78401196\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.567.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.584.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.615.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.631.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.648.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.695.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.743.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.759.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.776.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.792.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.808.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.840.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.856.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.872.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.888.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.937.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:39.986.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:40.200.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:40.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:40.210.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:40.320.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:40.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.98119295 0.98119295\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.222.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.247.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.263.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.330.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.347.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.363.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.380.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.396.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.429.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.445.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.477.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.509.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.542.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.557.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.588.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.604.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.620.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.636.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.668.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.687.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.696.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:41.701.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.40406135 0.40406135\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.167.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.178.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.211.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.228.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.244.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.278.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.313.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.329.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.377.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.441.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.458.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.508.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.575.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.592.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.609.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.626.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.643.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.654.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:42.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "1.1191243 1.1191243\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.980.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.125.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.141.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.156.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.172.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.204.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.219.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.235.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.266.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.282.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.298.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.314.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.329.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.345.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.360.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.423.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.439.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.454.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.470.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.486.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.552.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.562.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:43.567.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.9111755 0.9111755\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.138.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.169.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.215.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.252.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.283.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.299.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.315.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.331.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.347.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.362.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.393.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.409.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.424.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.440.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.456.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.487.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.518.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.534.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.549.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.565.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.579.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:44.583.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n", + "True\n", + "0.7756543 0.7756543\n", + "all_finite forward True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.846.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.856.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.889.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.904.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.918.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.932.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.946.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.960.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:45.989.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.300.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.310.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.450.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embed_tokens True\n", + "position_embeddings False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer True\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.600.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.740.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.880.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.101.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.129.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.143.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.171.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.185.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.199.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.213.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.227.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.241.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.255.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.258.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "decoder_layer False\n", + "norm True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:55.991.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n", + "[WARNING] ME(1054275:281473393594400,MainProcess):2025-08-16-13:12:46.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" ] } ], diff --git a/mindnlp/core/_C/__init__.py b/mindnlp/core/_C/__init__.py index ea03367ea..c2f172428 100644 --- a/mindnlp/core/_C/__init__.py +++ b/mindnlp/core/_C/__init__.py @@ -1,9 +1,18 @@ +from typing import Any from mindspore import Generator as msGenerator +import mindspore +from mindnlp import core from . import _nn -from ..types import device as device_ from ..configs import DEVICE_TARGET +DEVICE_MAP = { + 'GPU': 'cuda', + 'Ascend': 'npu', + 'CPU': 'cpu' +} + + def _jit_set_profiling_executor(mode): pass @@ -30,6 +39,72 @@ def _debug_set_autodiff_subgraph_inlining(mode): DisableTorchFunctionSubclass = None + +class device(): + def __init__(self, type=None, index=None): + if type is not None: + if isinstance(type, str): + if ':' in type: + if index is not None: + raise ValueError("`type` must not include an index because index was " + f"passed explicitly: {type}") + _target, _id = type.split(':') + _id = int(_id) + else: + _target = type + _id = None if _target == 'cpu' else 0 + elif isinstance(type, device): + if index is not None: + raise ValueError("core.device(): When input is core.device, `index` can not be set.") + _target = type.type + _id = type.index + elif isinstance(type, int): + _id = type + try: + device_target = mindspore.get_current_device().device_target + except: + device_target = mindspore.get_context('device_target') + _target = DEVICE_MAP[device_target] + else: + print(type) + raise TypeError("core.device(): `type` must be type of 'str' or 'core.device'.") + else: + raise ValueError("core.device(): `type` can not be None") + + self.type = _target + self.index = _id + if DEVICE_TARGET == 'Ascned' and self.type == 'cuda': + self.type = 'npu' + + def __repr__(self): + if self.index is None: + return f"device(type={self.type})" + return f"device(type={self.type}, index={self.index})" + + def __eq__(self, __value): + if not isinstance(__value, device): + return False + return hash(self) == hash(__value) + + def __hash__(self): + return hash(self.type) ^ hash(self.index) + + def __gt__(self, other): + if self.type == 'cpu': + return False + return True + + def __enter__(self): + # self.prev_idx = torch.cuda._exchange_device(self.idx) + core._bind.set_device_in_context(self) + + def __exit__(self, type: Any, value: Any, traceback: Any): + # self.idx = torch.cuda._maybe_exchange_device(self.prev_idx) + core._bind.set_device_in_context(None) + return False + +device_ = device + class Generator(msGenerator): def __init__(self, device='cpu'): super().__init__() @@ -41,11 +116,14 @@ def __init__(self, device='cpu'): def device(self): if hasattr(self, '_device'): return self._device - return device_('cpu') + return device('cpu') default_generator = Generator() class Tag: pass def _log_api_usage_once(*args): - pass \ No newline at end of file + pass + +ScriptDict = dict +ScriptList = list \ No newline at end of file diff --git a/mindnlp/core/_C/_nn.py b/mindnlp/core/_C/_nn.py index 3bf10046c..ba9d2f9c3 100644 --- a/mindnlp/core/_C/_nn.py +++ b/mindnlp/core/_C/_nn.py @@ -1,5 +1,4 @@ from mindnlp import core -from ..types import device as device_ def _parse_to(*args, **kwargs): """ @@ -22,7 +21,7 @@ def _parse_to(*args, **kwargs): device = args[0] dtype = None elif isinstance(args[0], (str, int)): - device = device_(args[0]) + device = core.device(args[0]) dtype = None else: raise TypeError(f"Expected core.dtype or core.device, but got {type(args[0])}") diff --git a/mindnlp/core/__init__.py b/mindnlp/core/__init__.py index 71d1a45a6..cc8612827 100644 --- a/mindnlp/core/__init__.py +++ b/mindnlp/core/__init__.py @@ -36,17 +36,16 @@ preserve_format = None legacy_contiguous_format = None channels_last_3d = None +memory_format = None inf = float("inf") nan = float("nan") -from ._C import * from ._dtype import * from ._tensor import Tensor, tensor, is_tensor, \ LongTensor, FloatTensor, BoolTensor, HalfTensor, BFloat16Tensor, IntTensor -from .types import device +from ._C import * from ._C.size import Size -from .types import device from .autograd import * from .ops import * from .serialization import load, save @@ -57,8 +56,9 @@ from .func import vmap from .configs import set_pyboost +from . import _dynamo from . import profiler, cuda, amp, compiler, jit, version, __future__, overrides, \ - return_types, linalg, fx, backends, testing, nn, fft, _jit_internal, utils, optim + return_types, linalg, fx, backends, nn, fft, _jit_internal, utils, optim, testing from ._lowrank import svd_lowrank from .random import get_rng_state, initial_seed, manual_seed, seed, set_rng_state diff --git a/mindnlp/core/_bind.py b/mindnlp/core/_bind.py index cad31e74b..ca7a96b7a 100644 --- a/mindnlp/core/_bind.py +++ b/mindnlp/core/_bind.py @@ -1,7 +1,7 @@ import ctypes from typing import Any from ._dtype import * -from .types import device as device_ +from ._C import device as device_ from .configs import ON_A1 DEFAULT_DTYPE, DEFAULT_DEVICE = float32, device_('cpu') diff --git a/mindnlp/core/_dtype.py b/mindnlp/core/_dtype.py index c291d87e6..08489d610 100644 --- a/mindnlp/core/_dtype.py +++ b/mindnlp/core/_dtype.py @@ -46,6 +46,11 @@ def __gt__(self, other): float8_e4m3fn = None # TODO: not support fp8 for now float8_e5m2 = None +float8_e4m3fnuz = None +float8_e5m2fnuz = None +complex32 = None +cfloat = complex32 +cdouble = complex64 uint1 = None uint2 = None diff --git a/mindnlp/core/_dynamo/__init__.py b/mindnlp/core/_dynamo/__init__.py index 283a3fa56..62e799a95 100644 --- a/mindnlp/core/_dynamo/__init__.py +++ b/mindnlp/core/_dynamo/__init__.py @@ -20,6 +20,7 @@ ) from . import eval_frame +# from . import config def reset(): pass diff --git a/mindnlp/core/_dynamo/config.py b/mindnlp/core/_dynamo/config.py new file mode 100644 index 000000000..47f511dbe --- /dev/null +++ b/mindnlp/core/_dynamo/config.py @@ -0,0 +1,677 @@ +""" +Configuration module for TorchDynamo compiler and optimization settings. + +This module contains various configuration flags and settings that control TorchDynamo's +behavior, including: +- Runtime behavior flags (e.g., guard settings, specialization options) +- Debugging and development options +- Performance tuning parameters +- Feature toggles for experimental features +""" + +import getpass +import os +import sys +import tempfile +from os.path import abspath, dirname +from typing import Any, Callable, Literal, Optional, TYPE_CHECKING, Union + +from torch._environment import is_fbcode +from torch.utils._config_module import Config, get_tristate_env, install_config_module + + +# to configure logging for dynamo, aot, and inductor +# use the following API in the torch._logging module +# torch._logging.set_logs(dynamo=, aot=, inductor) +# or use the environment variable TORCH_LOGS="dynamo,aot,inductor" (use a prefix + to indicate higher verbosity) +# see this design doc for more detailed info +# Design doc: https://docs.google.com/document/d/1ZRfTWKa8eaPq1AxaiHrq4ASTPouzzlPiuquSBEJYwS8/edit# +# the name of a file to write the logs to +# [@compile_ignored: debug] +log_file_name: Optional[str] = None + +# [@compile_ignored: debug] Verbose will print full stack traces on warnings and errors +verbose = os.environ.get("TORCHDYNAMO_VERBOSE", "0") == "1" + +# [@compile_ignored: runtime_behaviour] verify the correctness of optimized backend +verify_correctness = False + +# need this many ops to create an FX graph (deprecated: not used) +minimum_call_count = 1 + +# turn on/off DCE pass (deprecated: always true) +dead_code_elimination = True + +# disable (for a function) when cache reaches this size + +# controls the maximum number of cache entries with a guard on same ID_MATCH'd +# object. It also controls the maximum size of cache entries if they don't have +# any ID_MATCH'd guards. +# [@compile_ignored: runtime_behaviour] +recompile_limit = 8 + +# [@compile_ignored: runtime_behaviour] safeguarding to prevent horrible recomps +accumulated_recompile_limit = 256 + +# [@compile_ignored: runtime_behaviour] skip tracing recursively if cache limit is hit (deprecated: does not do anything) +skip_code_recursive_on_recompile_limit_hit = True + +# raise a hard error if cache limit is hit. If you are on a model where you +# know you've sized the cache correctly, this can help detect problems when +# you regress guards/specialization. This works best when recompile_limit = 1. +# This flag is incompatible with: suppress_errors. +# [@compile_ignored: runtime_behaviour] +fail_on_recompile_limit_hit = False + +cache_size_limit: int = Config(alias="torch._dynamo.config.recompile_limit") +accumulated_cache_size_limit: int = Config( + alias="torch._dynamo.config.accumulated_recompile_limit" +) + +# (deprecated: does not do anything) +skip_code_recursive_on_cache_limit_hit: bool = Config( + alias="torch._dynamo.config.skip_code_recursive_on_recompile_limit_hit" +) +fail_on_cache_limit_hit: bool = Config( + alias="torch._dynamo.config.fail_on_recompile_limit_hit" +) + +# whether or not to specialize on int inputs. This only has an effect with +# dynamic_shapes; when dynamic_shapes is False, we ALWAYS specialize on int +# inputs. Note that assume_static_by_default will also cause ints to get +# specialized, so this is mostly useful for export, where we want inputs +# to be dynamic, but accesses to ints should NOT get promoted into inputs. +specialize_int = False + +# Whether or not to specialize on float inputs. Dynamo will always promote +# float inputs into Tensor inputs, but at the moment, backends inconsistently +# support codegen on float (this is to be fixed). +specialize_float = False + +# legacy config, does nothing now! +dynamic_shapes = True + +use_lazy_graph_module = ( + os.environ.get("TORCH_COMPILE_USE_LAZY_GRAPH_MODULE", "1") == "1" +) + +# This is a temporarily flag, which changes the behavior of dynamic_shapes=True. +# When assume_static_by_default is True, we only allocate symbols for shapes marked dynamic via mark_dynamic. +# NOTE - this flag can be removed once we can run dynamic_shapes=False w/ the mark_dynamic API +# see [Note - on the state of mark_dynamic] +assume_static_by_default = True + +# This flag changes how dynamic_shapes=True works, and is meant to be used in conjunction +# with assume_static_by_default=True. +# With this flag enabled, we always compile a frame as fully static for the first time, and, if we fail +# any guards due to wobbles in shape, we recompile with *all* the wobbled shapes as being marked dynamic. +automatic_dynamic_shapes = True + +# Valid options: "dynamic", "unbacked" +automatic_dynamic_shapes_mark_as: Literal["dynamic", "unbacked"] = "dynamic" + +# This flag changes how the shapes of parameters are treated. +# If this flag is set to True, then the shapes of torch.nn.Parameter as well as of torch.Tensor are attempted to be dynamic +# If this flag is set to False, then the shapes of torch.nn.Parameter are assumed to be static, +# while the shapes of torch.Tensor are assumed to be dynamic. +force_parameter_static_shapes = True + +# This flag ensures that the shapes of a nn module are always assumed to be static +# If the flag is set to True, then the shapes of a nn.module are assumed to be static +# If the flag is set to False, then the shapes of a nn.module can be dynamic +force_nn_module_property_static_shapes = True + +# Typically, if you mark_dynamic a dimension, we will error if the dimension +# actually ended up getting specialized. This knob changes the behavior so +# that we don't error at all. This is helpful for our CI where I'm using a +# heuristic to mark batch dimensions as dynamic and the heuristic may get it +# wrong. +allow_ignore_mark_dynamic = False + +# Set this to False to assume nn.Modules() contents are immutable (similar assumption as freezing) +guard_nn_modules = True + +# Uses CPython internal dictionary tags to detect mutation. There is some +# overlap between guard_nn_modules_using_dict_tags and guard_nn_modules flag. +# guard_nn_modules unspecializes the nn module instance and adds guard for each +# relevant member of the nn modules. On the other hand, +# guard_nn_modules_using_dict_tags specializes on each nn module instance but +# uses low overhead dict version matching to detect mutations, obviating the +# need to guard on members of the nn modules. With +# guard_nn_modules_using_dict_tags, the guard_nn_modules is not really required +# but kept around for debugging and discussing unspecializing nn module +# variables. +# TODO(janimesh, voz): Remove both of these flags (or at least guard_nn_modules) +# once we have reached stability for the guard_nn_modules_using_dict_tags. +guard_nn_modules_using_dict_tags = True + +# Flag to enable preparation for graph freezing, so that the named parameters and +# buffers are passed as params_flat in tracing context by AOT autograd. +# Non-Inductor backends can use this list for graph freezing. +prepare_freezing = os.environ.get("TORCHDYNAMO_PREPARE_FREEZING", "0") == "1" + +# NOTE this has been deprecated, it does nothing now. +traceable_tensor_subclasses: set[type[Any]] = set() + +# If a tensor subclass is put into this set, Dynamo will model its instasnces in +# a very conservative and limited way (most likely causing lots of graph breaks +# if one apply tensor ops on these instances). This is useful if you encounter +# internal compiler errors from Dynamo which are caused by tensor subclasses, +# and you are willing to tolerate potential graph breaks rather than hard error. +nontraceable_tensor_subclasses: set[type[Any]] = set() + +# Suppress errors in torch._dynamo.optimize, instead forcing a fallback to eager. +# This is a good way to get your model to work one way or another, but you may +# lose optimization opportunities this way. Devs, if your benchmark model is failing +# this way, you should figure out why instead of suppressing it. +# This flag is incompatible with: fail_on_recompile_limit_hit. +suppress_errors = bool(os.environ.get("TORCHDYNAMO_SUPPRESS_ERRORS", False)) + +# Record and write an execution record of the current frame to a file +# if an exception is encountered +# @compile_ignored[debug] +replay_record_enabled = os.environ.get("TORCH_COMPILE_REPLAY_RECORD", "0") == "1" + +# Rewrite assert statement in python with torch._assert +rewrite_assert_with_torch_assert = True + +# Disable dynamo +disable = os.environ.get("TORCH_COMPILE_DISABLE", "0") == "1" + +# [@compile_ignored: runtime_behaviour] Get a cprofile trace of Dynamo +cprofile = os.environ.get("TORCH_COMPILE_CPROFILE", False) + +# legacy config, does nothing now! +skipfiles_inline_module_allowlist: dict[Any, Any] = {} + +# If a string representing a PyTorch module is in this ignorelist, +# the `allowed_functions.is_allowed` function will not consider it +# when creating a list of PyTorch functions that will appear in +# FX IR. +allowed_functions_module_string_ignorelist = { + "torch.distributions", + "torch.testing", + "torch._refs", + "torch._prims", + "torch._decomp", +} + +# Debug Flag to try minifier at different stages. Possible values are {None, "aot", "dynamo"} +# None - Minifier is switched off +# dynamo - Runs minifier on the TorchDynamo produced graphs, if compilation fails +# aot - Runs minifier on the Aot Autograd produced graphs, if compilation fails +# [@compile_ignored: debug] +repro_after = os.environ.get("TORCHDYNAMO_REPRO_AFTER", None) + +# Compiler compilation debug info +# 1: Dumps the original graph out to repro.py if compilation fails +# 2: Dumps a minifier_launcher.py if compilation fails. +# 3: Always dumps a minifier_launcher.py. Good for segfaults. +# 4: Dumps a minifier_launcher.py if the accuracy fails. +# [@compile_ignored: debug] +repro_level = int(os.environ.get("TORCHDYNAMO_REPRO_LEVEL", 2)) + +# By default, we try to detect accuracy failure by running both forward +# and backward of a torchdynamo produced graph (if you are using repro_after +# 'dynamo'). This setting forces us to only test the forward graph and +# not the backward graph. This can be helpful if you're trying to debug +# an inference only problem, but the minifier seems to be choking on the +# backwards step +# TODO: Detect this situation automatically so the user doesn't need +# to manually configure this +# [@compile_ignored: debug] +repro_forward_only = os.environ.get("TORCHDYNAMO_REPRO_FORWARD_ONLY") == "1" + +# The tolerance we should use when testing if a compiled graph +# has diverged so that we should treat it as an accuracy failure +# [@compile_ignored: debug] +repro_tolerance = 1e-3 + + +# Whether to ignore non-floating point values when checking accuracy. +# Checking accuracy of non-floating point values such as boolean tensors +# can lead to false positives. +# [@compile_ignored: debug] +repro_ignore_non_fp = os.environ.get("TORCHDYNAMO_REPRO_IGNORE_NON_FP") == "1" + +# If True, when testing if two models are the same, we will test them against +# a third fp64 reference and only report a problem if the RMSE relative to the +# fp64 is greater. However, this will use more memory; you may disable this +# if memory usage is too high. +# [@compile_ignored: runtime_behaviour] +same_two_models_use_fp64 = True + +# Not all backends support scalars. Some calls on torch.Tensor (like .item()) return a scalar type. +# When this flag is set to False, we introduce a graph break instead of capturing. +# This requires dynamic_shapes to be True. +capture_scalar_outputs = os.environ.get("TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS") == "1" + +# Not all backends support operators that have dynamic output shape (e.g., +# nonzero, unique). When this flag is set to False, we introduce a graph +# break instead of capturing. This requires dynamic_shapes to be True. +# If you set this to True, you probably also want capture_scalar_outputs +# (these are separated for historical reasons). +capture_dynamic_output_shape_ops = ( + os.environ.get("TORCHDYNAMO_CAPTURE_DYNAMIC_OUTPUT_SHAPE_OPS", "0") == "1" +) + +# hybrid backed unbacked symints +prefer_deferred_runtime_asserts_over_guards = False + +# For complex dynamic shapes guards that we're unable to specify with dynamo/export's +# range constraints + dims + derived dims language, we raise constraint violation +# errors or specialize by default. If set to True, this flag avoids crashing/specialization, +# and allows complex guards as runtime assertions in the graph. +allow_complex_guards_as_runtime_asserts = False + +# By default, dynamo will treat all ints as backed SymInts, which means (1) it +# will wait to see the int change over multiple runs before generalizing and +# (2) it will still always 0/1 specialize an int. When true, this knob +# forces dynamo to treat _length_per_key and _offset_per_key on +# KeyedJaggedTensor from torchrec as size-like unbacked SymInts, so that +# they (1) generalize immediately and (2) unsoundly never compare equal to +# 0/1. This is not on by default as AOTAutograd/Inductor cannot currently +# compile this code; however, this can be useful for export. +force_unspec_int_unbacked_size_like_on_torchrec_kjt = False + +# Currently, Dynamo will always specialize on int members of NN module. +# However, there could be cases where this is undesirable, e.g., when tracking +# step count leading to constant recompilation and eventually eager fallback. +# Setting this flag to True will allow int members to be potentially unspecialized +# through dynamic shape mechanism. +# Defaults to False for BC. +allow_unspec_int_on_nn_module = False + +# Specify how to optimize a compiled DDP module. The flag accepts a boolean +# value or a string. There are 3 modes. +# 1. "ddp_optimizer" (or True): with "ddp_optimizer", Dynamo will automatically +# split model graph into pieces to match DDP bucket sizes to allow DDP +# comm/compute overlap. +# 2. "python_reducer" (experimental): this optimization requires the usage +# of compiled_autograd. With "python_reducer", DDP will disable the C++ reducer +# and use the Python reducer to allow compiled_autograd to trace the +# communication and allow comm/compute overlap without graph-breaks. +# 3. "no_optimization" (or False): Dynamo won't split the model graph, nor +# will Python reducer be used. With this mode, there will be no graph-breaks +# and the original DDP C++ reducer will be used. There will no comm/compute +# overlap. This mode CANNOT be used with compiled_autograd. +# Note that to avoid breaking the existing usage, mode 1 and mode 4 can be +# specified with a boolean value. True is using ddp_optimizer and False is +# no optimization. +optimize_ddp: Union[ + bool, + Literal[ + "ddp_optimizer", + "python_reducer", + "python_reducer_without_compiled_forward", + "no_optimization", + ], +] = True + +# By default, Dynamo emits runtime asserts (e.g. torch._check, torch._check_is_size) in the graph. +# In some cases those asserts could be performance costly +# E.g. torch._check(tensor[0].item() > 2) for tensor on cuda will require cuda sync. +# Setting this to True keeps them hinting to symbolic shapes engine, +# but not be emitted in the graph. +do_not_emit_runtime_asserts: bool = ( + os.environ.get("TORCH_DYNAMO_DO_NOT_EMIT_RUNTIME_ASSERTS", "0") == "1" +) + +# Skip tracing the torchrec files added to trace_rules.FBCODE_SKIP_DIRS +skip_torchrec = True + +# Don't apply most trace_rules.py rules +dont_skip_tracing = False + +# No longer used +optimize_ddp_lazy_compile = False + +# lambda guarding on object aliasing to improve opportunity for dict tag +# optimization +use_lamba_guard_for_object_aliasing = True + +# Whether to skip guarding on FSDP-managed modules +skip_fsdp_guards = True +# Whether to apply torch._dynamo.disable() to FSDP2 hooks. +# Defaults to True. If Traceable FSDP2 is used, set this to False. +skip_fsdp_hooks = True + +# Make dynamo skip guarding on hooks on nn modules +# Note: unsafe: if your model actually has hooks and you remove them, or doesn't and you add them, +# dynamo will not notice and will execute whichever version you first compiled. +skip_nnmodule_hook_guards = True + +# Make dynamo skip no tensor aliasing guard on parameters +# Note: unsafe: if you compile a function with different parameters as inputs, +# and then later pass on the same parameter as two inputs, dynamo will not +# notice and lead to incorrect result. +skip_no_tensor_aliasing_guards_on_parameters = True + +# Considers a tensor immutable if it is one of the values of a dictionary, and +# the dictionary tag is same across invocation calls. +skip_tensor_guards_with_matching_dict_tags = True + +# Skips guards on func.__defaults__ if the element to be guarded is a constant +skip_guards_on_constant_func_defaults = True + + +# The recursive-dict-tag guard relies on the class/function identity staying +# stable. We therefore assume that the following function dunder attributes +# are **never rebound** to a different object: +# +# • __code__ • __closure__ +# • __defaults__ • __kwdefaults__ +# • __annotations__ • __mro__ +# +# It is fine to mutate the objects they already point to (e.g. tweak an element +# inside __defaults__), but assignments like +# +# foo.__defaults__ = (3, 4) # REBIND - NOT SUPPORTED +# +# would invalidate the optimization. This type of rebinding is rare, so we +# assume that the rebinding never happens for guard purposes. Set the flag +# below to False only in environments where such rebinding is known to occur. +assume_dunder_attributes_remain_unchanged = True + +# Speedup guard execution of nested nn modules by recursively checking for dict +# tags to avoid full guard execution. +use_recursive_dict_tags_for_guards = True + +# Maximum number of objects for which we check dict pointers tags. This is +# useful for regional compilation. +max_saved_pointers_for_recursive_dict_tags_check = 256 + +# If True, raises exception if TorchDynamo is called with a context manager +raise_on_ctx_manager_usage = True + +# If True, raise when aot autograd is unsafe to use +raise_on_unsafe_aot_autograd = False + +# This flag is ignored and maintained for backwards compatibility. +error_on_nested_jit_trace = True + +# If true, error with a better message if we symbolically trace over a +# dynamo-optimized function. If false, silently suppress dynamo. +error_on_nested_fx_trace = True + +# Disables graph breaking on rnn. YMMV with backends. +allow_rnn = False + +# If true, enables feature that captures PyTorch sparsity in the +# exported FX graph. This flag should become the default eventually +# and be removed, but currently provides a way to fall back to old +# graph breaking behavior. +capture_sparse_compute = False if is_fbcode() else True + +# If true, error if we try to compile a function that has +# been seen before. +# [@compile_ignored: runtime_behaviour] +error_on_recompile = False + +# [@compile_ignored: debug] Whether to report any guard failures (deprecated: does not do anything) +report_guard_failures = True + +# [@compile_ignored: debug] root folder of the project +base_dir = dirname(dirname(dirname(abspath(__file__)))) + +# Trace through NumPy or graphbreak +trace_numpy = True + +# Default NumPy dtypes when tracing with torch.compile +# We default to 64bits. For efficiency, one may want to change these to float32 +numpy_default_float = "float64" +numpy_default_complex = "complex128" +numpy_default_int = "int64" + +# use numpy's PRNG if True, pytorch otherwise +use_numpy_random_stream = False + +# Use C++ guard manager (deprecated: always true) +enable_cpp_guard_manager = True + +# Use C++ guard manager for symbolic shapes +enable_cpp_symbolic_shape_guards = not is_fbcode() + +# Enable tracing through contextlib.contextmanager +enable_trace_contextlib = True + +# Enable tracing through unittest +enable_trace_unittest = False + +# Enable tracing generator functions lazily. If False, Dynamo will exhaust +# generators upon first execution. And if True, the generator will be accessed lazily +enable_faithful_generator_behavior = True + +# Inline inbuilt nn modules +inline_inbuilt_nn_modules = Config( # type: ignore[var-annotated] + default=True, + justknob="pytorch/compiler:inline_inbuilt_nn_modules", +) + +# Install "free" tensor variables (globals, non-locals, nn module attributes) +# as graph attributes. This is useful for export, as it +# produces a consistent number of inputs to the graph. +install_free_tensors = False + +# Use C++ FrameLocalsMapping (raw array view of Python frame fastlocals) (deprecated: always True) +enable_cpp_framelocals_guard_eval = True + +# Whether to automatically find and replace identical graph +# regions with a call to invoke_subgraph +use_graph_deduplication = False + +# Whether to track nodes for deduplication (testing only) +# This flag is ignored if use_graph_deduplication is True +track_nodes_for_deduplication = False + +# Whether to lint the graph after each region is replaced +# (Debug) +graph_deduplication_lint = False + +# Issues a warning in Python 3.13.0 for possibly slower guard evaluation and +# instructs user to attempt using 3.13.1+, where the CPython bug is fixed. +# Should be disabled in dynamo-wrapped tests since some tests check that no warnings are issued. +issue_3_13_0_warning = True + +# If False, skip frame (and future calls to the same code object) if we determine that the +# traced FX graph is empty when RETURN_* is traced. +allow_empty_graphs = False + +# When set, total compile time instruction count is recorded using +# torch._dynamo.utilsCompileTimeInstructionCounter. +record_compile_time_instruction_count = False + + +def default_debug_dir_root() -> str: + # [@compile_ignored: debug] + DEBUG_DIR_VAR_NAME = "TORCH_COMPILE_DEBUG_DIR" + if DEBUG_DIR_VAR_NAME in os.environ: + return os.path.join(os.environ[DEBUG_DIR_VAR_NAME], "torch_compile_debug") + elif is_fbcode(): + return os.path.join( + tempfile.gettempdir(), getpass.getuser(), "torch_compile_debug" + ) + else: + return os.path.join(os.getcwd(), "torch_compile_debug") + + +# [@compile_ignored: debug] +debug_dir_root = default_debug_dir_root() + +# [@compile_ignored: debug] +_save_config_ignore = { + "repro_after", + "repro_level", + # workaround: "cannot pickle PyCapsule" + "constant_functions", + # workaround: "cannot pickle module" + "skipfiles_inline_module_allowlist", +} + +# for backend="cudagraphs", mutations on input be sent to the cudagraph backend +# or replayed in aot_autograd epilogue. default is False because mutation on inputs +# can prevent cudagraphing. +cudagraph_backend_keep_input_mutation = False + +# enable cudagraph support for mutated inputs from prior cudagraph pool +cudagraph_backend_support_input_mutation = False + +# When True, only ops that have the torch.Tag.pt2_compliant tag +# will be allowed into the graph; all other ops will be disallowed +# and will fall back to eager-mode PyTorch. Useful to ensure +# correctness of custom ops. +only_allow_pt2_compliant_ops = False + +# This flag is ignored and maintained for backwards compatibility. +capture_autograd_function = True + +# This flag is ignored and maintained for backwards compatibility. +capture_func_transforms = True + +# If to log Dynamo compilation metrics into log files (for OSS) and Scuba tables (for fbcode). +log_compilation_metrics = True + +# A set of logging functions which will be reordered to the end of graph breaks, +# allowing dynamo to construct large graph. Note that there are some +# limitations to this, such as how it does not correctly print objects that were +# mutated after the print statement. +reorderable_logging_functions: set[Callable[[Any], None]] = set() + +# A set of methods that will be ignored while tracing, +# to prevent graph breaks. +# Add logging.Logger. to ignore all calls for method, +# or logger. to ignore calls for method from this logger instance only. +ignore_logger_methods: set[Callable[..., Any]] = set() + +# simulates what would happen if we didn't have support for BUILD_SET opcode, +# used for testing +inject_BUILD_SET_unimplemented_TESTING_ONLY = False + +_autograd_backward_strict_mode_banned_ops = [ + "layout", + "is_neg", + "is_conj", + "is_pinned", +] + +_autograd_backward_strict_mode_conditional_banned_ops = [ + "stride", + "storage_offset", + "is_contiguous", +] + +# Enables caching of dispatches to fake tensors. +fake_tensor_cache_enabled = ( + os.environ.get("TORCH_FAKE_TENSOR_DISPATCH_CACHE", "1") == "1" +) + +# Enables cross checking between the fake tensor cache and dispatch. +fake_tensor_cache_crosscheck_enabled = ( + os.environ.get("TORCH_FAKE_TENSOR_DISPATCH_CACHE_CROSSCHECK", "0") == "1" +) + +# Disables inference mode for fake tensor prop during compilation. At runtime, +# the inference_mode is still respected. +fake_tensor_disable_inference_mode = True + +# Experimental feature for running automatic caching precompile. +# Enables automatic DynamoCache save/load +caching_precompile = os.environ.get("TORCH_CACHING_PRECOMPILE", "0") == "1" + +# Enables the Compiled Autograd engine to trace autograd calls made under torch.compile(). +# Note: AOTAutograd will still trace and partition an AOT backward graph local to that +# compiled region. But AOTAutograd traces without knowledge of backward hooks which are +# coordinated by the Autograd engine, and under the hood, it uses the torch.autograd.grad +# API, so it cannot capture gradient accumulation operations (AccumulateGrad). +# +# Compiled Autograd will trace all autograd operations as seen by the Autograd engine. +# This flag will also lift certain restrictions during the forward trace such as +# registering backward hooks on tensors contained within the compiled region. +compiled_autograd = False + + +# Checks if we should graph break when seeing nn parameter constructors +# in dynamo; this is so that we clearly fail and ask users to move outside +# the function as opposed to trying to support the ctor with unclear semantics +# See https://github.com/pytorch/pytorch/issues/157452 for more context +graph_break_on_nn_param_ctor = True + +# Overrides torch.compile() kwargs for Compiled Autograd: +compiled_autograd_kwargs_override: dict[str, Any] = {} + +# Enables use of collectives *during* compilation to synchronize behavior +# across ranks. Today, this is used solely to modify automatic_dynamic_shapes +# behavior, making it so that we infer that if an input is dynamic by +# inspecting whether or not its input size varies across ranks. Because +# this synchronization uses collectives, all ranks must run compilation at +# the same time; ranks must not diverge with graph breaks. This can be most +# reliably achieved by ensuring PT2 only is run on SPMD programs. If this +# invariant is inviolated, you will likely deadlock NCCL and encounter a +# NCCL timeout. +enable_compiler_collectives = os.environ.get("TORCH_COMPILER_COLLECTIVES", "0") == "1" + +# Enables a local, filesystem "profile" which can be used for automatic +# dynamic decisions, analogous to profile-guided optimization. This config +# ONLY has an effect if torch.compiler.config.workflow_id is specified, +# which specifies the name of the profile we will save/load. +# +# The idea is that if we observe that a particular input is dynamic over +# multiple iterations on one run, we can save a profile with this information +# so the next time we run we can just make it dynamic the first time around, +# skipping an unnecessary static compilation. The profile can be soundly +# stale, if it is wrong, it just means we may make more things dynamic than +# was actually necessary (NB: this /can/ cause a failure if making something +# dynamic causes the compiler to stop working because you tickled a latent +# bug.) +# +# The profile is ONLY guaranteed to work if the user source code is 100% +# unchanged. Applying the profile if there are user code changes is only +# best effort otherwise. In particular, we identify particular code objects +# by filename, line number and name of their function, so adding/removing newlines +# will typically cause cache misses. We continuously update the profile, +# so if we only discover something is dynamic on the second run, we will update +# the profile for subsequent runs. +automatic_dynamic_local_pgo: bool = Config( + justknob="pytorch/remote_cache:enable_local_automatic_dynamic_pgo", + env_name_force="TORCH_DYNAMO_AUTOMATIC_DYNAMIC_LOCAL_PGO", + default=True, +) + +# Like above, but using remote cache +automatic_dynamic_remote_pgo: Optional[bool] = get_tristate_env( + "TORCH_DYNAMO_AUTOMATIC_DYNAMIC_REMOTE_PGO" +) + +# temporary config to kill later +_unsafe_skip_fsdp_module_guards = ( + os.environ.get("UNSAFE_SKIP_FSDP_MODULE_GUARDS", "0") == "1" +) + +# Common prefix to append to the id of each compile run to filter out data +pt2_compile_id_prefix: Optional[str] = os.environ.get("PT2_COMPILE_ID_PREFIX", None) + +# Run GC at the end of compilation +run_gc_after_compile = Config( # type: ignore[var-annotated] + default=True, + justknob="pytorch/compiler:enable_run_gc_after_compile", + env_name_default="TORCH_DYNAMO_RUN_GC_AFTER_COMPILE", +) + +# Takes the function/module decorated with torch.compile and passes it through a +# wrapper. This ensures that nn.module hooks are also compiled in the same frame. +wrap_top_frame = False + +# Flag to record runtime overhead in profile traces. Used for pre-graph bytecode +# and AOTAutograd runtime wrapper. +record_runtime_overhead = True + +# HACK: this is for testing custom ops profiling only +_custom_ops_profile: Optional[Any] = None + +if TYPE_CHECKING: + from torch.utils._config_typing import * # noqa: F401, F403 + + def _make_closure_patcher(**changes: Any) -> Any: ... + + +install_config_module(sys.modules[__name__]) \ No newline at end of file diff --git a/mindnlp/core/_dynamo/decorators.py b/mindnlp/core/_dynamo/decorators.py index 667bf45cc..fba2ada41 100644 --- a/mindnlp/core/_dynamo/decorators.py +++ b/mindnlp/core/_dynamo/decorators.py @@ -38,4 +38,18 @@ def fn(a): if isinstance(fn, (list, tuple)): return [allow_in_graph(x) for x in fn] assert callable(fn), "allow_in_graph expects a callable" + return fn + +def disable(fn=None, recursive=True, *, reason=None, wrapping=True): # type: ignore[no-untyped-def] + """ + Decorator to disable TorchDynamo + + If recursive=True, Dynamo is completely skipped on the decorated function + frame as well as the recursively invoked functions. + + If recursive=False, Dynamo skips frames associated with the function code, + but still process recursively invoked frames. + + If reason is provided, it will be printed when Dynamo attempts to trace the disabled function. + """ return fn \ No newline at end of file diff --git a/mindnlp/core/_dynamo/trace_rules.py b/mindnlp/core/_dynamo/trace_rules.py new file mode 100644 index 000000000..55bf403f2 --- /dev/null +++ b/mindnlp/core/_dynamo/trace_rules.py @@ -0,0 +1,8 @@ +import os + +def _as_posix_path(path: str) -> str: + posix_path = Path(os.path.normpath(path)).as_posix() + # os.path.normpath and pathlib.Path remove trailing slash, so we need to add it back + if path.endswith((os.path.sep, "/")): + posix_path += "/" + return posix_path diff --git a/mindnlp/core/_environment.py b/mindnlp/core/_environment.py new file mode 100644 index 000000000..51bdb165b --- /dev/null +++ b/mindnlp/core/_environment.py @@ -0,0 +1,2 @@ +def is_fbcode() -> bool: + return False \ No newline at end of file diff --git a/mindnlp/core/_guards.py b/mindnlp/core/_guards.py new file mode 100644 index 000000000..a111a12f8 --- /dev/null +++ b/mindnlp/core/_guards.py @@ -0,0 +1,1169 @@ +from __future__ import annotations + +import contextlib +import dataclasses +import enum +import functools +import logging +import re +import threading +import traceback +import unittest.mock +import weakref +from abc import abstractmethod +from collections import defaultdict +from contextlib import contextmanager +from dataclasses import dataclass +from typing import ( + Any, + Callable, + Generic, + NamedTuple, + Optional, + TYPE_CHECKING, + TypeVar, + Union, +) + +import torch +from mindnlp.core.utils import _pytree as pytree +from mindnlp.core.utils._backport_slots import dataclass_slots +from mindnlp.core.utils._traceback import CapturedTraceback, format_frame +from mindnlp.core.utils.weak import WeakTensorKeyDictionary + + +log = logging.getLogger(__name__) + + +if TYPE_CHECKING: + from collections.abc import Generator, Iterator + from types import CodeType + + import sympy + + from mindnlp.core._dynamo.codegen import PyCodegen + from mindnlp.core._funccore._aot_autograd.schemas import ViewAndMutationMeta + from mindnlp.core._subclasses.fake_tensor import FakeTensorMode + + +""" +core._guards is the definitional source of truth for general purpose guard structures. + +An important thing to keep in mind here is the preservation of layering. There should be no dynamo notions, +and no guard installation notions here. +""" + +COMPILE_ID_PATTERN = re.compile(r"^(?P\d+)/(?P\d+)$") +CA_COMPILE_ID_PATTERN = re.compile( + r"^!(?P\d+)(?:/(?P\d+)/(?P\d+))?$" +) + +# [Note: Updating CompiledId] +# +# CompiledId represents a unique program-level identifier, and we want to keep that +# property as the codebase evolves. This property is relied on even outside of the pytorch +# repo, e.g. tlparse or other internal tooling. The in-memory format can be freely changed, +# as those dependencies only consume the string serialization. +# +# The string form should be: +# 1. Program-level uid: CompileId can uniquely identify a compiled graph. +# 2. Storage efficient: This object is logged in nearly every entry. We should elide symbols when possible. +# 3. Compact: The string form is directly displayed by some tools. Special symbols are okay. + + +# TODO: mark as kw_only=True once we drop support for str: + # NOTE: Keep this in sync with both from_string and the tlparse repo + if self.compiled_autograd_id is not None: + assert (self.frame_id is None) == (self.frame_compile_id is None) + frame_str = "" + if self.frame_id is not None: + frame_str = f"/{self.frame_id}/{self.frame_compile_id}" + + return f"!{self.compiled_autograd_id}{frame_str}" + else: + assert self.frame_id is not None and self.frame_compile_id is not None + return f"{self.frame_id}/{self.frame_compile_id}" + + @classmethod + def from_string(cls, compile_id: Optional[str]) -> Optional[CompileId]: + """ + Factory method that creates a CompileId from its string representation. + Keep this in sync with the __str__ method. + """ + if compile_id is None: + return None + try: + for pattern in (COMPILE_ID_PATTERN, CA_COMPILE_ID_PATTERN): + if match := pattern.match(compile_id): + groups = match.groupdict() + for k, v in groups.items(): + if v is not None: + groups[k] = int(v) + return cls(**groups) # type: ignore[arg-type] + else: + raise ValueError + + except Exception as e: + raise ValueError(f"Invalid compile_id '{compile_id}'") from e + + +class TraceId(NamedTuple): + compile_id: CompileId + # This starts off as 0, and every time we restart analysis it goes + # up by one + attempt: int + + def __str__(self) -> str: + # Keep this in sync with tlparse repo + if self.attempt == 0: + return str(self.compile_id) + else: + return f"{self.compile_id}_{self.attempt}" + + +class GuardSource(enum.Enum): + LOCAL = 0 + GLOBAL = 1 + LOCAL_SPECIALIZED_NN_MODULE = 2 + GLOBAL_SPECIALIZED_NN_MODULE = 3 + CONSTANT = 4 + RANDOM_VALUE = 5 + SHAPE_ENV = 6 + LOCAL_FSDP_MODULE = 7 + GLOBAL_FSDP_MODULE = 8 + BACKWARD_STATE = 9 + EPHEMERAL = 10 + SYNTHETIC_LOCAL = 11 + LOCAL_UNSPECIALIZED_NN_MODULE = 12 + GLOBAL_UNSPECIALIZED_NN_MODULE = 13 + LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE = 14 + GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE = 15 + + def is_fsdp_module(self) -> bool: + return self in (GuardSource.GLOBAL_FSDP_MODULE, GuardSource.LOCAL_FSDP_MODULE) + + def is_specialized_nn_module(self) -> bool: + import core._dynamo.config as config + + if config._unsafe_skip_fsdp_module_guards: + return ( + self + in ( + GuardSource.GLOBAL_SPECIALIZED_NN_MODULE, + GuardSource.LOCAL_SPECIALIZED_NN_MODULE, + ) + or self.is_fsdp_module() + ) + return self in ( + GuardSource.GLOBAL_SPECIALIZED_NN_MODULE, + GuardSource.LOCAL_SPECIALIZED_NN_MODULE, + ) + + def is_unspecialized_nn_module(self) -> bool: + return self in ( + GuardSource.GLOBAL_UNSPECIALIZED_NN_MODULE, + GuardSource.LOCAL_UNSPECIALIZED_NN_MODULE, + GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + ) + + def is_unspecialized_builtin_nn_module(self) -> bool: + return self in ( + GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + ) + + def is_local(self) -> bool: + return self in ( + GuardSource.LOCAL, + GuardSource.LOCAL_SPECIALIZED_NN_MODULE, + GuardSource.LOCAL_FSDP_MODULE, + GuardSource.LOCAL_UNSPECIALIZED_NN_MODULE, + GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + ) + + +""" +Base class for a "GuardBuilder" role. + +The GuardBuilderBase role is to represent a scope within which to build a guard. The name is a little +confusing, as its not a builder, but for the sake of avoiding a lot of renames and keeping the original reference +to torchdynamo's GuardBuilder. + +Note: create_fn is invoked with a GuardBuilderBase and a Guard. A GuardBuilder is chosen based +on GuardSource's select function. + +There is value in keeping this GuardBuilderBase empty to keep layering clean. +""" + + +class GuardBuilderBase: + pass + + +@dataclasses.dataclass(frozen=True) +class SLoc: + framework_loc: Optional[Union[traceback.FrameSummary, str]] + maybe_user_loc: Optional[str] + + def __str__(self) -> str: + floc = ( + self.framework_loc + if isinstance(self.framework_loc, str) + else format_frame(self.framework_loc) + ) + if self.maybe_user_loc is not None: + return f"{self.maybe_user_loc} ({floc})" + else: + return f"({floc})" + + +class ShapeGuard(NamedTuple): + expr: sympy.logic.boolalg.Boolean + sloc: SLoc + size_oblivious: bool + + +@dataclass_slots +@dataclasses.dataclass +class Guard: + # originating_source is the source that called the make_guard method to + # construct this guard object. The property name specifies what exactly it + # is the guard is guarding on. The meaning of the name is dependent on the + # create_fn; you must look at the use-site inside create_fn to know what + # name means. + # + # That being said, although you might think this is just a "name", name is + # usually an arbitrary Python expression that will be evaluated with all + # globals (and locals, if you create a LOCAL guard) to extract the Python + # object that we want to perform guard tests on. This evaluation + # typically happens in GuardBuilder.eval. In these cases, name is + # typically produced by originating_source.name() (not to be confused with + # GuardSource - the property source). + # + # Occasionally, name is not a valid Python expression; sometimes + # it is meaningless. Example create_fns that are like this include + # GRAD_MODE and SHAPE_ENV. + originating_source: Source + create_fn: Callable[[GuardBuilderBase, Guard], Any] + + # Export only. These values are written to at time of guard check_fn creation. + guard_types: Optional[list[str]] = None + code_list: Optional[list[str]] = None + obj_weakref: Optional[object] = None + guarded_class_weakref: Optional[weakref.ReferenceType[Any]] = None + + stack: Optional[CapturedTraceback] = None + user_stack: Optional[traceback.StackSummary] = None + _hash: Optional[int] = None + + def __hash__(self) -> int: + if self._hash is None: + self._hash = hash((self.name, self.source, id(self.create_fn))) + return self._hash + + def sort_key(self) -> tuple[bool, int, int, str, int]: + # Put the duplicate input guards at the end. The duplicate guards have + # two sources while guard.name only considers one source. + + is_duplicate_input = ( + isinstance(self.create_fn, functools.partial) + and self.create_fn.func is core._dynamo.guards.GuardBuilder.DUPLICATE_INPUT + ) + return ( + is_duplicate_input, + self.source.value if self.source else -1, + len(self.name), + self.name, + self.inner_create_fn().__code__.co_firstlineno, + ) + + def __lt__(self, other: Guard) -> bool: + return self.sort_key() < other.sort_key() + + def inner_create_fn(self) -> Callable[[GuardBuilderBase, Guard], Any]: + if isinstance(self.create_fn, functools.partial): + return self.create_fn.func + else: + return self.create_fn + + @property + def name(self) -> str: + return self.originating_source.name() + + @property + def source(self) -> GuardSource: + return self.originating_source.guard_source() + + @staticmethod + def weakref_to_str(obj_weakref: object) -> str: + """ + This is a workaround of a Python weakref bug. + + `obj_weakref` is instance returned by `weakref.ref`, + `str(obj_weakref)` is buggy if the original obj overrides __getattr__, e.g: + + class MyConfig(dict): + def __getattr__(self, x): + return self[x] + + obj = MyConfig(offset=5) + obj_weakref = weakref.ref(obj) + str(obj_weakref) # raise error: KeyError: '__name__' + """ + if isinstance(obj_weakref, weakref.ReferenceType): + obj = obj_weakref() + if obj is not None: + return f"" + else: + return f"" + else: + return str(obj_weakref) + + def __repr__(self) -> str: + s = f""" + {self.source.name.lower() if self.source else ""} {repr(self.name)} {self.inner_create_fn().__name__} + {{ + 'guard_types': {self.guard_types}, + 'code': {self.code_list}, + 'obj_weakref': {self.weakref_to_str(self.obj_weakref)} + 'guarded_class': {self.guarded_class_weakref} + }} + """ + return s + + def __str__(self) -> str: + output = f"Name: {repr(self.name)}\n" + source = self.source.name.lower() if self.source else "" + output += f" Source: {source}\n" + output += f" Create Function: {self.inner_create_fn().__name__}\n" + output += f" Guard Types: {self.guard_types}\n" + output += f" Code List: {self.code_list}\n" + output += f" Object Weakref: {self.weakref_to_str(self.obj_weakref)}\n" + output += f" Guarded Class Weakref: {self.guarded_class_weakref}\n" + return output + + def create(self, builder: GuardBuilderBase) -> Any: + try: + return self.create_fn(builder, self) + except Exception: + log.exception("Error while creating guard:\n%s", str(self).rstrip()) + if self.stack: + log.error("Created at:\n%s", "".join(self.stack.format()[-4:]).rstrip()) + raise + + def is_specialized_nn_module(self) -> bool: + return self.source.is_specialized_nn_module() + + def is_fsdp_module(self) -> bool: + return self.source.is_fsdp_module() + + def is_local(self) -> bool: + return self.source.is_local() + + def set_export_info( + self, + guard_type: str, + guarded_class: Optional[weakref.ReferenceType[Any]], + code_list: list[str], + obj_weakref: object, + ) -> None: + if not self.guard_types: + self.guard_types = [] + + self.guard_types.append(guard_type) + + assert self.guarded_class_weakref in ( + guarded_class, + None, + ), "Guarded class id must be identical, or None" + self.guarded_class_weakref = guarded_class + + if not self.code_list: + self.code_list = code_list + else: + self.code_list.extend(code_list) + + # Some objects are ephemeral, e.g., list[slice(1, 2)]. If we have + # multiple guards on the same object, the weakref can die between the + # invocation of set_export_info calls. So a dead weakref is also + # acceptable. + assert ( + self.obj_weakref in (obj_weakref, None) + or callable(self.obj_weakref) + and self.obj_weakref() is None + ), "Guarded object must be identical, None or ephemeral (dead weakref)" + self.obj_weakref = obj_weakref + + +T = TypeVar("T") + +""" +Parent structure for guard env expressions. +A GuardEnvExpr can have any subtype. +Note: All subtypes must be handled exhaustively in +core._dynamo.guards._parse_guard_env_guards to avoid a RuntimeError. +""" + + +@dataclasses.dataclass(frozen=True) +class GuardEnvExpr: + pass + + +""" +A class representing a pair of duplicate inputs. +input_pos_a and input_pos_b are input positions we have deduped. +""" + + +@dataclasses.dataclass(frozen=True) +class DuplicateInputs(GuardEnvExpr): + input_source_a: Source + input_source_b: Source + + def __post_init__(self) -> None: + assert self.input_source_a != self.input_source_b + + +""" +A class representing storage overlap relations among inputs that aliases the same storage. + +Given that a set of tensors alias the same storage, this guard checks whether they actually +have overlapping storages. + +While non_overlapping_sources represent input tensors that definitely don't have any storage +overlapping with any other input, overlapping_sources represent tensors that either: + +1. Do overlap some other input tensor +2. Might not overlap some other input tensor, but we are not sure +""" + + +@dataclasses.dataclass(frozen=True) +class StorageOverlap(GuardEnvExpr): + overlapping_sources: list[Source] + non_overlapping_sources: list[Source] + + +""" +Checkpointable is an interface for driving state snapshotting, left purposely vague for now. + +copy_graphstate() -> T, a somewhat legacy name, is expected to emit a snapshot of any type that +can also be taken in at restore_graphstate(T) calls. + +When to snapshot, is, at the moment, an implementation detail of upstream callers. Checkpointable +does not provide any guarantees around consistency, idempotency, or safety of calling its APIs, yet. + +In the future, it will have a closer coupling to a generic Checkpoint management system. +""" + + +class Checkpointable(Generic[T]): + @abstractmethod + def copy_graphstate(self) -> T: ... + + @abstractmethod + def restore_graphstate(self, state: T) -> None: ... + + +class GuardsCheckpointState: + """ + The GuardCheckpointState - it is the T of Checkpointable[T] for GuardsContext + """ + + dynamo_guards: set[Guard] = set() + + def __init__(self, dynamo_guards: set[Guard]) -> None: + self.dynamo_guards = dynamo_guards + + def diff(self, other: GuardsCheckpointState) -> Optional[set[Guard]]: + """ + Produces a delta against another GuardsCheckpointState. + + Returns None if no delta is found, otherwise, return a set() of mismatched + Guard type objects. + """ + r = self.dynamo_guards.difference(other.dynamo_guards) + if len(r) == 0: + return None + return r + + def __eq__(self, other: object) -> bool: + if not isinstance(other, GuardsCheckpointState): + return False + return self.diff(other) is None + + +class ModuleContextCheckpointState: + nn_modules: dict[str, core.nn.Module] = {} + + def __init__(self, nn_modules: dict[str, core.nn.Module]) -> None: + self.nn_modules = nn_modules + + def diff(self, other: ModuleContextCheckpointState) -> Optional[set[str]]: + """ + Produces a delta against another ModuleContextCheckpointState. + + Returns None if no delta is found, otherwise, return a set() of mismatched + module key names. + """ + r = set(self.nn_modules.keys()).difference(set(other.nn_modules.keys())) + if len(r) == 0: + return None + return r + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ModuleContextCheckpointState): + return False + return self.diff(other) is None + + +class ModuleContext(Checkpointable[ModuleContextCheckpointState]): + def __init__(self) -> None: + self.nn_modules: dict[str, Any] = {} + + def copy_graphstate(self) -> ModuleContextCheckpointState: + return ModuleContextCheckpointState(dict(self.nn_modules)) + + def restore_graphstate(self, state: ModuleContextCheckpointState) -> None: + assert isinstance(state, ModuleContextCheckpointState) + self.nn_modules = state.nn_modules + + +class GlobalContextCheckpointState: + global_state: dict[str, tuple[Callable, Any]] = {} + + def __init__(self, global_states: dict[str, tuple[Callable, Any]]) -> None: + self.global_state = global_states + + def diff(self, other: GlobalContextCheckpointState) -> Optional[set[str]]: + """ + Produces a delta against another GlobalContextCheckpointState. + + Returns None if no delta is found, otherwise, return a set() of mismatched + global key names. + """ + r = set(self.global_state.keys()).difference(set(other.global_state.keys())) + if len(r) == 0: + return None + return r + + def __eq__(self, other: object) -> bool: + if not isinstance(other, GlobalContextCheckpointState): + return False + return self.diff(other) is None + + +class GlobalContext(Checkpointable[GlobalContextCheckpointState]): + """ + This keeps track of the global torch state during tracing of a function. + For example, core.is_grad_enabled. + """ + + _supported_global_states = { + "grad_enabled", + "autocast_enabled", + "autocast_cpu_enabled", + "autocast_gpu_dtype", + "autocast_cpu_dtype", + "autocast_cache_enabled", + } + + def __init__(self) -> None: + self.global_state: dict[str, tuple[Callable, Any]] = {} + + def copy_graphstate(self) -> GlobalContextCheckpointState: + return GlobalContextCheckpointState(self.global_state) + + def restore_graphstate(self, state: GlobalContextCheckpointState) -> None: + assert isinstance(state, GlobalContextCheckpointState) + self.global_state = state.global_state + assert ( + len(self.global_state) == len(self._supported_global_states) + and set(self.global_state.keys()) == self._supported_global_states + ), "Global state mismatch" + for func, args in self.global_state.values(): + func(args) + + +# Like a Set[Guard] but will record the user stack on all guards at the +# time they were installed at their destination +class GuardsSet: + def __init__(self, inner: Optional[set[Guard]] = None) -> None: + if inner is None: + inner = set() + self.inner = inner + + def __iter__(self) -> Iterator[Guard]: + return iter(self.inner) + + def __len__(self) -> int: + return len(self.inner) + + # Subtraction along with bool is typically used to determine the delta of + # added guards between checkpoints for higher order ops + def __sub__(self, other: GuardsSet) -> GuardsSet: + return GuardsSet(self.inner - other.inner) + + def __bool__(self) -> bool: + return bool(self.inner) + + def add( + self, guard: Guard, *, collect_debug_stack: bool = True, skip: int = 0 + ) -> None: + if guard in self.inner: + return + if collect_debug_stack: + if guard.stack is None: + guard.stack = CapturedTraceback.extract(skip=1 + skip) + if guard.user_stack is None: + guard.user_stack = TracingContext.extract_stack() + self.inner.add(guard) + + def update(self, *others: set[Guard]) -> None: + for o in others: + for g in o: + self.add(g, skip=1) + + def remove_guards_with_source(self, source: Source) -> None: + """Delete all guards that contains a given source""" + from ._dynamo.source import is_from_source + + self.inner = { + g for g in self.inner if not is_from_source(g.originating_source, source) + } + + +""" +A GuardsContext is a checkpointable representation of all the guards in the current tracing +context. It's lifecycle is bound 1:1 to the tracing context, and it should never be instantiated +directly outside of it. For passing around internal state representations of this object, +prefer to extract them with copy_graphstate to produce a GuardsCheckpointState. +""" + + +class GuardsContext(Checkpointable[GuardsCheckpointState]): + def __init__(self) -> None: + self.dynamo_guards: GuardsSet = GuardsSet() + self.aotautograd_guards: list[GuardEnvExpr] = [] + + def copy_graphstate(self) -> GuardsCheckpointState: + return GuardsCheckpointState(set(self.dynamo_guards.inner)) + + def restore_graphstate(self, state: GuardsCheckpointState) -> None: + # NB: "steals" the passed in state + assert isinstance(state, GuardsCheckpointState) + self.dynamo_guards = GuardsSet(state.dynamo_guards) + + +class HopSubgraphCache: + @abstractmethod + def add_dynamo_installed_submodule(self, fn_id: int, identifier: str) -> None: ... + + @abstractmethod + def get_dynamo_installed_submodules(self, fn_id: int) -> list[str]: ... + + @abstractmethod + def add_autograd_key_entry(self, identifier: str, key: Callable) -> None: ... + + @abstractmethod + def get_autograd_key_entry(self, identifier: str) -> Optional[Callable]: ... + + @abstractmethod + def add_proxy_dispatch_entry(self, identifier: str, key: Callable) -> None: ... + + @abstractmethod + def get_proxy_dispatch_entry(self, identifier: str) -> Optional[Callable]: ... + + @abstractmethod + def add_lazy_bwd_entry( + self, + identifier: str, + tangent_metadata: tuple[object], + gmod: core.fx.GraphModule, + ) -> int: ... + + @abstractmethod + def get_lazy_bwd_entry( + self, identifier: str, tangent_metadata: tuple[object] + ) -> tuple[Optional[core.fx.GraphModule], Optional[int]]: ... + + +class InvokeSubgraphCache(HopSubgraphCache): + def __init__(self) -> None: + self.autograd_cache: dict[str, Callable] = {} + self.proxy_dispatch_cache: dict[str, Callable] = {} + self.dynamo_installed_submodules: dict[int, list[str]] = defaultdict(list) + self.lazy_bwd_cache: dict[ + str, dict[tuple[object], tuple[core.fx.GraphModule, int]] + ] = defaultdict(dict) + + def add_dynamo_installed_submodule(self, fn_id: int, identifier: str) -> None: + self.dynamo_installed_submodules[fn_id].append(identifier) + + def get_dynamo_installed_submodules(self, fn_id: int) -> list[str]: + return self.dynamo_installed_submodules.get(fn_id, []) + + def add_autograd_key_entry(self, identifier: str, key: Callable) -> None: + self.autograd_cache[identifier] = key + + def get_autograd_key_entry(self, identifier: str) -> Optional[Callable]: + return self.autograd_cache.get(identifier, None) + + def add_proxy_dispatch_entry(self, identifier: str, key: Callable) -> None: + self.proxy_dispatch_cache[identifier] = key + + def get_proxy_dispatch_entry(self, identifier: str) -> Optional[Callable]: + return self.proxy_dispatch_cache.get(identifier, None) + + def add_lazy_bwd_entry( + self, + identifier: str, + tangent_metadata: tuple[object], + gmod: core.fx.GraphModule, + ) -> int: + # Save the number of existing graph modules in the dictionary to get the suffix + num_gmods = len(self.lazy_bwd_cache[identifier]) + self.lazy_bwd_cache[identifier][tangent_metadata] = (gmod, num_gmods) + return num_gmods + + def get_lazy_bwd_entry( + self, identifier: str, tangent_metadata: tuple[object] + ) -> tuple[Optional[core.fx.GraphModule], Optional[int]]: + if identifier not in self.lazy_bwd_cache: + return (None, None) + + return self.lazy_bwd_cache[identifier].get(tangent_metadata, (None, None)) + + +class HopDispatchSetCache: + def __init__(self) -> None: + # Delayed import to avoid circular dependency + from core._higher_order_ops.invoke_subgraph import invoke_subgraph + + self.hop_cache_map = {invoke_subgraph: InvokeSubgraphCache()} + + def get_cache( + self, op: core._ops.HigherOrderOperator + ) -> Optional[HopSubgraphCache]: + if op not in self.hop_cache_map: + return None + return self.hop_cache_map[op] # type: ignore[index] + + +_TLS = threading.local() + +""" +TracingContext is the source of truth for all currently accumulated information +needed to trace. Its lifecycle is kept 1:1 when using TorchDynamo, but other systems +are open to managing their own TracingContext with that in mind. + +The purpose of TracingContext is not to be a dumping ground, or god object, but rather to avoid +having to plumb complex subsystems across multiple verticals. + +Ex: A common example is guard accumulation between dynamo, shape_env, aot_autograd, and inductor. +Accessing the current tracing context via +TracingContext.get() allows users to accumulate their own guards for processing, without needing to know how +to plumb objects back up to where frame interpretation happened. + +Note that you can end up with multiple TracingContext for a single compilation +of a frame, as we reset the TracingContext whenever we restart analysis. +CompileContext is a more overarching context that encompasses multiple restarts. +""" + + +class CompileContext: + @staticmethod + def get() -> CompileContext: + assert _TLS.compile_context is not None + return _TLS.compile_context + + @staticmethod + def try_get() -> Optional[CompileContext]: + return getattr(_TLS, "compile_context", None) + + def __init__(self, compile_id: Optional[CompileId]) -> None: + assert compile_id is None or isinstance(compile_id, CompileId) + self.compile_id: Optional[CompileId] = compile_id + self.attempt = 0 + # Verbose ShapeEnv guards produced. + self.shape_env_guards: list[str] = [] + + @staticmethod + def current_compile_id() -> Optional[CompileId]: + self = CompileContext.try_get() + if self is None: + return None + return self.compile_id + + @staticmethod + def current_trace_id() -> Optional[TraceId]: + self = CompileContext.try_get() + if self is None: + return None + if self.compile_id is None: + return None + return TraceId(self.compile_id, self.attempt) + + +class TracingContext: + """ + Provides the currently installed TracingContext, or None. + + Note that it is a staticmethod, and invocations outside of `with tracing()` (see below), are valid but + will return None. + """ + + @staticmethod + def try_get() -> Optional[TracingContext]: + return getattr(_TLS, "tracing_context", None) + + @staticmethod + def get() -> TracingContext: + if ctx := TracingContext.try_get(): + return ctx + raise RuntimeError( + "TracingContext.get() must be called within an ongoing trace." + ) + + def __init__(self, fake_mode: Optional[FakeTensorMode]) -> None: + self.guards_context = GuardsContext() + self.module_context = ModuleContext() + self.global_context = GlobalContext() + self.previously_inlined_functions: dict[Any, Any] = dict() + self.previously_cleaned_instructions: dict[Any, Any] = dict() + self.fake_mode: Optional[FakeTensorMode] = fake_mode + self.frame_summary_stack: list[traceback.FrameSummary] = [] + # This is morally part of frame_summary_stack, but it is kept separate + # for clarity. As we process a frame, this variable gets updated + # to keep track of what line we are in the function. We make a + # function call, this gets cleared and the frame location is pushed + # to frame_summary_stack (prepping this variable for the inner frame's + # progress) + self.loc_in_frame: Optional[tuple[str, int, str]] = None + # this is only set after aot_autograd + self.fw_metadata: Optional[ViewAndMutationMeta] = None + # this is only set after aot_autograd + self.aot_graph_name: Optional[list[str]] = None + self.params_flat: Optional[list[Any]] = None + self.params_flat_unwrap_subclasses: Optional[list[Any]] = None + self.params_unwrapped_to_flat_index: Optional[list[Any]] = None + # this is for extended return calling convention from backend + # compiler to aot_autograd + # Per output, what the compiler specified stride of the output is, + # or None if no stride is known. This is always the HINT, it + # is never a SymInt (it would be better if it was a SymInt, but + # I can't conveniently get this from Inductor atm. Also, be + # careful not to accidentally induce guards on the SymInt if + # you ever do change this in aot_autograd.py; you should check + # on permutations preferentially.) + self.output_strides: Optional[list[Optional[tuple[int, ...]]]] = None + # When this is True, whenever we encounter an int in Dynamo tracing, + # we will (1) force unspec it and (2) force it as a size-like unbacked + # integer. This is currently used when processing certain lists of + # ints that are known to be size-like and may have 0/1 entries that we + # must not specialize on. + self.force_unspec_int_unbacked_size_like = False + # See note [Tensor Fakification and Symbol Caching] + self.tensor_to_context = WeakTensorKeyDictionary() + + # If this true, Aot Autograd will return output Fake Tensors with appropriate + # meta on the first invocation + # see note: [Returning Fake Tensors on First AOT Autograd Call] + self.fakify_first_call = False + self.hop_dispatch_set_cache = HopDispatchSetCache() + # list of code objects for inlined functions + self.traced_code: list[CodeType] = [] + + def clear(self) -> None: + # Look at the note in output_graph.py in function `save_global_state` + # for the context on clearing global context. + self.global_context.global_state = {} + self.previously_inlined_functions.clear() + self.previously_cleaned_instructions.clear() + + @staticmethod + @contextmanager + def patch(**kwargs: Any) -> Generator[None, None, None]: + prior = {} + ctx = TracingContext.get() + + for key in kwargs.keys(): + # KeyError on invalid entry + prior[key] = getattr(ctx, key) + for key, val in kwargs.items(): + setattr(ctx, key, val) + try: + yield + finally: + for key, val in prior.items(): + setattr(ctx, key, val) + + @staticmethod + def extract_stack() -> traceback.StackSummary: + self = TracingContext.try_get() + if self is None: + return traceback.StackSummary() + stack = self.frame_summary_stack + if self.loc_in_frame is not None: + stack = stack + [self._populate_loc_in_frame_summary()] + return traceback.StackSummary.from_list(stack) + + def _populate_loc_in_frame_summary(self) -> traceback.FrameSummary: + assert self.loc_in_frame is not None + filename, lineno, frame_name = self.loc_in_frame + return traceback.FrameSummary(filename, lineno, frame_name, lookup_line=False) + + # Call this when you want to call into some code that isn't necessarily + # associated with the current frame state + @staticmethod + @contextlib.contextmanager + def clear_frame() -> Generator[None, None, None]: + tc = TracingContext.get() + with ( + unittest.mock.patch.object(tc, "frame_summary_stack", []), + unittest.mock.patch.object(tc, "loc_in_frame", None), + ): + try: + yield + except Exception as e: + # Prevent real_stack from getting attached + # + # The invariant is that if an Exception as real_stack, we've + # appropriately attached a user stack and we no longer need to + # attach anything. Because we cannot conveniently interpose + # when an exception is thrown, we instead interpose everywhere + # we set what the user stack is set (using the context + # manager). However, our compiler stack does "tail calls" + # (when it calls into user compiler), at which point the + # parent exception frames would incorrectly attach an + # incorrect frame. + # + # However, if, somehow, someone raised an exception with this + # scope that had a stack (for example, because they are + # restoring the user stack state appropriately as they process + # node by node), we should respect it. Thus, we cannot + # unconditionally set None. + if not hasattr(e, "real_stack"): + e.real_stack = None # type: ignore[attr-defined] + raise + + @staticmethod + @contextlib.contextmanager + def current_frame( + frame_summary: Optional[traceback.FrameSummary], + ) -> Generator[None, None, None]: + # frame_summary can be None to solely take advantage of real_stack + # attachment to thrown exceptions + tc = TracingContext.get() + if frame_summary is not None: + tc.frame_summary_stack.append(frame_summary) + old = tc.loc_in_frame + tc.loc_in_frame = None + try: + yield + except Exception as e: + if not hasattr(e, "real_stack"): + e.real_stack = tc.extract_stack() # type: ignore[attr-defined] + raise + finally: + if frame_summary is not None: + tc.frame_summary_stack.pop() + tc.loc_in_frame = old + + @staticmethod + @contextlib.contextmanager + def report_output_strides() -> Generator[ + Optional[list[Optional[tuple[int, ...]]]], None, None + ]: + tc = TracingContext.try_get() + if tc is None: + yield None + return + old_output_strides = tc.output_strides + tc.output_strides = [] + try: + yield tc.output_strides + finally: + tc.output_strides = old_output_strides + + @staticmethod + def set_current_loc(filename: str, lineno: int, frame_name: str) -> None: + # Save the current location in the frame. Lazily generate the + # framesummary. + TracingContext.get().loc_in_frame = (filename, lineno, frame_name) + + @staticmethod + def get_traced_code() -> Optional[list[CodeType]]: + tc = TracingContext.try_get() + if tc is None: + return None + return tc.traced_code + + +@contextmanager +def compile_context( + context: Optional[CompileContext], +) -> Generator[Optional[CompileContext], None, None]: + old_context = getattr(_TLS, "compile_context", None) + _TLS.compile_context = context + try: + yield context + finally: + _TLS.compile_context = old_context + + +@contextmanager +def tracing( + context: Optional[TracingContext], +) -> Generator[Optional[TracingContext], None, None]: + """ + This function installs the passed in tracing context as a dynamic scoped + global variable. + + Calls to TracingContext.get() while not under a `with tracing()` context + will return None. + """ + old_context = getattr(_TLS, "tracing_context", None) + _TLS.tracing_context = context + try: + yield context + except Exception as e: + if not hasattr(e, "real_stack") and context is not None: + e.real_stack = context.extract_stack() # type: ignore[attr-defined] + raise + finally: + if ( + context is not None + and context.fake_mode is not None + and context.fake_mode.shape_env is not None + ): + context.fake_mode.shape_env.cleanup() + _TLS.tracing_context = old_context + + +# Subclasses can be found in torch/_dynamo/source.py +# TODO(voz): Consider a toplevel torch/_source.py +@dataclasses.dataclass(frozen=True) +class Source: + def is_dict_key(self) -> bool: + return False + + def is_ephemeral(self) -> bool: + return False + + def reconstruct(self, codegen: PyCodegen) -> None: + raise NotImplementedError + + def guard_source(self) -> GuardSource: + raise NotImplementedError + + def name(self) -> str: + raise NotImplementedError + + def make_guard(self, fn: Callable[..., Any]) -> Guard: + if self.guard_source() is GuardSource.CONSTANT: + raise NotImplementedError + return Guard(self, fn) + + def is_specialized_nn_module(self) -> bool: + return self.guard_source().is_specialized_nn_module() + + def subguards_allowed(self) -> bool: + """True if you can guard on attributes of this""" + return self.guard_source() != GuardSource.SYNTHETIC_LOCAL + + +# Subclasses can be found in torch/_dynamo/source.py +@dataclasses.dataclass(frozen=True) +class ChainedSource(Source): + base: Source + + def is_dict_key(self) -> bool: + # Recurse until you either hit a ConstDictKey or a Source + return self.base.is_dict_key() + + def is_ephemeral(self) -> bool: + return self.base.is_ephemeral() + + def get_base(self) -> Source: + current: Source = self + while isinstance(current, ChainedSource): + current = current.base + return current + + +def detect_fake_mode(inputs: Any = None) -> Optional[FakeTensorMode]: + """ + Attempts to "detect" what the current fake mode is. If there is one ambiently + available from TracingContext, we preferentially use that. Otherwise, we + heuristically detect the fake mode via the following sources, in order of + priority: + + - Currently active fake mode on stack + - Fake mode associated with passed in tensors (inputs does not + have to be flattened) + """ + from core._subclasses.fake_tensor import FakeTensor, FakeTensorMode + + fake_modes = [] + + if context := TracingContext.try_get(): + fake_mode = context.fake_mode + if fake_mode is not None: + fake_modes.append((fake_mode, "tracing context", 0)) + + from core.utils._python_dispatch import _get_current_dispatch_mode_stack + + for i, m in enumerate(reversed(_get_current_dispatch_mode_stack())): + if isinstance(m, FakeTensorMode): + fake_modes.append((m, "active fake mode", i)) + + flat_inputs = pytree.tree_leaves(inputs) + for i, flat_input in enumerate(flat_inputs): + if isinstance(flat_input, FakeTensor): + fake_modes.append((flat_input.fake_mode, "fake tensor input", i)) + + if fake_modes: + fake_mode, desc1, i1 = fake_modes[0] + for m, desc2, i2 in fake_modes[1:]: + assert fake_mode is m, ( + f"fake mode ({fake_mode}) from {desc1} {i1} doesn't match mode ({m}) from {desc2} {i2}\n\n" + f"fake mode from {desc1} {i1} allocated at:\n{fake_mode.stack}\n" + f"fake mode from {desc2} {i2} allocated at:\n{m.stack}" + ) + return fake_mode + else: + return None + + +def active_fake_mode() -> Optional[FakeTensorMode]: + """ + Inspects the dispatch mode stack for an active fake mode and returns it. + Returns None if no fake mode is active. + """ + from core._subclasses.fake_tensor import FakeTensorMode + from core.utils._python_dispatch import _get_current_dispatch_mode_stack + + for _, m in enumerate(reversed(_get_current_dispatch_mode_stack())): + if isinstance(m, FakeTensorMode): + return m + + return None \ No newline at end of file diff --git a/mindnlp/core/_logging/__init__.py b/mindnlp/core/_logging/__init__.py new file mode 100644 index 000000000..6dbd83b29 --- /dev/null +++ b/mindnlp/core/_logging/__init__.py @@ -0,0 +1,20 @@ +# Top level logging module for torch logging +# Design doc: https://docs.google.com/document/d/1ZRfTWKa8eaPq1AxaiHrq4ASTPouzzlPiuquSBEJYwS8/edit# +# Simple setup for onboarding (see above doc for more detail): +# 1. register any top-level log qualified name for your module in torch._logging._registrations (see there for examples) +# 2. register any artifacts ( below) in torch._logging._registrations +# a. call getArtifactLogger(__name__, ) at your logging site instead of the standard logger to log your artifact +import mindnlp.core._logging._registrations + +from ._internal import ( + _init_logs, + DEFAULT_LOGGING, + dtrace_structured, + get_structured_logging_overhead, + getArtifactLogger, + hide_warnings, + LazyString, + set_logs, + trace_structured, + warning_once, +) \ No newline at end of file diff --git a/mindnlp/core/_logging/_internal.py b/mindnlp/core/_logging/_internal.py new file mode 100644 index 000000000..aab44a5f0 --- /dev/null +++ b/mindnlp/core/_logging/_internal.py @@ -0,0 +1,1433 @@ +# mypy: allow-untyped-defs +import contextlib +import functools +import hashlib +import importlib.util +import itertools +import json +import logging +import os +import os.path +import pathlib +import re +import sys +import tempfile +import time +import warnings +from collections import defaultdict +from dataclasses import dataclass, field +from typing import Any, Callable, Generic, Optional, Union +from typing_extensions import ParamSpec +from weakref import WeakSet + +import mindnlp.core._logging.structured +from mindnlp.core._guards import CompileId +from mindnlp.core._utils_internal import log_trace_structured_event +from mindnlp.core.utils._traceback import CapturedTraceback + + +_P = ParamSpec("_P") + +log = logging.getLogger(__name__) + +# This is a synthetic logger which doesn't correspond to an actual logger, +# but handles all of our "tracing" logging, which is structured and doesn't go +# to stderr but always goes to a dedicated log file. We don't put these +# loggers in the classic module hierarchy, because we don't want a suppression +# of logs to also cause a trace to get suppressed (traces typically are not +# collected, unless we are in prod, in which case they always are collected.) +# +# TODO: Maybe we should allow for some sub-hierarchy so you can control which +# traces you want to collect, for performance reasons. +# +# See https://docs.google.com/document/d/1CX_hJ0PNy9f3R1y8TJrfkSeLkvGjjjLU84BSXgS2AZ8/edit +trace_log = logging.getLogger("core.__trace") + +DEFAULT_LOG_LEVEL = logging.WARNING +LOG_ENV_VAR = "TORCH_LOGS" +LOG_OUT_ENV_VAR = "TORCH_LOGS_OUT" +LOG_FORMAT_ENV_VAR = "TORCH_LOGS_FORMAT" +LOG_TRACE_ID_FILTER = "TORCH_LOGS_TRACE_ID_FILTER" +TRACE_ENV_VAR = "TORCH_TRACE" +DTRACE_ENV_VAR = "TORCH_DTRACE" + +LOG_TRACE_HANDLER: Optional["LazyTraceHandler"] = None + +GET_DTRACE_STRUCTURED = False + + +@dataclass +class LogRegistry: + # shorthand name to log qualified name + # Note: this only contains loggers registered + # from register_log + # e.g. "dynamo" -> "core._dynamo" + log_alias_to_log_qnames: dict[str, list[str]] = field(default_factory=dict) + + # artifact logger qualified names, + # this is populated lazily, as calls to getArtifactLogger + # currently formatted as .__ + # e.g. "core._dynamo.convert_frame.__guards" + artifact_log_qnames: set[str] = field(default_factory=set) + + # child logs of registered logs if specified via open + # registration by the user (ie placing "core._dynamo.output_graph" in the env var) + # these need to be tracked so their levels can be reset properly + # e.g. "core._dynamo.output_graph" + child_log_qnames: set[str] = field(default_factory=set) + + # artifact names, populated by register_artifact + # e.g. "guards" + artifact_names: set[str] = field(default_factory=set) + + # Artifacts that should be visible by default in the error message + visible_artifacts: set[str] = field(default_factory=set) + + # A short description of each artifact + artifact_descriptions: dict[str, str] = field(default_factory=dict) + + # artifacts which are not displayed unless explicitly named in the + # settings. Ex. output_code is NOT displayed even if the inductor + # log level is set to DEBUG. It must be explicitly named in the settings + off_by_default_artifact_names: set[str] = field(default_factory=set) + + # logging format string for artifacts + artifact_log_formatters: dict[str, logging.Formatter] = field(default_factory=dict) + + def is_artifact(self, name): + return name in self.artifact_names + + def is_log(self, alias): + return alias in self.log_alias_to_log_qnames + + # register a log with an alias + def register_log(self, alias, log_qnames: Union[str, list[str]]) -> None: + if isinstance(log_qnames, str): + log_qnames = [log_qnames] + self.log_alias_to_log_qnames[alias] = log_qnames + + # register an artifact name + def register_artifact_name( + self, name, description, visible, off_by_default, log_format + ) -> None: + self.artifact_names.add(name) + if visible: + self.visible_artifacts.add(name) + self.artifact_descriptions[name] = description + + # if off by default, don't enable it + # when log_name's log_level is set to DEBUG + if off_by_default: + self.off_by_default_artifact_names.add(name) + + if log_format is not None: + self.artifact_log_formatters[name] = logging.Formatter(log_format) + + # register the qualified name of an artifact log + # this is needed to know which logs need to be reset + # whenever the log_state is changed + def register_artifact_log(self, artifact_log_qname) -> None: + self.artifact_log_qnames.add(artifact_log_qname) + + def register_child_log(self, log_qname) -> None: + self.child_log_qnames.add(log_qname) + + # flattens all the qnames together (TODO: consider memoizing?) + def get_log_qnames(self) -> set[str]: + return set(itertools.chain.from_iterable(self.log_alias_to_log_qnames.values())) + + def get_artifact_log_qnames(self): + return set(self.artifact_log_qnames) + + def get_child_log_qnames(self): + return set(self.child_log_qnames) + + def is_off_by_default(self, artifact_qname): + return artifact_qname in self.off_by_default_artifact_names + + +@dataclass +class LogState: + # qualified log names -> currently set log level + log_qname_to_level: dict[str, str] = field(default_factory=dict) + + # the set of currently enabled artifacts + artifact_names: set[str] = field(default_factory=set) + + def enable_artifact(self, artifact_name) -> None: + self.artifact_names.add(artifact_name) + + def is_artifact_enabled(self, name): + return name in self.artifact_names + + def enable_log(self, log_qnames, log_level) -> None: + if isinstance(log_qnames, str): + log_qnames = [log_qnames] + for log_qname in log_qnames: + self.log_qname_to_level[log_qname] = log_level + + def get_log_level_pairs(self): + """Returns all qualified module names for which the user requested + explicit logging settings. + + .. warning: + + This function used to return all loggers, regardless of whether + or not the user specified them or not; it now only returns logs + which were explicitly mentioned by the user (and torch, which + always is implicitly requested when we initialize our logging + subsystem.) + """ + return self.log_qname_to_level.items() + + def clear(self) -> None: + self.log_qname_to_level.clear() + self.artifact_names.clear() + + +log_registry = LogRegistry() +log_state = LogState() + +# sample usage: core._logging.set_logs(**core._logging.DEFAULT_LOGGING) +DEFAULT_LOGGING = { + "dynamo": logging.INFO, + "aot": logging.INFO, + "inductor": logging.INFO, + "fsdp": logging.INFO, + "ddp_graphs": True, + "graph_breaks": True, + "guards": True, + "recompiles": True, + "dynamic": logging.INFO, +} + + +def set_logs( + *, + all: Optional[int] = None, + dynamo: Optional[int] = None, + aot: Optional[int] = None, + autograd: Optional[int] = None, + dynamic: Optional[int] = None, + inductor: Optional[int] = None, + distributed: Optional[int] = None, + c10d: Optional[int] = None, + ddp: Optional[int] = None, + fsdp: Optional[int] = None, + dtensor: Optional[int] = None, + onnx: Optional[int] = None, + bytecode: bool = False, + aot_graphs: bool = False, + aot_joint_graph: bool = False, + ddp_graphs: bool = False, + graph: bool = False, + graph_code: bool = False, + graph_code_verbose: bool = False, + graph_breaks: bool = False, + graph_sizes: bool = False, + guards: bool = False, + recompiles: bool = False, + recompiles_verbose: bool = False, + trace_source: bool = False, + trace_call: bool = False, + trace_bytecode: bool = False, + output_code: bool = False, + kernel_code: bool = False, + schedule: bool = False, + perf_hints: bool = False, + pre_grad_graphs: bool = False, + post_grad_graphs: bool = False, + ir_pre_fusion: bool = False, + ir_post_fusion: bool = False, + onnx_diagnostics: bool = False, + fusion: bool = False, + overlap: bool = False, + export: Optional[int] = None, + modules: Optional[dict[str, Union[int, bool]]] = None, + cudagraphs: bool = False, + sym_node: bool = False, + compiled_autograd: bool = False, + compiled_autograd_verbose: bool = False, + cudagraph_static_inputs: bool = False, + benchmarking: bool = False, + autotuning: bool = False, + graph_region_expansion: bool = False, + inductor_metrics: bool = False, + hierarchical_compile: bool = False, + compute_dependencies: bool = False, +) -> None: + """ + Sets the log level for individual components and toggles individual log + artifact types. + + .. warning:: This feature is a prototype and may have compatibility + breaking changes in the future. + + .. note:: The ``TORCH_LOGS`` environment variable has complete precedence + over this function, so if it was set, this function does nothing. + + A component is a set of related features in PyTorch. All of the log + messages emitted from a given component have their own log levels. If the + log level of a particular message has priority greater than or equal to its + component's log level setting, it is emitted. Otherwise, it is suppressed. + This allows you to, for instance, silence large groups of log messages that + are not relevant to you and increase verbosity of logs for components that + are relevant. The expected log level values, ordered from highest to lowest + priority, are: + + * ``logging.CRITICAL`` + * ``logging.ERROR`` + * ``logging.WARNING`` + * ``logging.INFO`` + * ``logging.DEBUG`` + * ``logging.NOTSET`` + + See documentation for the Python ``logging`` module for more information on + log levels: ``_ + + An artifact is a particular type of log message. Each artifact is assigned + to a parent component. A component can emit many different kinds of + artifacts. In general, an artifact is emitted if either its corresponding + setting in the argument list below is turned on or if its parent component + is set to a log level less than or equal to the log level of the artifact. + + Keyword args: + all (:class:`Optional[int]`): + The default log level for all components. Default: ``logging.WARN`` + + dynamo (:class:`Optional[int]`): + The log level for the TorchDynamo component. Default: ``logging.WARN`` + + aot (:class:`Optional[int]`): + The log level for the AOTAutograd component. Default: ``logging.WARN`` + + autograd (:class:`Optional[int]`): + The log level for autograd. Default: ``logging.WARN`` + + inductor (:class:`Optional[int]`): + The log level for the TorchInductor component. Default: ``logging.WARN`` + + dynamic (:class:`Optional[int]`): + The log level for dynamic shapes. Default: ``logging.WARN`` + + distributed (:class:`Optional[int]`): + Whether to log c10d communication operations and other debug info from PyTorch Distributed components. + Default: ``logging.WARN`` + + c10d (:class:`Optional[int]`): + Whether to log c10d communication operations related debug info in PyTorch Distributed components. + Default: ``logging.WARN`` + + ddp (:class:`Optional[int]`): + Whether to log debug info related to ``DistributedDataParallel``(DDP) from PyTorch Distributed components. + Default: ``logging.WARN`` + + fsdp (:class:`Optional[int]`): + Whether to log debug info related to ``FullyShardedDataParallel``(FSDP) in PyTorch Distributed components. + Default: ``logging.WARN`` + + dtensor (:class:`Optional[int]`): + Whether to log debug info related to ``DTensor``(DTensor) in PyTorch Distributed components. + Default: ``logging.WARN`` + + onnx (:class:`Optional[int]`): + The log level for the ONNX exporter component. Default: ``logging.WARN`` + + bytecode (:class:`bool`): + Whether to emit the original and generated bytecode from TorchDynamo. + Default: ``False`` + + aot_graphs (:class:`bool`): + Whether to emit the graphs generated by AOTAutograd. Default: ``False`` + + aot_joint_graph (:class:`bool`): + Whether to emit the joint forward-backward graph generated by AOTAutograd. Default: ``False`` + + ddp_graphs (:class:`bool`): + Whether to emit graphs generated by DDPOptimizer. Default: ``False`` + + graph (:class:`bool`): + Whether to emit the graph captured by TorchDynamo in tabular format. + Default: ``False`` + + graph_code (:class:`bool`): + Whether to emit the python source of the graph captured by TorchDynamo. + Default: ``False`` + + graph_code_verbose (:class:`bool`): + Whether to emit verbose/intermediate FX pass logs for graph code. Default: ``False`` + + graph_breaks (:class:`bool`): + Whether to emit the graph breaks encountered by TorchDynamo. + Default: ``False`` + + graph_sizes (:class:`bool`): + Whether to emit tensor sizes of the graph captured by TorchDynamo. + Default: ``False`` + + guards (:class:`bool`): + Whether to emit the guards generated by TorchDynamo for each compiled + function. Default: ``False`` + + recompiles (:class:`bool`): + Whether to emit a guard failure reason and message every time + TorchDynamo recompiles a function. Default: ``False`` + + recompiles_verbose (:class:`bool`): + Whether to emit all guard failure reasons when TorchDynamo recompiles + a function, even those that are not actually run. Default: ``False`` + + trace_source (:class:`bool`): + Whether to emit when TorchDynamo begins tracing a new line. Default: ``False`` + + trace_call (:class:`bool`): + Whether to emit detailed line location when TorchDynamo creates an FX node + corresponding to function call. Python 3.11+ only. Default: ``False`` + + trace_bytecode (:class:`bool`): + Whether to emit bytecode instructions and traced stack state as TorchDynamo + traces bytecode. Default: ``False`` + + output_code (:class:`bool`): + Whether to emit the TorchInductor output code on a per-graph basis. Default: ``False`` + + kernel_code (:class:`bool`): + Whether to emit the TorchInductor output code on a per-kernel bases. Default: ``False`` + + schedule (:class:`bool`): + Whether to emit the TorchInductor schedule. Default: ``False`` + + perf_hints (:class:`bool`): + Whether to emit the TorchInductor perf hints. Default: ``False`` + + pre_grad_graphs (:class:`bool`): + Whether to emit the graphs before inductor grad passes. Default: ``False`` + + post_grad_graphs (:class:`bool`): + Whether to emit the graphs generated by after post grad passes. Default: ``False`` + + ir_pre_fusion (:class:`bool`): + Whether to emit the graphs before inductor fusion passes. Default: ``False`` + + ir_post_fusion (:class:`bool`): + Whether to emit the graphs after inductor fusion passes. Default: ``False`` + + onnx_diagnostics (:class:`bool`): + Whether to emit the ONNX exporter diagnostics in logging. Default: ``False`` + + fusion (:class:`bool`): + Whether to emit detailed Inductor fusion decisions. Default: ``False`` + + overlap (:class:`bool`): + Whether to emit detailed Inductor compute/comm overlap decisions. Default: ``False`` + + sym_node (:class:`bool`): + Whether to emit debug info for various SymNode opterations. Default: ``False`` + + export (:class:`Optional[int]`): + The log level for export. Default: ``logging.WARN`` + + benchmarking (:class:`bool`): + Whether to emit detailed Inductor benchmarking information. Default: ``False`` + + modules (dict): + This argument provides an alternate way to specify the above log + component and artifact settings, in the format of a keyword args + dictionary given as a single argument. There are two cases + where this is useful (1) if a new log component or artifact has + been registered but a keyword argument for it has not been added + to this function and (2) if the log level for an unregistered module + needs to be set. This can be done by providing the fully-qualified module + name as the key, with the log level as the value. Default: ``None`` + + cudagraph_static_inputs (:class:`bool`): + Whether to emit debug info for cudagraph static input detection. Default: ``False`` + + autotuning (:class:`bool`): + Autotuning choice logs, such as kernel source, perf, and tuning parameters. Default: ``False`` + + graph_region_expansion (:class:`bool`): + Whether to emit the detailed steps of the duplicate graph region tracker expansion algorithm. Default: ``False`` + + inductor_metrics (:class:`bool`): + Whether to estimate the runtimes of the nodes in a graph and log them to the metrics table. Default: ``False`` + + hierarchical_compile (:class:`bool`): + Whether to emit debug info for hierarchical compilation. Default: ``False`` + + Example:: + + >>> # xdoctest: +SKIP + >>> import logging + + # The following changes the "dynamo" component to emit DEBUG-level + # logs, and to emit "graph_code" artifacts. + + >>> core._logging.set_logs(dynamo=logging.DEBUG, graph_code=True) + + # The following enables the logs for a different module + + >>> core._logging.set_logs(modules={"unregistered.module.name": logging.DEBUG}) + """ + # ignore if env var is set + if LOG_ENV_VAR in os.environ: + log.warning( + "Using TORCH_LOGS environment variable for log settings, ignoring call to set_logs" + ) + return + + log_state.clear() + + modules = modules or {} + + def _set_logs(**kwargs) -> None: + for alias, val in itertools.chain(kwargs.items(), modules.items()): # type: ignore[union-attr] + if val is None: + continue + + if log_registry.is_artifact(alias): + if not isinstance(val, bool): + raise ValueError( + f"Expected bool to enable artifact {alias}, received {val}" + ) + + if val: + log_state.enable_artifact(alias) + elif log_registry.is_log(alias) or alias in log_registry.child_log_qnames: + if val not in logging._levelToName: + raise ValueError( + f"Unrecognized log level for log {alias}: {val}, valid level values " + f"are: {','.join([str(k) for k in logging._levelToName.keys()])}" + ) + + log_state.enable_log( + log_registry.log_alias_to_log_qnames.get(alias, alias), val + ) + elif _is_valid_module(alias): + if not _has_registered_parent(alias): + log_registry.register_log(alias, alias) + else: + log_registry.register_child_log(alias) + log_state.enable_log( + log_registry.log_alias_to_log_qnames.get(alias, alias), val + ) + else: + raise ValueError( + f"Unrecognized log or artifact name passed to set_logs: {alias}" + ) + + _init_logs() + + _set_logs( + torch=all, + dynamo=dynamo, + aot=aot, + autograd=autograd, + inductor=inductor, + dynamic=dynamic, + bytecode=bytecode, + aot_graphs=aot_graphs, + aot_joint_graph=aot_joint_graph, + ddp_graphs=ddp_graphs, + distributed=distributed, + c10d=c10d, + ddp=ddp, + fsdp=fsdp, + dtensor=dtensor, + graph=graph, + graph_code=graph_code, + graph_code_verbose=graph_code_verbose, + graph_breaks=graph_breaks, + graph_sizes=graph_sizes, + guards=guards, + recompiles=recompiles, + recompiles_verbose=recompiles_verbose, + trace_source=trace_source, + trace_call=trace_call, + trace_bytecode=trace_bytecode, + output_code=output_code, + kernel_code=kernel_code, + schedule=schedule, + perf_hints=perf_hints, + pre_grad_graphs=pre_grad_graphs, + post_grad_graphs=post_grad_graphs, + ir_pre_fusion=ir_pre_fusion, + ir_post_fusion=ir_post_fusion, + onnx=onnx, + onnx_diagnostics=onnx_diagnostics, + fusion=fusion, + overlap=overlap, + sym_node=sym_node, + export=export, + cudagraphs=cudagraphs, + compiled_autograd=compiled_autograd, + compiled_autograd_verbose=compiled_autograd_verbose, + cudagraph_static_inputs=cudagraph_static_inputs, + benchmarking=benchmarking, + autotuning=autotuning, + graph_region_expansion=graph_region_expansion, + inductor_metrics=inductor_metrics, + hierarchical_compile=hierarchical_compile, + compute_dependencies=compute_dependencies, + ) + + +def get_loggers() -> list[logging.Logger]: + """ + Returns: a list of all registered loggers + """ + return [logging.getLogger(qname) for qname in log_registry.get_log_qnames()] + + +def register_log(setting_name, log_name) -> None: + """ + Enables a log to be controlled by the env var and user API with the setting_name + Args: + setting_name: the shorthand name used in the env var and user API + log_name: the log name that the setting_name is associated with + """ + log_registry.register_log(setting_name, log_name) + + +def register_artifact( + setting_name, description, visible=False, off_by_default=False, log_format=None +) -> None: + """ + Enables an artifact to be controlled by the env var and user API with name + Args: + setting_name: the shorthand name used in the env var and user API + description: A description of what this outputs + visible: Whether it gets suggested to users by default + off_by_default: whether this artifact should be logged when the ancestor loggers + are enabled at level DEBUG + """ + log_registry.register_artifact_name( + setting_name, description, visible, off_by_default, log_format + ) + + +def getArtifactLogger(module_qname, artifact_name) -> logging.Logger: + if artifact_name not in log_registry.artifact_names: + raise ValueError( + f"Artifact name: {repr(artifact_name)} not registered," + f"please call register_artifact({repr(artifact_name)}) in core._logging.registrations." + ) + qname = module_qname + f".__{artifact_name}" + log = logging.getLogger(qname) + log.artifact_name = artifact_name # type: ignore[attr-defined] + log_registry.register_artifact_log(qname) + configure_artifact_log(log) + return log + + +INCR_VERBOSITY_CHAR = "+" +DECR_VERBOSITY_CHAR = "-" +VERBOSITY_REGEX = ( + "(" + + "|".join([re.escape(INCR_VERBOSITY_CHAR), re.escape(DECR_VERBOSITY_CHAR)]) + + "?)" +) + + +def configure_artifact_log(log) -> None: + # If the artifact is off by default, then it should only be logged when explicitly + # enabled; set propagate to False so that this artifact is not propagated + # to its ancestor logger + if log_registry.is_off_by_default(log.artifact_name): + log.propagate = False + + # enable artifact logging when explicitly enabled + if log_state.is_artifact_enabled(log.artifact_name): + log.setLevel(logging.DEBUG) + log.propagate = True + + +# match a comma separated list of loggable names (whitespace allowed after commas) +def _gen_settings_regex(): + return re.compile(r"((\+|-)?[\w\.]+,\s*)*(\+|-)?[\w\.]+?") + + +def _validate_settings(settings): + return re.fullmatch(_gen_settings_regex(), settings) is not None + + +def help_message(verbose=False): + def pad_to(s, length=30): + assert len(s) <= length + return s + " " * (length - len(s)) + + if verbose: + printed_artifacts = log_registry.artifact_names + else: + printed_artifacts = log_registry.visible_artifacts + if verbose: + heading = "All registered names" + else: + heading = "Visible registered names (use TORCH_LOGS='+help' for full list)" + lines = ( + ["all"] + + sorted(log_registry.log_alias_to_log_qnames.keys()) + + sorted( + [ + f"{pad_to(name)}\t{log_registry.artifact_descriptions[name]}" + for name in printed_artifacts + ] + ) + ) + setting_info = " " + "\n ".join(lines) + examples = """ +Examples: + TORCH_LOGS="+dynamo,aot" will set the log level of TorchDynamo to + logging.DEBUG and AOT to logging.INFO + + TORCH_LOGS="-dynamo,+inductor" will set the log level of TorchDynamo to + logging.ERROR and TorchInductor to logging.DEBUG + + TORCH_LOGS="aot_graphs" will enable the aot_graphs artifact + + TORCH_LOGS="+dynamo,schedule" will enable set the log level of TorchDynamo + to logging.DEBUG and enable the schedule artifact + + TORCH_LOGS="+some.random.module,schedule" will set the log level of + some.random.module to logging.DEBUG and enable the schedule artifact + + TORCH_LOGS_FORMAT="%(levelname)s: %(message)s" or any provided format + string will set the output format + Valid keys are "levelname", "message", "pathname", "levelno", "lineno", + "filename" and "name". + + TORCH_LOGS_OUT=/tmp/output.txt will output the logs to /tmp/output.txt as + well. This is useful when the output is long. +""" # flake8: noqa: B950 + msg = f""" +TORCH_LOGS Info +{examples} + +{heading} +{setting_info} +""" + return msg + + +def _invalid_settings_err_msg(settings, verbose=False): + valid_settings = ( + ["all"] + + list(log_registry.log_alias_to_log_qnames.keys()) + + list(log_registry.artifact_names) + ) + valid_settings = ", ".join(sorted(valid_settings)) + msg = f""" +Invalid log settings: {settings}, must be a comma separated list of fully +qualified module names, registered log names or registered artifact names. +For more info on various settings, try TORCH_LOGS="help" +Valid settings: +{valid_settings} +""" + return msg + + +def process_env_var_string_for_windows(env_var_str: str) -> str: + """ + When we setup logging config as guide: https://docs.pycore.org/docs/stable/logging.html + Such as: + TORCH_LOGS="+schedule,+inductor,+output_code" + + On Linux, it shows as: + declare -x SSH_TTY="/dev/pts/0" + declare -x TERM="xterm" + declare -x TORCH_LOGS="+schedule,+inductor,+output_code" + declare -x USER="xu" + + On Windows, it shows as: + TORCHINDUCTOR_WINDOWS_TESTS=1 + TORCH_LOGS="+schedule,+inductor,+output_code" + UCRTVersion=10.0.22000.0 + + For Linux, it shows quotes by default, And Windows is not shows quotes. + Besides that, Windows would auto assemble quotes when env var processing. + On Linux, we will get variable: "+schedule,+inductor,+output_code" + On Windows, we will get variable: '"+schedule,+inductor,+output_code"' + + So, we need remove the outer quotes for Windows. + """ + _IS_WINDOWS = sys.platform == "win32" + + def remove_outer_quotes(s: str) -> str: + if len(s) >= 2 and ( + (s[0] == '"' and s[-1] == '"') or (s[0] == "'" and s[-1] == "'") + ): + return s[1:-1] + return s + + if _IS_WINDOWS: + env_var_str = remove_outer_quotes(env_var_str) + + return env_var_str + + +@functools.lru_cache +def _parse_log_settings(settings): + settings = process_env_var_string_for_windows(settings) + + if settings == "": + return {} + + if settings == "help": + raise ValueError(help_message(verbose=False)) + elif settings == "+help": + raise ValueError(help_message(verbose=True)) + if not _validate_settings(settings): + raise ValueError(_invalid_settings_err_msg(settings)) + + settings = re.sub(r"\s+", "", settings) + log_names = settings.split(",") + + def get_name_level_pair(name): + clean_name = name.replace(INCR_VERBOSITY_CHAR, "") + clean_name = clean_name.replace(DECR_VERBOSITY_CHAR, "") + + if name[0] == INCR_VERBOSITY_CHAR: + level = logging.DEBUG + elif name[0] == DECR_VERBOSITY_CHAR: + level = logging.ERROR + else: + level = logging.INFO + + return clean_name, level + + log_state = LogState() + + for name in log_names: + name, level = get_name_level_pair(name) + + if name == "all": + name = "torch" + + if log_registry.is_log(name): + assert level is not None + log_qnames = log_registry.log_alias_to_log_qnames[name] + log_state.enable_log(log_qnames, level) + elif log_registry.is_artifact(name): + log_state.enable_artifact(name) + elif _is_valid_module(name): + if not _has_registered_parent(name): + log_registry.register_log(name, name) + else: + log_registry.register_child_log(name) + log_state.enable_log(name, level) + else: + raise ValueError(_invalid_settings_err_msg(settings)) + + return log_state + + +def _is_valid_module(qname): + spec = importlib.util.find_spec(qname) + return spec is not None + + +def _update_log_state_from_env() -> None: + global log_state + log_setting = os.environ.get(LOG_ENV_VAR, None) + if log_setting is not None: + log_state = _parse_log_settings(log_setting) + + +def _has_registered_parent(log_qname) -> bool: + cur_log = logging.getLogger(log_qname) + + registered_log_qnames = log_registry.get_log_qnames() + + while cur_log.parent: + if cur_log.name in registered_log_qnames: + return True + cur_log = cur_log.parent + + return False + + +def make_module_path_relative(abs_path): + """ + Given an absolute filepath corresponding to a Python module which was + loaded via normal import mechanisms using sys.path, convert it into + a relative path relative to one of the Python search paths. + """ + + abs_path = pathlib.Path(abs_path).resolve() + + for path in sys.path: + try: + rel_path = abs_path.relative_to(path) + except ValueError: + continue + else: + return str(rel_path) + + return str(abs_path) + + +# apply custom formats to artifacts when necessary +class TorchLogsFormatter(logging.Formatter): + def __init__( + self, *, trace: bool = False, trace_id_filter: Optional[set[str]] = None + ) -> None: + super().__init__() + self._is_trace = trace + self._trace_id_filter = trace_id_filter + + def format(self, record): + artifact_name = getattr(logging.getLogger(record.name), "artifact_name", None) + if artifact_name is not None: + artifact_formatter = log_registry.artifact_log_formatters.get( + artifact_name, None + ) + if artifact_formatter is not None: + return artifact_formatter.format(record) + + record.message = record.getMessage() + record.asctime = self.formatTime(record, "%m%d %H:%M:%S") + + # exception handling - copied from logging.Formatter.format + s = record.message + if record.exc_info: + # Cache the traceback text to avoid converting it multiple times + # (it's constant anyway) + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + if record.exc_text: + if s[-1:] != "\n": + s = s + "\n" + s = s + record.exc_text + if record.stack_info: + if s[-1:] != "\n": + s = s + "\n" + s = s + self.formatStack(record.stack_info) + + record.rankprefix = "" + if not self._is_trace and dist.is_available() and dist.is_initialized(): + record.rankprefix = f"[rank{dist.get_rank()}]:" + + record.traceid = "" + if ( + not self._is_trace + and (trace_id := core._guards.CompileContext.current_trace_id()) + is not None + ): + record.traceid = f" [{trace_id}]" + + glog_level_to_abbr = { + "DEBUG": "V", # V is for VERBOSE in glog + "INFO": "I", + "WARNING": "W", + "ERROR": "E", + "CRITICAL": "C", + } + + shortlevel = glog_level_to_abbr.get(record.levelname, record.levelname) + + record.artifactprefix = "" + if artifact_name is not None: + record.artifactprefix = f" [__{artifact_name}]" + + filepath = make_module_path_relative(record.pathname) + + if ( + self._trace_id_filter + and record.traceid.strip() not in self._trace_id_filter + ): + return "" + + prefix = ( + f"{record.rankprefix}{shortlevel}{record.asctime}.{int(record.msecs * 1000):06d} {record.process} " + f"{filepath}:" + f"{record.lineno}]{record.traceid}{record.artifactprefix}" + ) + if self._is_trace: + assert s == "" + try: + r = f"{prefix} {json.dumps(record.metadata)}" + except TypeError: + log.warning("failing metadata: %r", record.metadata) + raise + if record.payload is not None: + r += "".join(f"\n\t{l}" for l in record.payload.split("\n")) + return r + else: + lines = s.split("\n") + return "\n".join(f"{prefix} {l}" for l in lines) + + +def _default_formatter(): + fmt = os.environ.get(LOG_FORMAT_ENV_VAR, None) + trace_id_filter = { + item.strip() + for item in os.environ.get(LOG_TRACE_ID_FILTER, "").split(",") + if item.strip() + } + if fmt is None: + return TorchLogsFormatter(trace_id_filter=trace_id_filter) + else: + if fmt in ("short", "basic"): + fmt = logging.BASIC_FORMAT + return logging.Formatter(fmt) + + +DEFAULT_FORMATTER = _default_formatter() + + +def _setup_handlers(create_handler_fn, log) -> None: + debug_handler = _track_handler(create_handler_fn()) + debug_handler.setFormatter(DEFAULT_FORMATTER) + debug_handler.setLevel(logging.DEBUG) + log.addHandler(debug_handler) + + +handlers = WeakSet() # type: ignore[var-annotated] + + +# mark handlers that we've created +# so we don't modify user handlers +def _track_handler(handler): + handlers.add(handler) + return handler + + +def _is_torch_handler(handler): + return handler in handlers + + +# clears all torch handlers on specified loggers +def _clear_handlers(log) -> None: + to_remove = [handler for handler in log.handlers if _is_torch_handler(handler)] + for handler in to_remove: + log.removeHandler(handler) + + +def _reset_logs() -> None: + # reset all registered logs + for log_qname in log_registry.get_log_qnames(): + log = logging.getLogger(log_qname) + log.setLevel(logging.WARNING) + log.propagate = False + _clear_handlers(log) + + # reset all artifact and child logs + for artifact_log_qname in itertools.chain( + log_registry.get_artifact_log_qnames(), log_registry.get_child_log_qnames() + ): + log = logging.getLogger(artifact_log_qname) + log.setLevel(logging.NOTSET) + log.propagate = True + + trace_log.propagate = False + _clear_handlers(trace_log) + + +def _get_log_state(): + return log_state + + +def _set_log_state(state) -> None: + global log_state + log_state = state + + +def _init_logs(log_file_name=None) -> None: + global GET_DTRACE_STRUCTURED + + _reset_logs() + _update_log_state_from_env() + + out = os.environ.get(LOG_OUT_ENV_VAR, None) + if out is not None: + log_file_name = out + + # First, reset all known (registered) loggers to NOTSET, so that they + # respect their parent log level + for log_qname in log_registry.get_log_qnames(): + # But not the top level torch level: this defaults to WARNING so + # that our log messages don't leak to the lower levels + if log_qname == "torch": + continue + log = logging.getLogger(log_qname) + log.setLevel(logging.NOTSET) + + # Now, for all loggers which the user requested to have non-standard + # logging behavior, modify their log levels + for log_qname, level in log_state.get_log_level_pairs(): + log = logging.getLogger(log_qname) + log.setLevel(level) + + # Finally, setup handlers for all registered loggers + for log_qname in log_registry.get_log_qnames(): + log = logging.getLogger(log_qname) + _setup_handlers( + logging.StreamHandler, + log, + ) + + if log_file_name is not None: + _setup_handlers( + lambda: logging.FileHandler(log_file_name), + log, + ) + + # configure artifact loggers, note: this must happen last + # since the levels of ancestor loggers are taken into account + for artifact_log_qname in log_registry.get_artifact_log_qnames(): + log = logging.getLogger(artifact_log_qname) + configure_artifact_log(log) + + # Setup handler for the special trace_log, with different default + # configuration + trace_dir_name = os.environ.get(TRACE_ENV_VAR, None) + + if dtrace_dir_name := os.environ.get(DTRACE_ENV_VAR, None): + GET_DTRACE_STRUCTURED = True + trace_dir_name = dtrace_dir_name + + # This handler may remove itself if trace_dir_name is None and we are not + # actually in an FB environment. This allows us to defer actually + # initializing it until we actually need to log anything. This is + # important because JK initializes a C++ singleton, which will pork our + # process if we subsequently fork. + global LOG_TRACE_HANDLER + if LOG_TRACE_HANDLER is None: + LOG_TRACE_HANDLER = LazyTraceHandler(trace_dir_name) + # This log is ALWAYS at debug level. We will additionally test if there + # are any handlers before deciding to actually call logging on this. Do + # not manually call + trace_log.setLevel(logging.DEBUG) + trace_log_handler = _track_handler(LOG_TRACE_HANDLER) + trace_log_handler.setFormatter(TorchLogsFormatter(trace=True)) + trace_log.addHandler(trace_log_handler) + + +class LazyTraceHandler(logging.StreamHandler): + """Like FileHandler, but the file is allocated lazily only upon the first log message""" + + def __init__(self, root_dir: Optional[str]) -> None: + # This is implemented in the same way that delay is implemented on + # FileHandler + self.root_dir = root_dir + logging.Handler.__init__(self) + self.stream = None + self._builtin_open = open + + # cloned from FileHandler in cpython + def close(self) -> None: + self.acquire() + try: + try: + if self.stream: + try: + self.flush() + finally: + stream = self.stream + self.stream = None + if hasattr(stream, "close"): + stream.close() + finally: + # Issue #19523: call unconditionally to + # prevent a handler leak when delay is set + # Also see Issue #42378: we also rely on + # self._closed being set to True there + logging.StreamHandler.close(self) + finally: + self.release() + + def emit(self, record) -> None: + if self.stream is None: + if self.root_dir is None: + TRACE_LOG_DIR = "/logs" + + import core.version as torch_version + + if ( + hasattr(torch_version, "git_version") + and os.getenv("MAST_HPC_JOB_NAME") is None + ): + log.info( + "LazyTraceHandler: disabled because not fbcode or conda on mast" + ) + elif not core._utils_internal.justknobs_check("pytorch/trace:enable"): + log.info( + "LazyTraceHandler: disabled because justknobs_check('pytorch/trace:enable') returned False" + ) + elif not os.path.exists(TRACE_LOG_DIR): + log.info( + "LazyTraceHandler: disabled because %s does not exist", + TRACE_LOG_DIR, + ) + elif not os.access(TRACE_LOG_DIR, os.W_OK): + log.info( + "LazyTraceHandler: disabled because %s is not writeable", + TRACE_LOG_DIR, + ) + else: + self.root_dir = TRACE_LOG_DIR + + if self.root_dir is not None: + os.makedirs(self.root_dir, exist_ok=True) + ranksuffix = "" + if dist.is_available() and dist.is_initialized(): + ranksuffix = f"rank_{dist.get_rank()}_" + self.stream = tempfile.NamedTemporaryFile( + mode="w+", + suffix=".log", + prefix=f"dedicated_log_torch_trace_{ranksuffix}", + dir=self.root_dir, + delete=False, + ) + log.info("LazyTraceHandler: logging to %s", self.stream.name) + else: + # We go poof, remove and no-op + trace_log.removeHandler(self) + return + if self.stream: + super().emit(record) + + +@functools.cache +def warning_once(logger_obj, *args, **kwargs) -> None: + """ + This function is similar to `logger.warning()`, but will emit the warning with the same message only once + Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the cache. + The assumption here is that all warning messages are unique across the code. If they aren't then need to switch to + another type of cache that includes the caller frame information in the hashing function. + """ + logger_obj.warning(*args, **kwargs) + + +def safe_grad_filter(message, category, filename, lineno, file=None, line=None) -> bool: + return "The .grad attribute of a Tensor" not in str(message) + + +def user_warning_filter( + message, category, filename, lineno, file=None, line=None +) -> bool: + return not category == UserWarning + + +@contextlib.contextmanager +def hide_warnings(filter_fn=lambda *args, **kwargs: True): + """ + A context manager that temporarily suppresses warnings, + using public API: https://docs.python.org/3/library/warnings.html#warnings.showwarning. + + Useful to hide warnings without mutating warnings module state, see: + https://github.com/pytorch/pytorch/issues/128427#issuecomment-2161496162. + + NOTE: Warnings issued under this context will still be cached in the __warningregistry__ + and count towards the once/default rule. So you should NEVER use this on a user-land function. + + Filter must implement the showwarning API: + def filter_fn(message, category, filename, lineno, file=None, line=None) -> bool: + return True # show this warning entry + """ + prior = warnings.showwarning + + def _showwarning(*args, **kwargs): + if filter_fn(*args, **kwargs): + prior(*args, **kwargs) + + try: + warnings.showwarning = _showwarning + yield + finally: + warnings.showwarning = prior + + +class LazyString(Generic[_P]): + def __init__( + self, func: Callable[_P, str], *args: _P.args, **kwargs: _P.kwargs + ) -> None: + self.func = func + self.args = args + self.kwargs = kwargs + + def __str__(self) -> str: + return self.func(*self.args, **self.kwargs) + + +# Logs the time it takes to do structured logging by frame/compile id +# key is always {frame_id}_{frame_compile_id} +structured_logging_overhead: dict[str, float] = defaultdict(float) + + +def add_structured_logging_overhead(time_spent: float) -> None: + global structured_logging_overhead + key = None + if (trace_id := core._guards.CompileContext.current_trace_id()) is not None: + frame_id = trace_id.compile_id.frame_id + frame_compile_id = trace_id.compile_id.frame_compile_id + # Why not trace_id.attempt, like structured logging? + # We aggregate across all attempts because + # a compilation metric is logged per successful attempt + key = f"{frame_id}_{frame_compile_id}" + # TODO: deal with structured logging that occurs outside of specific compile ids + # It's hard to figure out where we would log that if we want it in compilation metrics + # itself. + if key is not None: + key = str(key) + structured_logging_overhead[key] += time_spent + + +def get_structured_logging_overhead() -> Optional[float]: + key = None + if (trace_id := core._guards.CompileContext.current_trace_id()) is not None: + frame_id = trace_id.compile_id.frame_id + frame_compile_id = trace_id.compile_id.frame_compile_id + key = f"{frame_id}_{frame_compile_id}" + if key is not None: + return structured_logging_overhead.get(key) + else: + return None + + +def trace_structured_artifact( + name: str, # this will go in metadata + encoding: str, + payload_fn: Callable[[], Optional[Union[str, object]]] = lambda: None, + compile_id: Optional[CompileId] = None, +) -> None: + trace_structured( + "artifact", + metadata_fn=lambda: { + "name": name, + "encoding": encoding, + }, + payload_fn=payload_fn, + compile_id=compile_id, + ) + + +def trace_structured( + name: str, + # NB: metadata expected to be dict so adding more info is forward compatible + # Tuple[str, int] is a special case for string interning + metadata_fn: Callable[[], Union[dict[str, Any], tuple[str, int]]] = dict, + *, + payload_fn: Callable[[], Optional[Union[str, object]]] = lambda: None, + suppress_context: bool = False, + expect_trace_id: bool = True, # Whether or not we expect to have a current trace id + record_logging_overhead: bool = True, # Whether or not to record the time spent on structured logging + compile_id: Optional[CompileId] = None, # Optional if unavailable in the trace +) -> None: + """ + metadata is an arbitrary JSON compatible struct, but it's expected to not be + too long (e.g., less than 1MB) + + payload is an arbitrary string, which can be arbitrarily long (but expected to have + newlines so no lines are too long) + """ + assert name not in [ + "rank", + "compiled_autograd_id", + "frame_id", + "frame_compile_id", + "attempt", + "severity", + "timestamp", + "pathname", + "thread", + ] + assert callable(metadata_fn), ( + f"metadata_fn should be callable, but got {type(metadata_fn)}" + ) + assert callable(payload_fn), ( + f"payload_fn should be callable, but got {type(payload_fn)}" + ) + # trace_log never propagates and is ALWAYS DEBUG, so also check that there + # are handlers instead of checking the log level + if trace_log.handlers: + start_time = time.time_ns() + record: dict[str, object] = {} + record[name] = metadata_fn() + if not suppress_context: + # TODO: Actually, the rank probably should just be emitted once at + # the top, and not repeatedly spammed in all the logs, since it + # never changes and we assume no interleaving + if dist.is_available() and dist.is_initialized(): + record["rank"] = dist.get_rank() + + trace_id = core._guards.CompileContext.current_trace_id() + if expect_trace_id and trace_id is None and compile_id is None: + # Record the stack of the log call to better diagnose why we + # don't have a frame id for it + record["stack"] = core._logging.structured.from_traceback( + CapturedTraceback.extract(skip=1).summary() + ) + else: + cid = trace_id.compile_id if trace_id else compile_id + if cid is not None: + if cid.compiled_autograd_id is not None: + record["compiled_autograd_id"] = cid.compiled_autograd_id + if cid.frame_id is not None: + record["frame_id"] = cid.frame_id + if cid.frame_compile_id is not None: + record["frame_compile_id"] = cid.frame_compile_id + if trace_id: + record["attempt"] = trace_id.attempt + + payload = payload_fn() + if payload is not None: + if not isinstance(payload, str): + if isinstance(payload, list): + # special case to look better + payload = "[\n" + ",\n".join(json.dumps(i) for i in payload) + "\n]" + else: + + def json_default(obj): + # Sets aren't json serializable + if isinstance(obj, set): + return list(obj) + raise TypeError( + f"Object of type {type(obj)} is not JSON serializable" + ) + + # force newlines so we are unlikely to overflow line limit + payload = json.dumps(payload, default=json_default, indent=0) + h = hashlib.md5(usedforsecurity=False) + h.update(payload.encode("utf-8")) + record["has_payload"] = h.hexdigest() + trace_log.debug( + "", extra={"metadata": record, "payload": payload}, stacklevel=2 + ) + log_trace_structured_event(name, record) + + if record_logging_overhead: + # Convert to seconds from nanoseconds, add it to the frame compile total + structured_logging_overhead_s = (time.time_ns() - start_time) / 1e9 + add_structured_logging_overhead(structured_logging_overhead_s) + + +def dtrace_structured( + name: str, + # NB: metadata expected to be dict so adding more info is forward compatible + # Tuple[str, int] is a special case for string interning + metadata_fn: Callable[[], Union[dict[str, Any], tuple[str, int]]] = dict, + *, + payload_fn: Callable[[], Optional[Union[str, object]]] = lambda: None, + suppress_context: bool = False, + expect_trace_id: bool = False, # Whether or not we expect to have a current trace id + record_logging_overhead: bool = True, # Whether or not to record the time spent on structured logging +) -> None: + """ + For logging more detailed information used for debugging. This may result in + the program becoming slow. + """ + if GET_DTRACE_STRUCTURED: + trace_structured( + name, + metadata_fn, + payload_fn=payload_fn, + suppress_context=suppress_context, + expect_trace_id=expect_trace_id, + record_logging_overhead=record_logging_overhead, + ) + + +import mindnlp.core._guards +import mindnlp.core._utils_internal +import mindnlp.core.distributed as dist \ No newline at end of file diff --git a/mindnlp/core/_logging/_registrations.py b/mindnlp/core/_logging/_registrations.py new file mode 100644 index 000000000..a5b77158c --- /dev/null +++ b/mindnlp/core/_logging/_registrations.py @@ -0,0 +1,249 @@ +# flake8: noqa: B950 +from ._internal import register_artifact, register_log + + +DYNAMIC = [ + "torch.fx.experimental.symbolic_shapes", + "torch.fx.experimental.sym_node", + "torch.fx.experimental.recording", +] +DISTRIBUTED = [ + "torch.distributed", + "torch._dynamo.backends.distributed", + "torch.nn.parallel.distributed", +] + +register_log( + "async_compile", + [ + "torch._inductor.async_compile", + "torch._inductor.compile_worker.tracked_process_pool", + ], +) +register_log( + "cache", ("torch._inductor.remote_cache", "torch._inductor.fb.remote_cache") +) +register_log("dynamo", ["torch._dynamo", *DYNAMIC]) +register_log("fake_tensor", ["torch._subclasses.fake_tensor"]) +register_log("aot", ["torch._functorch.aot_autograd", "torch._functorch._aot_autograd"]) +register_log("autograd", "torch.autograd") +register_log("inductor", ["torch._inductor", "torch._inductor.cudagraph_trees"]) + +register_artifact( + "cudagraphs", + "Logs information from wrapping inductor generated code with cudagraphs.", +) + +register_log("dynamic", DYNAMIC) +register_log("torch", "torch") +register_log("distributed", DISTRIBUTED) +register_log( + "c10d", ["torch.distributed.distributed_c10d", "torch.distributed.rendezvous"] +) +register_log( + "ddp", ["torch.nn.parallel.distributed", "torch._dynamo.backends.distributed"] +) +register_log("pp", ["torch.distributed.pipelining"]) +register_log("fsdp", ["torch.distributed.fsdp", "torch.distributed._composable.fsdp"]) +register_log("dtensor", ["torch.distributed._tensor", "torch.distributed.tensor"]) +register_log("onnx", "torch.onnx") +register_log( + "export", + [ + "torch._dynamo", + "torch.export", + "torch.export.dynamic_shapes", + *DYNAMIC, + "torch._export.converter", + "torch._export.non_strict_utils", + "torch._export.serde.serialize", + "torch.fx.experimental.proxy_tensor", + ], +) + +register_artifact( + "guards", + "This prints the guards for every compiled Dynamo frame. It does not tell you where the guards come from.", + visible=True, +) +register_artifact("verbose_guards", "", off_by_default=True) +register_artifact( + "bytecode", + "Prints the original and modified bytecode from Dynamo. Mostly useful if you're debugging our bytecode generation in Dynamo.", + off_by_default=True, +) +register_artifact( + "graph", + "Prints the dynamo traced graph (prior to AOTDispatch) in a table. If you prefer python code use `graph_code` instead. ", +) +register_artifact("graph_code", "Like `graph`, but gives you the Python code instead.") +register_artifact( + "graph_code_verbose", + "Verbose FX pass logs, e.g. from tensorify_python_scalars and runtime_assert.", +) +register_artifact( + "graph_sizes", "Prints the sizes of all FX nodes in the dynamo graph." +) +register_artifact( + "trace_source", + "As we execute bytecode, prints the file name / line number we are processing and the actual source code. Useful with `bytecode`", +) +register_artifact( + "trace_call", + "Like trace_source, but it will give you the per-expression blow-by-blow if your Python is recent enough.", +) +register_artifact( + "trace_bytecode", + "As we trace bytecode, prints the instruction and the current stack.", +) +register_artifact( + "aot_graphs", + "Prints the FX forward and backward graph generated by AOTDispatch, after partitioning. Useful to understand what's being given to Inductor", + visible=True, +) +register_artifact( + "aot_joint_graph", + "Print FX joint graph from AOTAutograd, prior to partitioning. Useful for debugging partitioning", +) +register_artifact( + "aot_graphs_effects", + "Prints the FX forward and backward graph generated by AOTDispatch, useful for debugging effects processing.", + visible=True, +) +register_artifact( + "pre_grad_graphs", + "Prints the FX graph before inductor pre grad passes. Useful to understand what's being given to Inductor before grad passes", +) +register_artifact( + "post_grad_graphs", + "Prints the FX graph generated by post grad passes. Useful to understand what's being given to Inductor after post grad passes", +) +register_artifact( + "ir_pre_fusion", + "Prints the IR before inductor fusion passes.", + off_by_default=True, +) +register_artifact( + "ir_post_fusion", + "Prints the IR after inductor fusion passes.", + off_by_default=True, +) +register_artifact( + "compiled_autograd", + "Prints various logs in compiled_autograd, including but not limited to the graphs. Useful for debugging compiled_autograd.", + visible=True, +) +register_artifact( + "compiled_autograd_verbose", + "Will affect performance. Prints compiled_autograd logs with C++ info e.g. autograd node -> fx node mapping", + off_by_default=True, +) +register_artifact( + "ddp_graphs", + "Only relevant for compiling DDP. DDP splits into multiple graphs to trigger comms early. This will print each individual graph here.", +) +register_artifact( + "recompiles", + "Prints the reason why we recompiled a graph. Very, very useful.", + visible=True, +) +register_artifact( + "recompiles_verbose", + "Prints all guard checks that fail during a recompilation. " + "At runtime, Dynamo will stop at the first failed check for each failing guard. " + "So not all logged failing checks are actually ran by Dynamo.", + visible=True, + off_by_default=True, +) +register_artifact( + "graph_breaks", + "Prints whenever Dynamo decides that it needs to graph break (i.e. create a new graph). Useful for debugging why torch.compile has poor performance", + visible=True, +) +register_artifact( + "not_implemented", + "Prints log messages whenever we return NotImplemented in a multi-dispatch, letting you trace through each object we attempted to dispatch to", +) +register_artifact( + "output_code", + "Prints the code that Inductor generates (either Triton or C++)", + off_by_default=True, + visible=True, +) +register_artifact( + "kernel_code", + "Prints the code that Inductor generates (on a per-kernel basis)", + off_by_default=True, + visible=True, +) +register_artifact( + "schedule", + "Inductor scheduler information. Useful if working on Inductor fusion algo", + off_by_default=True, +) +register_artifact("perf_hints", "", off_by_default=True) +register_artifact("onnx_diagnostics", "", off_by_default=True) +register_artifact("compute_dependencies", "", off_by_default=True) +register_artifact( + "fusion", + "Detailed Inductor fusion decisions. More detailed than 'schedule'", + off_by_default=True, +) +register_artifact( + "loop_ordering", + "Logs related to loop ordering", + off_by_default=True, +) +register_artifact( + "loop_tiling", + "Logs related to loop ordering", + off_by_default=True, +) + +register_artifact( + "overlap", + "Detailed Inductor compute/comm overlap decisions", + off_by_default=True, +) +register_artifact( + "sym_node", + "Logs extra info for various SymNode operations", + off_by_default=True, +) +register_artifact( + "trace_shape_events", + "Logs traces for every ShapeEnv operation that we record for replay", + off_by_default=True, +) +register_artifact( + "cudagraph_static_inputs", + "Logs static inputs handling in dynamo, AOT, and cudagraphs", + off_by_default=True, +) +register_artifact( + "benchmarking", + "Detailed Inductor benchmarking information.", + off_by_default=True, +) +register_artifact( + "autotuning", + "Autotuning choice logs, such as kernel source, perf, and tuning parameters.", + off_by_default=True, +) +register_artifact( + "graph_region_expansion", + "Logs detailed steps of the duplicate graph region tracker expansion algorithm", + off_by_default=True, +) + +register_artifact( + "inductor_metrics", + "Logs Inductor metrics, such as num_bytes, nodes_num_elem, node_runtimes", + off_by_default=True, +) +register_artifact( + "hierarchical_compile", + "Logs debug info for hierarchical compilation", + off_by_default=True, +) +register_artifact("custom_format_test_artifact", "Testing only", log_format="") \ No newline at end of file diff --git a/mindnlp/core/_logging/scribe.py b/mindnlp/core/_logging/scribe.py new file mode 100644 index 000000000..0e1a44723 --- /dev/null +++ b/mindnlp/core/_logging/scribe.py @@ -0,0 +1,63 @@ +from typing import Callable, Union +from typing_extensions import TypeAlias + + +try: + from fbscribelogger import ( # type: ignore[import-untyped, import-not-found] + make_scribe_logger, + ) +except ImportError: + TAtom: TypeAlias = Union[int, float, bool, str] + TField: TypeAlias = Union[TAtom, list[TAtom]] + TLazyField: TypeAlias = Union[TField, Callable[[], TField]] + + def make_scribe_logger(name: str, thrift_src: str) -> Callable[..., None]: + def inner(**kwargs: TLazyField) -> None: + pass + + return inner + + +open_source_signpost = make_scribe_logger( + "TorchOpenSourceSignpost", + """ +struct TorchOpenSourceSignpostLogEntry { + + # The commit SHA that triggered the workflow, e.g., 02a6b1d30f338206a71d0b75bfa09d85fac0028a. Derived from GITHUB_SHA. + 4: optional string commit_sha; + + # Commit date (not author date) of the commit in commit_sha as timestamp, e.g., 1724208105. Increasing if merge bot is used, though not monotonic; duplicates occur when stack is landed. + 5: optional i64 commit_date; + + # The fully-formed ref of the branch or tag that triggered the workflow run, e.g., refs/pull/133891/merge or refs/heads/main. Derived from GITHUB_REF. + 6: optional string github_ref; + + # Indicates if branch protections or rulesets are configured for the ref that triggered the workflow run. Derived from GITHUB_REF_PROTECTED. + 7: optional bool github_ref_protected; + + # A unique number for each attempt of a particular workflow run in a repository, e.g., 1. Derived from GITHUB_RUN_ATTEMPT. + 8: optional string github_run_attempt; + + # A unique number for each workflow run within a repository, e.g., 19471190684. Derived from GITHUB_RUN_ID. + 9: optional string github_run_id; + + # A unique number for each run of a particular workflow in a repository, e.g., 238742. Derived from GITHUB_RUN_NUMBER. + 10: optional string github_run_number_str; + + # The name of the current job. Derived from JOB_NAME, e.g., linux-jammy-py3.8-gcc11 / test (default, 3, 4, linux.2xlarge). + 11: optional string job_name; + + # The GitHub user who triggered the job. Derived from GITHUB_TRIGGERING_ACTOR. + 12: optional string github_triggering_actor; + 13: optional string name; # Event name + 14: optional string parameters; # Parameters (JSON data) + 16: optional string subsystem; # Subsystem the event is associated with + + # The unit timestamp in second for the Scuba Time Column override + 17: optional i64 time; + + # The weight of the record according to current sampling rate + 18: optional i64 weight; +} +""", # noqa: B950 +) \ No newline at end of file diff --git a/mindnlp/core/_logging/structured.py b/mindnlp/core/_logging/structured.py new file mode 100644 index 000000000..889b9919f --- /dev/null +++ b/mindnlp/core/_logging/structured.py @@ -0,0 +1,109 @@ +""" +Utilities for converting data types into structured JSON for dumping. +""" + +import inspect +import os +import traceback +from collections.abc import Sequence +from typing import Any, Optional + +import mindnlp.core._logging._internal + + +INTERN_TABLE: dict[str, int] = {} + + +DUMPED_FILES: set[str] = set() + + +def intern_string(s: Optional[str]) -> int: + if s is None: + return -1 + + r = INTERN_TABLE.get(s, None) + if r is None: + r = len(INTERN_TABLE) + INTERN_TABLE[s] = r + core._logging._internal.trace_structured( + "str", lambda: (s, r), suppress_context=True + ) + return r + + +def dump_file(filename: str) -> None: + if "eval_with_key" not in filename: + return + if filename in DUMPED_FILES: + return + DUMPED_FILES.add(filename) + from core.fx.graph_module import _loader + + core._logging._internal.trace_structured( + "dump_file", + metadata_fn=lambda: { + "name": filename, + }, + payload_fn=lambda: _loader.get_source(filename), + ) + + +def from_traceback(tb: Sequence[traceback.FrameSummary]) -> list[dict[str, Any]]: + # dict naming convention here coincides with + # python/combined_traceback.cpp + r = [ + { + "line": frame.lineno, + "name": frame.name, + "filename": intern_string(frame.filename), + "loc": frame.line, + } + for frame in tb + ] + return r + + +def get_user_stack(num_frames: int) -> list[dict[str, Any]]: + from core._guards import TracingContext + from core.utils._traceback import CapturedTraceback + + user_tb = TracingContext.extract_stack() + if user_tb: + return from_traceback(user_tb[-1 * num_frames :]) + + tb = CapturedTraceback.extract().summary() + + # Filter out frames that are within the torch/ codebase + torch_filepath = os.path.dirname(inspect.getfile(torch)) + os.path.sep + for i, frame in enumerate(reversed(tb)): + if torch_filepath not in frame.filename: + # Only display `num_frames` frames in the traceback + filtered_tb = tb[len(tb) - i - num_frames : len(tb) - i] + return from_traceback(filtered_tb) + + return from_traceback(tb[-1 * num_frames :]) + + +def get_framework_stack( + num_frames: int = 25, cpp: bool = False +) -> list[dict[str, Any]]: + """ + Returns the traceback for the user stack and the framework stack + """ + from core.fx.experimental.symbolic_shapes import uninteresting_files + from core.utils._traceback import CapturedTraceback + + tb = CapturedTraceback.extract(cpp=cpp).summary() + tb = [ + frame + for frame in tb + if ( + ( + frame.filename.endswith(".py") + and frame.filename not in uninteresting_files() + ) + or ("at::" in frame.name or "torch::" in frame.name) + ) + ] + + return from_traceback(tb[-1 * num_frames :]) \ No newline at end of file diff --git a/mindnlp/core/_prims/numpy.py b/mindnlp/core/_prims/numpy.py index e88e688a2..0599c42d6 100644 --- a/mindnlp/core/_prims/numpy.py +++ b/mindnlp/core/_prims/numpy.py @@ -518,3 +518,14 @@ def randperm_ext(n, seed, offset, dtype): __all__.append('randperm_ext') +def embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq): + out = np.take(weight.numpy(), input.numpy(), axis=0) + return core.Tensor.from_numpy(out) + +__all__.append('embedding') + +def randn(size, seed, offset, dtype): + out = np.random.randn(*size).astype(core.dtype2np[dtype]) + return core.Tensor.from_numpy(out) + +__all__.append('randn') diff --git a/mindnlp/core/_tensor.py b/mindnlp/core/_tensor.py index b90a7504c..cca179812 100644 --- a/mindnlp/core/_tensor.py +++ b/mindnlp/core/_tensor.py @@ -18,7 +18,6 @@ class StubTensor: pass from . import ops, _dtype from ._bind import get_device_in_context, device_, get_default_dtype -from .storage import UntypedStorage from ._utils import _rebuild_tensor_v2 from ._C.size import Size from .configs import DEVICE_TARGET, CPU_USE_NUMPY_OP @@ -93,6 +92,8 @@ def __init__(self, *args, **kwargs): Tensor.__init__ = __init__ origin_setitem = Tensor.__setitem__ +Tensor._device = device_('cpu') + def tensor(data, *, dtype=None, device=None, requires_grad=False): if isinstance(data, Tensor): UserWarning("To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than core.tensor(sourceTensor).") @@ -2065,7 +2066,7 @@ def std(self, dim=None, *, correction=1, keepdim=False): # Tensor.untyped_storage def untyped_storage(self): - return UntypedStorage(self) + return core.UntypedStorage(self) # Tensor.storage_offset @@ -2508,6 +2509,10 @@ def backward(self): def log_softmax(self, dim): return ops.log_softmax(self, dim) + @property + def is_nested(self): + return False + def enable_mindspore_patch(): fn_keys = list(TensorPlaceHolder.__dict__) fn_keys.remove('__doc__') diff --git a/mindnlp/core/_utils_internal.py b/mindnlp/core/_utils_internal.py new file mode 100644 index 000000000..d11bd7329 --- /dev/null +++ b/mindnlp/core/_utils_internal.py @@ -0,0 +1,358 @@ +# mypy: allow-untyped-defs +import functools +import logging +import os +import sys +import tempfile +import typing_extensions +from typing import Any, Callable, Optional, TypeVar +from typing_extensions import ParamSpec + +from mindnlp import core + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +log = logging.getLogger(__name__) + +if os.environ.get("TORCH_COMPILE_STROBELIGHT", False): + import shutil + + if not shutil.which("strobeclient"): + log.info( + "TORCH_COMPILE_STROBELIGHT is true, but seems like you are not on a FB machine." + ) + else: + log.info("Strobelight profiler is enabled via environment variable") + StrobelightCompileTimeProfiler.enable() + +# this arbitrary-looking assortment of functionality is provided here +# to have a central place for overridable behavior. The motivating +# use is the FB build environment, where this source file is replaced +# by an equivalent. + +if os.path.basename(os.path.dirname(__file__)) == "shared": + torch_parent = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) +else: + torch_parent = os.path.dirname(os.path.dirname(__file__)) + + +def get_file_path(*path_components: str) -> str: + return os.path.join(torch_parent, *path_components) + + +def get_file_path_2(*path_components: str) -> str: + return os.path.join(*path_components) + + +def get_writable_path(path: str) -> str: + if os.access(path, os.W_OK): + return path + return tempfile.mkdtemp(suffix=os.path.basename(path)) + + +def prepare_multiprocessing_environment(path: str) -> None: + pass + + +def resolve_library_path(path: str) -> str: + return os.path.realpath(path) + + +def throw_abstract_impl_not_imported_error(opname, module, context): + if module in sys.modules: + raise NotImplementedError( + f"{opname}: We could not find the fake impl for this operator. " + ) + else: + raise NotImplementedError( + f"{opname}: We could not find the fake impl for this operator. " + f"The operator specified that you may need to import the '{module}' " + f"Python module to load the fake impl. {context}" + ) + + +# NB! This treats "skip" kwarg specially!! +def compile_time_strobelight_meta( + phase_name: str, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: + def compile_time_strobelight_meta_inner( + function: Callable[_P, _T], + ) -> Callable[_P, _T]: + @functools.wraps(function) + def wrapper_function(*args: _P.args, **kwargs: _P.kwargs) -> _T: + if "skip" in kwargs and isinstance(skip := kwargs["skip"], int): + kwargs["skip"] = skip + 1 + + # This is not needed but we have it here to avoid having profile_compile_time + # in stack traces when profiling is not enabled. + if not StrobelightCompileTimeProfiler.enabled: + return function(*args, **kwargs) + + return StrobelightCompileTimeProfiler.profile_compile_time( + function, phase_name, *args, **kwargs + ) + + return wrapper_function + + return compile_time_strobelight_meta_inner + + +# Meta only, see +# https://www.internalfb.com/intern/wiki/ML_Workflow_Observability/User_Guides/Adding_instrumentation_to_your_code/ +# +# This will cause an event to get logged to Scuba via the signposts API. You +# can view samples on the API at https://fburl.com/scuba/workflow_signpost/zh9wmpqs +# we log to subsystem "torch", and the category and name you provide here. +# Each of the arguments translate into a Scuba column. We're still figuring +# out local conventions in PyTorch, but category should be something like +# "dynamo" or "inductor", and name should be a specific string describing what +# kind of event happened. +# +# Killswitch is at +# https://www.internalfb.com/intern/justknobs/?name=pytorch%2Fsignpost#event +def signpost_event(category: str, name: str, parameters: dict[str, Any]): + log.info("%s %s: %r", category, name, parameters) + + +def add_mlhub_insight(category: str, insight: str, insight_description: str): + pass + + +def log_compilation_event(metrics): + log.info("%s", metrics) + + +def upload_graph(graph): + pass + + +def set_pytorch_distributed_envs_from_justknobs(): + pass + + +def log_export_usage(**kwargs): + pass + + +def log_draft_export_usage(**kwargs): + pass + + +def log_trace_structured_event(*args, **kwargs) -> None: + pass + + +def log_cache_bypass(*args, **kwargs) -> None: + pass + + +def log_torchscript_usage(api: str, **kwargs): + _ = api + return + + +def check_if_torch_exportable(): + return False + + +def export_training_ir_rollout_check() -> bool: + return True + + +def full_aoti_runtime_assert() -> bool: + return True + + +def log_torch_jit_trace_exportability( + api: str, + type_of_export: str, + export_outcome: str, + result: str, +): + _, _, _, _ = api, type_of_export, export_outcome, result + return + + +def justknobs_check(name: str, default: bool = True) -> bool: + """ + This function can be used to killswitch functionality in FB prod, + where you can toggle this value to False in JK without having to + do a code push. In OSS, we always have everything turned on all + the time, because downstream users can simply choose to not update + PyTorch. (If more fine-grained enable/disable is needed, we could + potentially have a map we lookup name in to toggle behavior. But + the point is that it's all tied to source code in OSS, since there's + no live server to query.) + + This is the bare minimum functionality I needed to do some killswitches. + We have a more detailed plan at + https://docs.google.com/document/d/1Ukerh9_42SeGh89J-tGtecpHBPwGlkQ043pddkKb3PU/edit + In particular, in some circumstances it may be necessary to read in + a knob once at process start, and then use it consistently for the + rest of the process. Future functionality will codify these patterns + into a better high level API. + + WARNING: Do NOT call this function at module import time, JK is not + fork safe and you will break anyone who forks the process and then + hits JK again. + """ + return default + + +def justknobs_getval_int(name: str) -> int: + """ + Read warning on justknobs_check + """ + return 0 + + +def is_fb_unit_test() -> bool: + return False + + +@functools.cache +def max_clock_rate(): + """ + unit: MHz + """ + if not core.version.hip: + from triton.testing import nvsmi + + return nvsmi(["clocks.max.sm"])[0] + else: + # Manually set max-clock speeds on ROCm until equivalent nvmsi + # functionality in triton.testing or via pyamdsmi enablement. Required + # for test_snode_runtime unit tests. + gcn_arch = str(core.cuda.get_device_properties(0).gcnArchName.split(":", 1)[0]) + if "gfx94" in gcn_arch: + return 1700 + elif "gfx90a" in gcn_arch: + return 1700 + elif "gfx908" in gcn_arch: + return 1502 + elif "gfx12" in gcn_arch: + return 1700 + elif "gfx11" in gcn_arch: + return 1700 + elif "gfx103" in gcn_arch: + return 1967 + elif "gfx101" in gcn_arch: + return 1144 + elif "gfx95" in gcn_arch: + return 1700 # TODO: placeholder, get actual value + else: + return 1100 + + +def get_mast_job_name_version() -> Optional[tuple[str, int]]: + return None + + +TEST_MASTER_ADDR = "127.0.0.1" +TEST_MASTER_PORT = 29500 +# USE_GLOBAL_DEPS controls whether __init__.py tries to load +# libtorch_global_deps, see Note [Global dependencies] +USE_GLOBAL_DEPS = True +# USE_RTLD_GLOBAL_WITH_LIBTORCH controls whether __init__.py tries to load +# _C.so with RTLD_GLOBAL during the call to dlopen. +USE_RTLD_GLOBAL_WITH_LIBTORCH = False +# If an op was defined in C++ and extended from Python using the +# core.library.register_fake, returns if we require that there be a +# m.set_python_module("mylib.ops") call from C++ that associates +# the C++ op with a python module. +REQUIRES_SET_PYTHON_MODULE = False + + +def maybe_upload_prof_stats_to_manifold(profile_path: str) -> Optional[str]: + print("Uploading profile stats (fb-only otherwise no-op)") + return None + + +def log_chromium_event_internal( + event: dict[str, Any], + stack: list[str], + logger_uuid: str, + start_time_ns: int, +): + return None + + +def record_chromium_event_internal( + event: dict[str, Any], +): + return None + + +def profiler_allow_cudagraph_cupti_lazy_reinit_cuda12(): + return True + + +def deprecated(): + """ + When we deprecate a function that might still be in use, we make it internal + by adding a leading underscore. This decorator is used with a private function, + and creates a public alias without the leading underscore, but has a deprecation + warning. This tells users "THIS FUNCTION IS DEPRECATED, please use something else" + without breaking them, however, if they still really really want to use the + deprecated function without the warning, they can do so by using the internal + function name. + """ + + def decorator(func: Callable[_P, _T]) -> Callable[_P, _T]: + # Validate naming convention – single leading underscore, not dunder + if not (func.__name__.startswith("_")): + raise ValueError( + "@deprecate must decorate a function whose name " + "starts with a single leading underscore (e.g. '_foo') as the api should be considered internal for deprecation." + ) + + public_name = func.__name__[1:] # drop exactly one leading underscore + module = sys.modules[func.__module__] + + # Don't clobber an existing symbol accidentally. + if hasattr(module, public_name): + raise RuntimeError( + f"Cannot create alias '{public_name}' -> symbol already exists in {module.__name__}. \ + Please rename it or consult a pytorch developer on what to do" + ) + + warning_msg = f"{func.__name__[1:]} is DEPRECATED, please consider using an alternative API(s). " + + # public deprecated alias + alias = typing_extensions.deprecated( + warning_msg, category=UserWarning, stacklevel=1 + )(func) + + alias.__name__ = public_name + + # Adjust qualname if nested inside a class or another function + if "." in func.__qualname__: + alias.__qualname__ = func.__qualname__.rsplit(".", 1)[0] + "." + public_name + else: + alias.__qualname__ = public_name + + setattr(module, public_name, alias) + + return func + + return decorator + + +def get_default_numa_options(): + """ + When using elastic agent, if no numa options are provided, we will use these + as the default. + + For external use cases, we return None, i.e. no numa binding. If you would like + to use torch's automatic numa binding capabilities, you should provide + NumaOptions to your launch config directly or use the numa binding option + available in torchrun. + + Must return None or NumaOptions, but not specifying to avoid circular import. + """ + return None + + +def log_triton_builds(fail: Optional[str]): + pass \ No newline at end of file diff --git a/mindnlp/core/amp/grad_scaler.py b/mindnlp/core/amp/grad_scaler.py index 7a87a3119..6d86ff998 100644 --- a/mindnlp/core/amp/grad_scaler.py +++ b/mindnlp/core/amp/grad_scaler.py @@ -21,14 +21,10 @@ def non_finite_check(inputs): status = core.tensor(np.array([0] * 8), dtype=core.int32, device='npu') status = core.depend(status, inputs) found_inf = core.npu_get_float_status_v2(status) - print('found_inf', found_inf) status = core.depend(status, found_inf) clear_status = core.npu_clear_float_status_v2(status) found_inf = core.depend(found_inf, clear_status) - print('found_inf', found_inf) found_inf = core.not_equal(found_inf, 0) - print('found_inf', found_inf) - print('after clear', core.npu_get_float_status_v2(status)) return found_inf.sum() found_inf = core.all_finite(inputs) # pylint: disable=invalid-unary-operand-type @@ -307,9 +303,7 @@ def _unscale_grads_( # per_device_inv_scale.get(device), # ) found_inf = per_device_found_inf.get(device) - print('found_inf before', found_inf) found_inf.copy_(non_finite_check(grads).to(found_inf.dtype)) - print('found_inf after', found_inf) for grad in grads: grad *= per_device_inv_scale.get(device) @@ -379,7 +373,6 @@ def _maybe_opt_step( **kwargs: Any, ) -> Optional[float]: retval: Optional[float] = None - print(sum(v.item() for v in optimizer_state["found_inf_per_device"].values())) if not sum(v.item() for v in optimizer_state["found_inf_per_device"].values()): retval = optimizer.step(*args, **kwargs) return retval @@ -426,8 +419,6 @@ def step( ) retval: Optional[float] = None - print('grad scaler') - print(getattr(optimizer, "_step_supports_amp_scaling", False)) if getattr(optimizer, "_step_supports_amp_scaling", False): # This optimizer has customized scale-handling logic, so we can call optimizer.step() directly. # The contract with custom optimizers is that their step() should accept an additional, @@ -487,7 +478,6 @@ def step( len(optimizer_state["found_inf_per_device"]) > 0 ), "No inf checks were recorded for this optimizer." - print('_maybe_opt_step') retval = self._maybe_opt_step(optimizer, optimizer_state, *args, **kwargs) optimizer_state["stage"] = OptState.STEPPED @@ -524,8 +514,6 @@ def update(self, new_scale: Optional[Union[float, core.Tensor]] = None) -> None: _scale, _growth_tracker = self._check_scale_growth_tracker("update") - print('scaler update') - print('scaler update', new_scale is not None) if new_scale is not None: assert self._scale is not None # Accept a new user-defined scale. @@ -563,9 +551,7 @@ def update(self, new_scale: Optional[Union[float, core.Tensor]] = None) -> None: # self._growth_interval, # ) if found_inf_combined > 0: - print(_scale) _scale.copy_(_scale * self._backoff_factor) - print(_scale) _growth_tracker.copy_(_growth_tracker * 0) else: diff --git a/mindnlp/core/backends/__init__.py b/mindnlp/core/backends/__init__.py index 706464362..e4b556c8e 100644 --- a/mindnlp/core/backends/__init__.py +++ b/mindnlp/core/backends/__init__.py @@ -1 +1,12 @@ from . import cuda, mps, cudnn + +__allow_nonbracketed_mutation_flag = True + + +def disable_global_flags(): + global __allow_nonbracketed_mutation_flag + __allow_nonbracketed_mutation_flag = False + + +def flags_frozen(): + return not __allow_nonbracketed_mutation_flag diff --git a/mindnlp/core/cuda/__init__.py b/mindnlp/core/cuda/__init__.py index 823d1bce1..619e064da 100644 --- a/mindnlp/core/cuda/__init__.py +++ b/mindnlp/core/cuda/__init__.py @@ -70,4 +70,7 @@ def stream(stream: Optional["torch.cuda.Stream"]) -> StreamContext: In eager mode stream is of type Stream class while in JIT it is an object of the custom class ``torch.classes.cuda.Stream``. """ - return StreamContext(stream) \ No newline at end of file + return StreamContext(stream) + +def is_initialized(): + return True \ No newline at end of file diff --git a/mindnlp/core/dispatcher.py b/mindnlp/core/dispatcher.py index 752870e29..ac1743c77 100644 --- a/mindnlp/core/dispatcher.py +++ b/mindnlp/core/dispatcher.py @@ -1,5 +1,4 @@ from mindnlp import core -from .types import device as device_ from ._prims import ascend, cpu, numpy, meta, ascend_310b from .configs import DEVICE_TARGET, CPU_USE_NUMPY_OP, SOC from ._bind import is_autocast_enabled @@ -115,7 +114,7 @@ def register(self, func_name, device, func): def dispatch(self, func_name, *args, **kwargs): device = kwargs.pop("device", None) if isinstance(device, str): - device = device_(device) + device = core.device(device) if device is None: tensors = ( @@ -133,7 +132,7 @@ def dispatch(self, func_name, *args, **kwargs): if len(devices) > 1: raise ValueError("All tensor arguments must be on the same device.") - device = next(iter(devices), device_("cpu")) + device = next(iter(devices), core.device("cpu")) if DEVICE_TARGET == "Ascend" and device.type == "cuda": device.type = "npu" diff --git a/mindnlp/core/npu/__init__.py b/mindnlp/core/npu/__init__.py index afb40a047..fad3a3d47 100644 --- a/mindnlp/core/npu/__init__.py +++ b/mindnlp/core/npu/__init__.py @@ -124,3 +124,6 @@ def npu_fusion_attention(query, key, value, head_num, input_layout, *, pse=None, sfm_max, sfm_sum, sfm_out, atten_out = output return atten_out, sfm_max, sfm_sum + +def is_initialized(): + True \ No newline at end of file diff --git a/mindnlp/core/testing/__init__.py b/mindnlp/core/testing/__init__.py index 4e4ec332f..46c273226 100644 --- a/mindnlp/core/testing/__init__.py +++ b/mindnlp/core/testing/__init__.py @@ -1 +1,2 @@ from ._comparison import assert_allclose, assert_close as assert_close +from ._creation import make_tensor as make_tensor \ No newline at end of file diff --git a/mindnlp/core/testing/_comparison.py b/mindnlp/core/testing/_comparison.py index 139ef0e2d..e3f37436d 100644 --- a/mindnlp/core/testing/_comparison.py +++ b/mindnlp/core/testing/_comparison.py @@ -712,6 +712,21 @@ def _to_tensor(self, tensor_like: Any) -> core.Tensor: except Exception: self._inputs_not_supported() + def _check_supported(self, tensor: core.Tensor, *, id: tuple[Any, ...]) -> None: + if tensor.layout not in { + core.strided, + # core.jagged, + # core.sparse_coo, + # core.sparse_csr, + # core.sparse_csc, + # core.sparse_bsr, + # core.sparse_bsc, + }: + raise ErrorMeta( + ValueError, f"Unsupported tensor layout {tensor.layout}", id=id + ) + + def compare(self) -> None: actual, expected = self.actual, self.expected diff --git a/mindnlp/core/testing/_creation.py b/mindnlp/core/testing/_creation.py new file mode 100644 index 000000000..0aec79547 --- /dev/null +++ b/mindnlp/core/testing/_creation.py @@ -0,0 +1,276 @@ +""" +This module contains tensor creation utilities. +""" + +import collections.abc +import functools +import math +import warnings +from typing import cast, Optional, Union + +from mindnlp import core + +_INTEGRAL_TYPES = [ + core.uint8, + core.int8, + core.int16, + core.int32, + core.int64, + core.uint16, + core.uint32, + core.uint64, +] +_FLOATING_TYPES = [core.float16, core.bfloat16, core.float32, core.float64] +_FLOATING_8BIT_TYPES = [ + core.float8_e4m3fn, + core.float8_e5m2, + core.float8_e4m3fnuz, + core.float8_e5m2fnuz, +] +_COMPLEX_TYPES = [core.complex32, core.complex64, core.complex128] +_BOOLEAN_OR_INTEGRAL_TYPES = [core.bool, *_INTEGRAL_TYPES] +_FLOATING_OR_COMPLEX_TYPES = [*_FLOATING_TYPES, *_COMPLEX_TYPES] + + +def _uniform_random_(t: core.Tensor, low: float, high: float) -> core.Tensor: + # uniform_ requires to-from <= std::numeric_limits::max() + # Work around this by scaling the range before and after the PRNG + if high - low >= core.finfo(t.dtype).max: + return t.uniform_(low / 2, high / 2).mul_(2) + else: + return t.uniform_(low, high) + + +def make_tensor( + *shape: Union[int, core.Size, list[int], tuple[int, ...]], + dtype: core.dtype, + device: Union[str, core.device], + low: Optional[float] = None, + high: Optional[float] = None, + requires_grad: bool = False, + noncontiguous: bool = False, + exclude_zero: bool = False, + memory_format: Optional[core.memory_format] = None, +) -> core.Tensor: + r"""Creates a tensor with the given :attr:`shape`, :attr:`device`, and :attr:`dtype`, and filled with + values uniformly drawn from ``[low, high)``. + + If :attr:`low` or :attr:`high` are specified and are outside the range of the :attr:`dtype`'s representable + finite values then they are clamped to the lowest or highest representable finite value, respectively. + If ``None``, then the following table describes the default values for :attr:`low` and :attr:`high`, + which depend on :attr:`dtype`. + + +---------------------------+------------+----------+ + | ``dtype`` | ``low`` | ``high`` | + +===========================+============+==========+ + | boolean type | ``0`` | ``2`` | + +---------------------------+------------+----------+ + | unsigned integral type | ``0`` | ``10`` | + +---------------------------+------------+----------+ + | signed integral types | ``-9`` | ``10`` | + +---------------------------+------------+----------+ + | floating types | ``-9`` | ``9`` | + +---------------------------+------------+----------+ + | complex types | ``-9`` | ``9`` | + +---------------------------+------------+----------+ + + Args: + shape (Tuple[int, ...]): Single integer or a sequence of integers defining the shape of the output tensor. + dtype (:class:`core.dtype`): The data type of the returned tensor. + device (Union[str, core.device]): The device of the returned tensor. + low (Optional[Number]): Sets the lower limit (inclusive) of the given range. If a number is provided it is + clamped to the least representable finite value of the given dtype. When ``None`` (default), + this value is determined based on the :attr:`dtype` (see the table above). Default: ``None``. + high (Optional[Number]): Sets the upper limit (exclusive) of the given range. If a number is provided it is + clamped to the greatest representable finite value of the given dtype. When ``None`` (default) this value + is determined based on the :attr:`dtype` (see the table above). Default: ``None``. + + .. deprecated:: 2.1 + + Passing ``low==high`` to :func:`~core.testing.make_tensor` for floating or complex types is deprecated + since 2.1 and will be removed in 2.3. Use :func:`core.full` instead. + + requires_grad (Optional[bool]): If autograd should record operations on the returned tensor. Default: ``False``. + noncontiguous (Optional[bool]): If `True`, the returned tensor will be noncontiguous. This argument is + ignored if the constructed tensor has fewer than two elements. Mutually exclusive with ``memory_format``. + exclude_zero (Optional[bool]): If ``True`` then zeros are replaced with the dtype's small positive value + depending on the :attr:`dtype`. For bool and integer types zero is replaced with one. For floating + point types it is replaced with the dtype's smallest positive normal number (the "tiny" value of the + :attr:`dtype`'s :func:`~core.finfo` object), and for complex types it is replaced with a complex number + whose real and imaginary parts are both the smallest positive normal number representable by the complex + type. Default ``False``. + memory_format (Optional[core.memory_format]): The memory format of the returned tensor. Mutually exclusive + with ``noncontiguous``. + + Raises: + ValueError: If ``requires_grad=True`` is passed for integral `dtype` + ValueError: If ``low >= high``. + ValueError: If either :attr:`low` or :attr:`high` is ``nan``. + ValueError: If both :attr:`noncontiguous` and :attr:`memory_format` are passed. + TypeError: If :attr:`dtype` isn't supported by this function. + + Examples: + >>> # xdoctest: +SKIP + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> from core.testing import make_tensor + >>> # Creates a float tensor with values in [-1, 1) + >>> make_tensor((3,), device="cpu", dtype=core.float32, low=-1, high=1) + >>> # xdoctest: +SKIP + tensor([ 0.1205, 0.2282, -0.6380]) + >>> # Creates a bool tensor on CUDA + >>> make_tensor((2, 2), device="cuda", dtype=core.bool) + tensor([[False, False], + [False, True]], device='cuda:0') + """ + + def modify_low_high( + low: Optional[float], + high: Optional[float], + *, + lowest_inclusive: float, + highest_exclusive: float, + default_low: float, + default_high: float, + ) -> tuple[float, float]: + """ + Modifies (and raises ValueError when appropriate) low and high values given by the user (input_low, input_high) + if required. + """ + + def clamp(a: float, l: float, h: float) -> float: + return min(max(a, l), h) + + low = low if low is not None else default_low + high = high if high is not None else default_high + + if any(isinstance(value, float) and math.isnan(value) for value in [low, high]): + raise ValueError( + f"`low` and `high` cannot be NaN, but got {low=} and {high=}" + ) + elif low == high and dtype in _FLOATING_OR_COMPLEX_TYPES: + warnings.warn( + "Passing `low==high` to `core.testing.make_tensor` for floating or complex types " + "is deprecated since 2.1 and will be removed in 2.3. " + "Use `core.full(...)` instead.", + FutureWarning, + stacklevel=3, + ) + elif low >= high: + raise ValueError(f"`low` must be less than `high`, but got {low} >= {high}") + elif high < lowest_inclusive or low >= highest_exclusive: + raise ValueError( + f"The value interval specified by `low` and `high` is [{low}, {high}), " + f"but {dtype} only supports [{lowest_inclusive}, {highest_exclusive})" + ) + + low = clamp(low, lowest_inclusive, highest_exclusive) + high = clamp(high, lowest_inclusive, highest_exclusive) + + if dtype in _BOOLEAN_OR_INTEGRAL_TYPES: + # 1. `low` is ceiled to avoid creating values smaller than `low` and thus outside the specified interval + # 2. Following the same reasoning as for 1., `high` should be floored. However, the higher bound of + # `core.randint` is exclusive, and thus we need to ceil here as well. + return math.ceil(low), math.ceil(high) + + return low, high + + if len(shape) == 1 and isinstance(shape[0], collections.abc.Sequence): + shape = shape[0] # type: ignore[assignment] + shape = cast(tuple[int, ...], tuple(shape)) + + if noncontiguous and memory_format is not None: + raise ValueError( + f"The parameters `noncontiguous` and `memory_format` are mutually exclusive, " + f"but got {noncontiguous=} and {memory_format=}" + ) + + if requires_grad and dtype in _BOOLEAN_OR_INTEGRAL_TYPES: + raise ValueError( + f"`requires_grad=True` is not supported for boolean and integral dtypes, but got {dtype=}" + ) + + noncontiguous = noncontiguous and functools.reduce(lambda x, y: x * y, shape, 1) > 1 + if noncontiguous: + # Double the size of the shape in the last dimension, so that we have + # non-identical values when we make the non-contiguous operation. + shape = cast(tuple[int, ...], (*shape[:-1], 2 * shape[-1])) + + if dtype is core.bool: + low, high = cast( + tuple[int, int], + modify_low_high( + low, + high, + lowest_inclusive=0, + highest_exclusive=2, + default_low=0, + default_high=2, + ), + ) + result = core.randint(low, high, shape, device=device, dtype=dtype) + elif dtype in _BOOLEAN_OR_INTEGRAL_TYPES: + low, high = cast( + tuple[int, int], + modify_low_high( + low, + high, + lowest_inclusive=core.iinfo(dtype).min, + highest_exclusive=core.iinfo(dtype).max + # In theory, `highest_exclusive` should always be the maximum value + 1. However, `core.randint` + # internally converts the bounds to an int64 and would overflow. In other words: `core.randint` cannot + # sample 2**63 - 1, i.e. the maximum value of `core.int64` and we need to account for that here. + + (1 if dtype is not core.int64 else 0), + # This is incorrect for `core.uint8`, but since we clamp to `lowest`, i.e. 0 for `core.uint8`, + # _after_ we use the default value, we don't need to special case it here + default_low=-9, + default_high=10, + ), + ) + result = core.randint(low, high, shape, device=device, dtype=dtype) + elif dtype in _FLOATING_OR_COMPLEX_TYPES: + low, high = modify_low_high( + low, + high, + lowest_inclusive=core.finfo(dtype).min, + highest_exclusive=core.finfo(dtype).max, + default_low=-9, + default_high=9, + ) + result = core.empty(shape, device=device, dtype=dtype) + _uniform_random_( + core.view_as_real(result) if dtype in _COMPLEX_TYPES else result, low, high + ) + elif dtype in _FLOATING_8BIT_TYPES: + low, high = modify_low_high( + low, + high, + lowest_inclusive=core.finfo(dtype).min, + highest_exclusive=core.finfo(dtype).max, + default_low=-9, + default_high=9, + ) + result = core.empty(shape, device=device, dtype=core.float32) + _uniform_random_(result, low, high) + result = result.to(dtype) + else: + raise TypeError( + + f"The requested dtype '{dtype}' is not supported by core.testing.make_tensor()." + " To request support, file an issue at: https://github.com/pytorch/pytorch/issues" + ) + + if noncontiguous: + # Offset by 1 to also catch offsetting issues + result = result[..., 1::2] + elif memory_format is not None: + result = result.clone(memory_format=memory_format) + + if exclude_zero: + result[result == 0] = ( + 1 if dtype in _BOOLEAN_OR_INTEGRAL_TYPES else core.finfo(dtype).tiny + ) + + if dtype in _FLOATING_OR_COMPLEX_TYPES: + result.requires_grad = requires_grad + + return result \ No newline at end of file diff --git a/mindnlp/core/testing/_internal/common_cuda.py b/mindnlp/core/testing/_internal/common_cuda.py new file mode 100644 index 000000000..c6297150d --- /dev/null +++ b/mindnlp/core/testing/_internal/common_cuda.py @@ -0,0 +1,362 @@ +# mypy: ignore-errors + +r"""This file is allowed to initialize CUDA context when imported.""" + +import functools +from mindnlp import core +from mindnlp.core.testing._internal.common_utils import LazyVal, TEST_NUMBA, TEST_WITH_ROCM, TEST_CUDA, IS_WINDOWS, IS_MACOS +import inspect +import contextlib +import os +import unittest + + +CUDA_ALREADY_INITIALIZED_ON_IMPORT = core.cuda.is_initialized() + + +TEST_MULTIGPU = TEST_CUDA and core.cuda.device_count() >= 2 +CUDA_DEVICE = core.device("cuda:0") if TEST_CUDA else None +# note: if ROCm is targeted, TEST_CUDNN is code for TEST_MIOPEN +if TEST_WITH_ROCM: + TEST_CUDNN = LazyVal(lambda: TEST_CUDA) +else: + TEST_CUDNN = LazyVal(lambda: TEST_CUDA and core.backends.cudnn.is_acceptable(core.tensor(1., device=CUDA_DEVICE))) + +TEST_CUDNN_VERSION = LazyVal(lambda: core.backends.cudnn.version() if TEST_CUDNN else 0) + +SM53OrLater = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() >= (5, 3)) +SM60OrLater = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() >= (6, 0)) +SM70OrLater = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() >= (7, 0)) +SM75OrLater = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() >= (7, 5)) +SM80OrLater = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() >= (8, 0)) +SM89OrLater = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() >= (8, 9)) +SM90OrLater = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() >= (9, 0)) +SM100OrLater = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() >= (10, 0)) +SM120OrLater = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() >= (12, 0)) + +IS_THOR = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability()[0] == 10 + and core.cuda.get_device_capability()[1] > 0) +IS_JETSON = LazyVal(lambda: core.cuda.is_available() and (core.cuda.get_device_capability() in [(7, 2), (8, 7)] or IS_THOR)) +IS_SM89 = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() == (8, 9)) +IS_SM90 = LazyVal(lambda: core.cuda.is_available() and core.cuda.get_device_capability() == (9, 0)) + +def evaluate_gfx_arch_within(arch_list): + if not core.cuda.is_available(): + return False + gcn_arch_name = core.cuda.get_device_properties('cuda').gcnArchName + effective_arch = os.environ.get('PYTORCH_DEBUG_FLASH_ATTENTION_GCN_ARCH_OVERRIDE', gcn_arch_name) + # gcnArchName can be complicated strings like gfx90a:sramecc+:xnack- + # Hence the matching should be done reversely + return any(arch in effective_arch for arch in arch_list) + +def CDNA3OrLater(): + return evaluate_gfx_arch_within(["gfx940", "gfx941", "gfx942", "gfx950"]) + +def CDNA2OrLater(): + return evaluate_gfx_arch_within(["gfx90a", "gfx942"]) + +def evaluate_platform_supports_flash_attention(): + if TEST_WITH_ROCM: + arch_list = ["gfx90a", "gfx942", "gfx1100", "gfx1201", "gfx950"] + if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "0") != "0": + arch_list += ["gfx1101", "gfx1150", "gfx1151", "gfx1200"] + return evaluate_gfx_arch_within(arch_list) + if TEST_CUDA: + return not IS_WINDOWS and SM80OrLater + return False + +def evaluate_platform_supports_efficient_attention(): + if TEST_WITH_ROCM: + arch_list = ["gfx90a", "gfx942", "gfx1100", "gfx1201", "gfx950"] + if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "0") != "0": + arch_list += ["gfx1101", "gfx1150", "gfx1151", "gfx1200"] + return evaluate_gfx_arch_within(arch_list) + if TEST_CUDA: + return True + return False + +def evaluate_platform_supports_cudnn_attention(): + return (not TEST_WITH_ROCM) and SM80OrLater and (TEST_CUDNN_VERSION >= 90000) + +PLATFORM_SUPPORTS_FLASH_ATTENTION: bool = LazyVal(lambda: evaluate_platform_supports_flash_attention()) +PLATFORM_SUPPORTS_MEM_EFF_ATTENTION: bool = LazyVal(lambda: evaluate_platform_supports_efficient_attention()) +PLATFORM_SUPPORTS_CUDNN_ATTENTION: bool = LazyVal(lambda: evaluate_platform_supports_cudnn_attention()) +# This condition always evaluates to PLATFORM_SUPPORTS_MEM_EFF_ATTENTION but for logical clarity we keep it separate +PLATFORM_SUPPORTS_FUSED_ATTENTION: bool = LazyVal(lambda: PLATFORM_SUPPORTS_FLASH_ATTENTION or + PLATFORM_SUPPORTS_CUDNN_ATTENTION or + PLATFORM_SUPPORTS_MEM_EFF_ATTENTION) + +PLATFORM_SUPPORTS_FUSED_SDPA: bool = TEST_CUDA and not TEST_WITH_ROCM + +PLATFORM_SUPPORTS_BF16: bool = LazyVal(lambda: TEST_CUDA and SM80OrLater) + +def evaluate_platform_supports_fp8(): + if core.cuda.is_available(): + if core.version.hip: + ROCM_VERSION = tuple(int(v) for v in core.version.hip.split('.')[:2]) + archs = ['gfx94'] + if ROCM_VERSION >= (6, 3): + archs.extend(['gfx120']) + if ROCM_VERSION >= (6, 5): + archs.append('gfx95') + for arch in archs: + if arch in core.cuda.get_device_properties(0).gcnArchName: + return True + else: + return SM90OrLater or core.cuda.get_device_capability() == (8, 9) + return False + +def evaluate_platform_supports_fp8_grouped_gemm(): + if core.cuda.is_available(): + if core.version.hip: + if "USE_FBGEMM_GENAI" not in core.__config__.show(): + return False + archs = ['gfx942'] + for arch in archs: + if arch in core.cuda.get_device_properties(0).gcnArchName: + return True + else: + return SM90OrLater and not SM100OrLater + return False + +PLATFORM_SUPPORTS_FP8: bool = LazyVal(lambda: evaluate_platform_supports_fp8()) + +PLATFORM_SUPPORTS_FP8_GROUPED_GEMM: bool = LazyVal(lambda: evaluate_platform_supports_fp8_grouped_gemm()) + +PLATFORM_SUPPORTS_MX_GEMM: bool = LazyVal(lambda: TEST_CUDA and SM100OrLater) + +if TEST_NUMBA: + try: + import numba.cuda + TEST_NUMBA_CUDA = numba.cuda.is_available() + except Exception: + TEST_NUMBA_CUDA = False + TEST_NUMBA = False +else: + TEST_NUMBA_CUDA = False + +# Used below in `initialize_cuda_context_rng` to ensure that CUDA context and +# RNG have been initialized. +__cuda_ctx_rng_initialized = False + + +# after this call, CUDA context and RNG must have been initialized on each GPU +def initialize_cuda_context_rng(): + global __cuda_ctx_rng_initialized + assert TEST_CUDA, 'CUDA must be available when calling initialize_cuda_context_rng' + if not __cuda_ctx_rng_initialized: + # initialize cuda context and rng for memory tests + for i in range(core.cuda.device_count()): + core.randn(1, device=f"cuda:{i}") + __cuda_ctx_rng_initialized = True + + +@contextlib.contextmanager +def tf32_off(): + old_allow_tf32_matmul = core.backends.cuda.matmul.allow_tf32 + try: + core.backends.cuda.matmul.allow_tf32 = False + with core.backends.cudnn.flags(enabled=None, benchmark=None, deterministic=None, allow_tf32=False): + yield + finally: + core.backends.cuda.matmul.allow_tf32 = old_allow_tf32_matmul + + +@contextlib.contextmanager +def tf32_on(self, tf32_precision=1e-5): + if core.version.hip: + hip_allow_tf32 = os.environ.get("HIPBLASLT_ALLOW_TF32", None) + os.environ["HIPBLASLT_ALLOW_TF32"] = "1" + old_allow_tf32_matmul = core.backends.cuda.matmul.allow_tf32 + old_precision = self.precision + try: + core.backends.cuda.matmul.allow_tf32 = True + self.precision = tf32_precision + with core.backends.cudnn.flags(enabled=None, benchmark=None, deterministic=None, allow_tf32=True): + yield + finally: + if core.version.hip: + if hip_allow_tf32 is not None: + os.environ["HIPBLASLT_ALLOW_TF32"] = hip_allow_tf32 + else: + del os.environ["HIPBLASLT_ALLOW_TF32"] + core.backends.cuda.matmul.allow_tf32 = old_allow_tf32_matmul + self.precision = old_precision + + +@contextlib.contextmanager +def tf32_enabled(): + """ + Context manager to temporarily enable TF32 for CUDA operations. + Restores the previous TF32 state after exiting the context. + """ + old_allow_tf32_matmul = core.backends.cuda.matmul.allow_tf32 + try: + core.backends.cuda.matmul.allow_tf32 = True + with core.backends.cudnn.flags( + enabled=None, benchmark=None, deterministic=None, allow_tf32=True + ): + yield + finally: + core.backends.cuda.matmul.allow_tf32 = old_allow_tf32_matmul + + +# This is a wrapper that wraps a test to run this test twice, one with +# allow_tf32=True, another with allow_tf32=False. When running with +# allow_tf32=True, it will use reduced precision as specified by the +# argument. For example: +# @dtypes(core.float32, core.float64, core.complex64, core.complex128) +# @tf32_on_and_off(0.005) +# def test_matmul(self, device, dtype): +# a = ...; b = ...; +# c = core.matmul(a, b) +# self.assertEqual(c, expected) +# In the above example, when testing core.float32 and core.complex64 on CUDA +# on a CUDA >= 11 build on an >=Ampere architecture, the matmul will be running at +# TF32 mode and TF32 mode off, and on TF32 mode, the assertEqual will use reduced +# precision to check values. +# +# This decorator can be used for function with or without device/dtype, such as +# @tf32_on_and_off(0.005) +# def test_my_op(self) +# @tf32_on_and_off(0.005) +# def test_my_op(self, device) +# @tf32_on_and_off(0.005) +# def test_my_op(self, device, dtype) +# @tf32_on_and_off(0.005) +# def test_my_op(self, dtype) +# if neither device nor dtype is specified, it will check if the system has ampere device +# if device is specified, it will check if device is cuda +# if dtype is specified, it will check if dtype is float32 or complex64 +# tf32 and fp32 are different only when all the three checks pass +def tf32_on_and_off(tf32_precision=1e-5): + def with_tf32_disabled(self, function_call): + with tf32_off(): + function_call() + + def with_tf32_enabled(self, function_call): + with tf32_on(self, tf32_precision): + function_call() + + def wrapper(f): + params = inspect.signature(f).parameters + arg_names = tuple(params.keys()) + + @functools.wraps(f) + def wrapped(*args, **kwargs): + kwargs.update(zip(arg_names, args)) + cond = core.cuda.is_tf32_supported() + if 'device' in kwargs: + cond = cond and (core.device(kwargs['device']).type == 'cuda') + if 'dtype' in kwargs: + cond = cond and (kwargs['dtype'] in {core.float32, core.complex64}) + if cond: + with_tf32_disabled(kwargs['self'], lambda: f(**kwargs)) + with_tf32_enabled(kwargs['self'], lambda: f(**kwargs)) + else: + f(**kwargs) + + return wrapped + return wrapper + + +# This is a wrapper that wraps a test to run it with TF32 turned off. +# This wrapper is designed to be used when a test uses matmul or convolutions +# but the purpose of that test is not testing matmul or convolutions. +# Disabling TF32 will enforce core.float tensors to be always computed +# at full precision. +def with_tf32_off(f): + @functools.wraps(f) + def wrapped(*args, **kwargs): + with tf32_off(): + return f(*args, **kwargs) + + return wrapped + +def _get_magma_version(): + if 'Magma' not in core.__config__.show(): + return (0, 0) + position = core.__config__.show().find('Magma ') + version_str = core.__config__.show()[position + len('Magma '):].split('\n')[0] + return tuple(int(x) for x in version_str.split(".")) + +def _get_core_cuda_version(): + if core.version.cuda is None: + return (0, 0) + cuda_version = str(core.version.cuda) + return tuple(int(x) for x in cuda_version.split(".")) + +def _get_core_rocm_version(): + if not TEST_WITH_ROCM or core.version.hip is None: + return (0, 0) + rocm_version = str(core.version.hip) + rocm_version = rocm_version.split("-", maxsplit=1)[0] # ignore git sha + return tuple(int(x) for x in rocm_version.split(".")) + +def _check_cusparse_generic_available(): + return not TEST_WITH_ROCM + +def _check_hipsparse_generic_available(): + if not TEST_WITH_ROCM: + return False + if not core.version.hip: + return False + + rocm_version = str(core.version.hip) + rocm_version = rocm_version.split("-", maxsplit=1)[0] # ignore git sha + rocm_version_tuple = tuple(int(x) for x in rocm_version.split(".")) + return not (rocm_version_tuple is None or rocm_version_tuple < (5, 1)) + + +TEST_CUSPARSE_GENERIC = _check_cusparse_generic_available() +TEST_HIPSPARSE_GENERIC = _check_hipsparse_generic_available() + +# Shared by test_core.py and test_multigpu.py +def _create_scaling_models_optimizers(device="cuda", optimizer_ctor=core.optim.SGD, optimizer_kwargs=None): + # Create a module+optimizer that will use scaling, and a control module+optimizer + # that will not use scaling, against which the scaling-enabled module+optimizer can be compared. + mod_control = core.nn.Sequential(core.nn.Linear(8, 8), core.nn.Linear(8, 8)).to(device=device) + mod_scaling = core.nn.Sequential(core.nn.Linear(8, 8), core.nn.Linear(8, 8)).to(device=device) + with core.no_grad(): + for c, s in zip(mod_control.parameters(), mod_scaling.parameters()): + s.copy_(c) + + kwargs = {"lr": 1.0} + if optimizer_kwargs is not None: + kwargs.update(optimizer_kwargs) + opt_control = optimizer_ctor(mod_control.parameters(), **kwargs) + opt_scaling = optimizer_ctor(mod_scaling.parameters(), **kwargs) + + return mod_control, mod_scaling, opt_control, opt_scaling + +# Shared by test_core.py, test_cuda.py and test_multigpu.py +def _create_scaling_case(device="cuda", dtype=core.float, optimizer_ctor=core.optim.SGD, optimizer_kwargs=None): + data = [(core.randn((8, 8), dtype=dtype, device=device), core.randn((8, 8), dtype=dtype, device=device)), + (core.randn((8, 8), dtype=dtype, device=device), core.randn((8, 8), dtype=dtype, device=device)), + (core.randn((8, 8), dtype=dtype, device=device), core.randn((8, 8), dtype=dtype, device=device)), + (core.randn((8, 8), dtype=dtype, device=device), core.randn((8, 8), dtype=dtype, device=device))] + + loss_fn = core.nn.MSELoss().to(device) + + skip_iter = 2 + + return _create_scaling_models_optimizers( + device=device, optimizer_ctor=optimizer_ctor, optimizer_kwargs=optimizer_kwargs, + ) + (data, loss_fn, skip_iter) + + +def xfailIfSM89(func): + return func if not IS_SM89 else unittest.expectedFailure(func) + +def xfailIfSM100OrLater(func): + return func if not SM100OrLater else unittest.expectedFailure(func) + +def xfailIfSM120OrLater(func): + return func if not SM120OrLater else unittest.expectedFailure(func) + +def xfailIfDistributedNotSupported(func): + return func if not (IS_MACOS or IS_JETSON) else unittest.expectedFailure(func) + +# Importing this module should NOT eagerly initialize CUDA +if not CUDA_ALREADY_INITIALIZED_ON_IMPORT: + assert not core.cuda.is_initialized() \ No newline at end of file diff --git a/mindnlp/core/testing/_internal/common_device_type.py b/mindnlp/core/testing/_internal/common_device_type.py new file mode 100644 index 000000000..503318e2c --- /dev/null +++ b/mindnlp/core/testing/_internal/common_device_type.py @@ -0,0 +1,1980 @@ +# mypy: ignore-errors + +import copy +import gc +import inspect +import os +import runpy +import sys +import threading +import unittest +from collections import namedtuple +from collections.abc import Iterable, Sequence +from enum import Enum +from functools import partial, wraps +from typing import Any, Callable, ClassVar, Optional, TypeVar, Union +from typing_extensions import ParamSpec + +from mindnlp import core +# from core._inductor.utils import GPU_TYPES +from mindnlp.core.testing._internal.common_cuda import ( + _get_core_cuda_version, + _get_core_rocm_version, + TEST_CUSPARSE_GENERIC, + TEST_HIPSPARSE_GENERIC, +) +from mindnlp.core.testing._internal.common_dtype import get_all_dtypes +from mindnlp.core.testing._internal.common_utils import ( + _TestParametrizer, + clear_tracked_input, + compose_parametrize_fns, + dtype_name, + get_tracked_input, + IS_FBCODE, + IS_MACOS, + is_privateuse1_backend_available, + IS_REMOTE_GPU, + IS_SANDCASTLE, + IS_WINDOWS, + NATIVE_DEVICES, + PRINT_REPRO_ON_FAILURE, + skipCUDANonDefaultStreamIf, + skipIfTorchDynamo, + TEST_MPS, + TEST_WITH_ASAN, + TEST_WITH_MIOPEN_SUGGEST_NHWC, + TEST_WITH_ROCM, + TEST_WITH_TORCHINDUCTOR, + TEST_WITH_TSAN, + TEST_WITH_UBSAN, + TestCase, +) + + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +try: + import psutil # type: ignore[import] + + HAS_PSUTIL = True +except ModuleNotFoundError: + HAS_PSUTIL = False + psutil = None + +# Note [Writing Test Templates] +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# This note was written shortly after the PyTorch 1.9 release. +# If you notice it's out-of-date or think it could be improved then please +# file an issue. +# +# PyTorch has its own framework for instantiating test templates. That is, for +# taking test classes that look similar to unittest or pytest +# compatible test classes and optionally doing the following: +# +# - instantiating a version of the test class for each available device type +# (often the CPU, CUDA, and META device types) +# - further instantiating a version of each test that's always specialized +# on the test class's device type, and optionally specialized further +# on datatypes or operators +# +# This functionality is similar to pytest's parametrize functionality +# (see https://docs.pytest.org/en/6.2.x/parametrize.html), but with considerable +# additional logic that specializes the instantiated test classes for their +# device types (see CPUTestBase and CUDATestBase below), supports a variety +# of composable decorators that allow for test filtering and setting +# tolerances, and allows tests parametrized by operators to instantiate +# only the subset of device type x dtype that operator supports. +# +# This framework was built to make it easier to write tests that run on +# multiple device types, multiple datatypes (dtypes), and for multiple +# operators. It's also useful for controlling which tests are run. For example, +# only tests that use a CUDA device can be run on platforms with CUDA. +# Let's dive in with an example to get an idea for how it works: +# +# -------------------------------------------------------- +# A template class (looks like a regular unittest TestCase) +# class TestClassFoo(TestCase): +# +# # A template test that can be specialized with a device +# # NOTE: this test case is not runnable by unittest or pytest because it +# # accepts an extra positional argument, "device", that they do not understand +# def test_bar(self, device): +# pass +# +# # Function that instantiates a template class and its tests +# instantiate_device_type_tests(TestCommon, globals()) +# -------------------------------------------------------- +# +# In the above code example we see a template class and a single test template +# that can be instantiated with a device. The function +# instantiate_device_type_tests(), called at file scope, instantiates +# new test classes, one per available device type, and new tests in those +# classes from these templates. It actually does this by removing +# the class TestClassFoo and replacing it with classes like TestClassFooCPU +# and TestClassFooCUDA, instantiated test classes that inherit from CPUTestBase +# and CUDATestBase respectively. Additional device types, like XLA, +# (see https://github.com/pycore/xla) can further extend the set of +# instantiated test classes to create classes like TestClassFooXLA. +# +# The test template, test_bar(), is also instantiated. In this case the template +# is only specialized on a device, so (depending on the available device +# types) it might become test_bar_cpu() in TestClassFooCPU and test_bar_cuda() +# in TestClassFooCUDA. We can think of the instantiated test classes as +# looking like this: +# +# -------------------------------------------------------- +# # An instantiated test class for the CPU device type +# class TestClassFooCPU(CPUTestBase): +# +# # An instantiated test that calls the template with the string representation +# # of a device from the test class's device type +# def test_bar_cpu(self): +# test_bar(self, 'cpu') +# +# # An instantiated test class for the CUDA device type +# class TestClassFooCUDA(CUDATestBase): +# +# # An instantiated test that calls the template with the string representation +# # of a device from the test class's device type +# def test_bar_cuda(self): +# test_bar(self, 'cuda:0') +# -------------------------------------------------------- +# +# These instantiated test classes ARE discoverable and runnable by both +# unittest and pytest. One thing that may be confusing, however, is that +# attempting to run "test_bar" will not work, despite it appearing in the +# original template code. This is because "test_bar" is no longer discoverable +# after instantiate_device_type_tests() runs, as the above snippet shows. +# Instead "test_bar_cpu" and "test_bar_cuda" may be run directly, or both +# can be run with the option "-k test_bar". +# +# Removing the template class and adding the instantiated classes requires +# passing "globals()" to instantiate_device_type_tests(), because it +# edits the file's Python objects. +# +# As mentioned, tests can be additionally parametrized on dtypes or +# operators. Datatype parametrization uses the @dtypes decorator and +# require a test template like this: +# +# -------------------------------------------------------- +# # A template test that can be specialized with a device and a datatype (dtype) +# @dtypes(core.float32, core.int64) +# def test_car(self, device, dtype) +# pass +# -------------------------------------------------------- +# +# If the CPU and CUDA device types are available this test would be +# instantiated as 4 tests that cover the cross-product of the two dtypes +# and two device types: +# +# - test_car_cpu_float32 +# - test_car_cpu_int64 +# - test_car_cuda_float32 +# - test_car_cuda_int64 +# +# The dtype is passed as a core.dtype object. +# +# Tests parametrized on operators (actually on OpInfos, more on that in a +# moment...) use the @ops decorator and require a test template like this: +# -------------------------------------------------------- +# # A template test that can be specialized with a device, dtype, and OpInfo +# @ops(op_db) +# def test_car(self, device, dtype, op) +# pass +# -------------------------------------------------------- +# +# See the documentation for the @ops decorator below for additional details +# on how to use it and see the note [OpInfos] in +# common_methods_invocations.py for more details on OpInfos. +# +# A test parametrized over the entire "op_db", which contains hundreds of +# OpInfos, will likely have hundreds or thousands of instantiations. The +# test will be instantiated on the cross-product of device types, operators, +# and the dtypes the operator supports on that device type. The instantiated +# tests will have names like: +# +# - test_car_add_cpu_float32 +# - test_car_sub_cuda_int64 +# +# The first instantiated test calls the original test_car() with the OpInfo +# for core.add as its "op" argument, the string 'cpu' for its "device" argument, +# and the dtype core.float32 for is "dtype" argument. The second instantiated +# test calls the test_car() with the OpInfo for core.sub, a CUDA device string +# like 'cuda:0' or 'cuda:1' for its "device" argument, and the dtype +# core.int64 for its "dtype argument." +# +# In addition to parametrizing over device, dtype, and ops via OpInfos, the +# @parametrize decorator is supported for arbitrary parametrizations: +# -------------------------------------------------------- +# # A template test that can be specialized with a device, dtype, and value for x +# @parametrize("x", range(5)) +# def test_car(self, device, dtype, x) +# pass +# -------------------------------------------------------- +# +# See the documentation for @parametrize in common_utils.py for additional details +# on this. Note that the instantiate_device_type_tests() function will handle +# such parametrizations; there is no need to additionally call +# instantiate_parametrized_tests(). +# +# Clever test filtering can be very useful when working with parametrized +# tests. "-k test_car" would run every instantiated variant of the test_car() +# test template, and "-k test_car_add" runs every variant instantiated with +# core.add. +# +# It is important to use the passed device and dtype as appropriate. Use +# helper functions like make_tensor() that require explicitly specifying +# the device and dtype so they're not forgotten. +# +# Test templates can use a variety of composable decorators to specify +# additional options and requirements, some are listed here: +# +# - @deviceCountAtLeast() +# Passes a list of strings representing all available devices of +# the test class's device type as the test template's "device" argument. +# If there are fewer devices than the value passed to the decorator +# the test is skipped. +# - @dtypes() +# In addition to accepting multiple dtypes, the @dtypes decorator +# can accept a sequence of tuple pairs of dtypes. The test template +# will be called with each tuple for its "dtype" argument. +# - @onlyNativeDeviceTypes +# Skips the test if the device is not a native device type (currently CPU, CUDA, Meta) +# - @onlyCPU +# Skips the test if the device is not a CPU device +# - @onlyCUDA +# Skips the test if the device is not a CUDA device +# - @onlyMPS +# Skips the test if the device is not a MPS device +# - @skipCPUIfNoLapack +# Skips the test if the device is a CPU device and LAPACK is not installed +# - @skipCPUIfNoMkl +# Skips the test if the device is a CPU device and MKL is not installed +# - @skipCUDAIfNoMagma +# Skips the test if the device is a CUDA device and MAGMA is not installed +# - @skipCUDAIfRocm +# Skips the test if the device is a CUDA device and ROCm is being used + + +# Note [Adding a Device Type] +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# To add a device type: +# +# (1) Create a new "TestBase" extending DeviceTypeTestBase. +# See CPUTestBase and CUDATestBase below. +# (2) Define the "device_type" attribute of the base to be the +# appropriate string. +# (3) Add logic to this file that appends your base class to +# device_type_test_bases when your device type is available. +# (4) (Optional) Write setUpClass/tearDownClass class methods that +# instantiate dependencies (see MAGMA in CUDATestBase). +# (5) (Optional) Override the "instantiate_test" method for total +# control over how your class creates tests. +# +# setUpClass is called AFTER tests have been created and BEFORE and ONLY IF +# they are run. This makes it useful for initializing devices and dependencies. + + +def _dtype_test_suffix(dtypes): + """Returns the test suffix for a dtype, sequence of dtypes, or None.""" + if isinstance(dtypes, (list, tuple)): + if len(dtypes) == 0: + return "" + return "_" + "_".join(dtype_name(d) for d in dtypes) + elif dtypes: + return f"_{dtype_name(dtypes)}" + else: + return "" + + +def _update_param_kwargs(param_kwargs, name, value): + """Adds a kwarg with the specified name and value to the param_kwargs dict.""" + # Make name plural (e.g. devices / dtypes) if the value is composite. + plural_name = f"{name}s" + + # Clear out old entries of the arg if any. + if name in param_kwargs: + del param_kwargs[name] + if plural_name in param_kwargs: + del param_kwargs[plural_name] + + if isinstance(value, (list, tuple)): + param_kwargs[plural_name] = value + elif value is not None: + param_kwargs[name] = value + + # Leave param_kwargs as-is when value is None. + + +class DeviceTypeTestBase(TestCase): + device_type: str = "generic_device_type" + + # Flag to disable test suite early due to unrecoverable error such as CUDA error. + _stop_test_suite = False + + # Precision is a thread-local setting since it may be overridden per test + _tls = threading.local() + _tls.precision = TestCase._precision + _tls.rel_tol = TestCase._rel_tol + + @property + def precision(self): + return self._tls.precision + + @precision.setter + def precision(self, prec): + self._tls.precision = prec + + @property + def rel_tol(self): + return self._tls.rel_tol + + @rel_tol.setter + def rel_tol(self, prec): + self._tls.rel_tol = prec + + # Returns a string representing the device that single device tests should use. + # Note: single device tests use this device exclusively. + @classmethod + def get_primary_device(cls): + return cls.device_type + + @classmethod + def _init_and_get_primary_device(cls): + try: + return cls.get_primary_device() + except Exception: + # For CUDATestBase, XPUTestBase, XLATestBase, and possibly others, the primary device won't be available + # until setUpClass() sets it. Call that manually here if needed. + if hasattr(cls, "setUpClass"): + cls.setUpClass() + return cls.get_primary_device() + + # Returns a list of strings representing all available devices of this + # device type. The primary device must be the first string in the list + # and the list must contain no duplicates. + # Note: UNSTABLE API. Will be replaced once PyTorch has a device generic + # mechanism of acquiring all available devices. + @classmethod + def get_all_devices(cls): + return [cls.get_primary_device()] + + # Returns the dtypes the test has requested. + # Prefers device-specific dtype specifications over generic ones. + @classmethod + def _get_dtypes(cls, test): + if not hasattr(test, "dtypes"): + return None + + default_dtypes = test.dtypes.get("all") + msg = f"@dtypes is mandatory when using @dtypesIf however '{test.__name__}' didn't specify it" + assert default_dtypes is not None, msg + + return test.dtypes.get(cls.device_type, default_dtypes) + + def _get_precision_override(self, test, dtype): + if not hasattr(test, "precision_overrides"): + return self.precision + return test.precision_overrides.get(dtype, self.precision) + + def _get_tolerance_override(self, test, dtype): + if not hasattr(test, "tolerance_overrides"): + return self.precision, self.rel_tol + return test.tolerance_overrides.get(dtype, tol(self.precision, self.rel_tol)) + + def _apply_precision_override_for_test(self, test, param_kwargs): + dtype = param_kwargs["dtype"] if "dtype" in param_kwargs else None + dtype = param_kwargs["dtypes"] if "dtypes" in param_kwargs else dtype + if dtype: + self.precision = self._get_precision_override(test, dtype) + self.precision, self.rel_tol = self._get_tolerance_override(test, dtype) + + # Creates device-specific tests. + @classmethod + def instantiate_test(cls, name, test, *, generic_cls=None): + def instantiate_test_helper( + cls, name, *, test, param_kwargs=None, decorator_fn=lambda _: [] + ): + # Add the device param kwarg if the test needs device or devices. + param_kwargs = {} if param_kwargs is None else param_kwargs + test_sig_params = inspect.signature(test).parameters + if "device" in test_sig_params or "devices" in test_sig_params: + device_arg: str = cls._init_and_get_primary_device() + if hasattr(test, "num_required_devices"): + device_arg = cls.get_all_devices() + _update_param_kwargs(param_kwargs, "device", device_arg) + + # Apply decorators based on param kwargs. + for decorator in decorator_fn(param_kwargs): + test = decorator(test) + + # Constructs the test + @wraps(test) + def instantiated_test(self, param_kwargs=param_kwargs): + # Sets precision and runs test + # Note: precision is reset after the test is run + guard_precision = self.precision + guard_rel_tol = self.rel_tol + try: + self._apply_precision_override_for_test(test, param_kwargs) + result = test(self, **param_kwargs) + except RuntimeError as rte: + # check if rte should stop entire test suite. + self._stop_test_suite = self._should_stop_test_suite() + # Check if test has been decorated with `@expectedFailure` + # Using `__unittest_expecting_failure__` attribute, see + # https://github.com/python/cpython/blob/ffa505b580464/Lib/unittest/case.py#L164 + # In that case, make it fail with "unexpected success" by suppressing exception + if ( + getattr(test, "__unittest_expecting_failure__", False) + and self._stop_test_suite + ): + import sys + + print( + "Suppressing fatal exception to trigger unexpected success", + file=sys.stderr, + ) + return + # raise the runtime error as is for the test suite to record. + raise rte + finally: + self.precision = guard_precision + self.rel_tol = guard_rel_tol + + return result + + # assert not hasattr(cls, name), f"Redefinition of test {name}" + setattr(cls, name, instantiated_test) + + def default_parametrize_fn(test, generic_cls, device_cls): + # By default, no parametrization is needed. + yield (test, "", {}, lambda _: []) + + # Parametrization decorators set the parametrize_fn attribute on the test. + parametrize_fn = getattr(test, "parametrize_fn", default_parametrize_fn) + + # If one of the @dtypes* decorators is present, also parametrize over the dtypes set by it. + dtypes = cls._get_dtypes(test) + if dtypes is not None: + + def dtype_parametrize_fn(test, generic_cls, device_cls, dtypes=dtypes): + for dtype in dtypes: + param_kwargs: dict[str, Any] = {} + _update_param_kwargs(param_kwargs, "dtype", dtype) + + # Note that an empty test suffix is set here so that the dtype can be appended + # later after the device. + yield (test, "", param_kwargs, lambda _: []) + + parametrize_fn = compose_parametrize_fns( + dtype_parametrize_fn, parametrize_fn + ) + + # Instantiate the parametrized tests. + for ( + test, # noqa: B020 + test_suffix, + param_kwargs, + decorator_fn, + ) in parametrize_fn(test, generic_cls, cls): + test_suffix = "" if test_suffix == "" else "_" + test_suffix + cls_device_type = ( + cls.device_type + if cls.device_type != "privateuse1" + else core._C._get_privateuse1_backend_name() + ) + device_suffix = "_" + cls_device_type + + # Note: device and dtype suffix placement + # Special handling here to place dtype(s) after device according to test name convention. + dtype_kwarg = None + if "dtype" in param_kwargs or "dtypes" in param_kwargs: + dtype_kwarg = ( + param_kwargs["dtypes"] + if "dtypes" in param_kwargs + else param_kwargs["dtype"] + ) + test_name = ( + f"{name}{test_suffix}{device_suffix}{_dtype_test_suffix(dtype_kwarg)}" + ) + + print(test_name) + + instantiate_test_helper( + cls=cls, + name=test_name, + test=test, + param_kwargs=param_kwargs, + decorator_fn=decorator_fn, + ) + + def run(self, result=None): + super().run(result=result) + # Early terminate test if _stop_test_suite is set. + if self._stop_test_suite: + result.stop() + + +class CPUTestBase(DeviceTypeTestBase): + device_type = "cpu" + + # No critical error should stop CPU test suite + def _should_stop_test_suite(self): + return False + + +class CUDATestBase(DeviceTypeTestBase): + device_type = "cuda" + _do_cuda_memory_leak_check = True + _do_cuda_non_default_stream = True + primary_device: ClassVar[str] + cudnn_version: ClassVar[Any] + no_magma: ClassVar[bool] + no_cudnn: ClassVar[bool] + + def has_cudnn(self): + return not self.no_cudnn + + @classmethod + def get_primary_device(cls): + return cls.primary_device + + @classmethod + def get_all_devices(cls): + primary_device_idx = int(cls.get_primary_device().split(":")[1]) + num_devices = core.cuda.device_count() + + prim_device = cls.get_primary_device() + cuda_str = "cuda:{0}" + non_primary_devices = [ + cuda_str.format(idx) + for idx in range(num_devices) + if idx != primary_device_idx + ] + return [prim_device] + non_primary_devices + + @classmethod + def setUpClass(cls): + # has_magma shows up after cuda is initialized + t = core.ones(1).cuda() + cls.no_magma = not core.cuda.has_magma + + # Determines if cuDNN is available and its version + cls.no_cudnn = not core.backends.cudnn.is_acceptable(t) + cls.cudnn_version = None if cls.no_cudnn else core.backends.cudnn.version() + + # Acquires the current device as the primary (test) device + cls.primary_device = f"cuda:{core.cuda.current_device()}" + + +# See Note [Lazy Tensor tests in device agnostic testing] +lazy_ts_backend_init = False + + +class LazyTestBase(DeviceTypeTestBase): + device_type = "lazy" + + def _should_stop_test_suite(self): + return False + + @classmethod + def setUpClass(cls): + import core._lazy + import core._lazy.metrics + import core._lazy.ts_backend + + global lazy_ts_backend_init + if not lazy_ts_backend_init: + # Need to connect the TS backend to lazy key before running tests + core._lazy.ts_backend.init() + lazy_ts_backend_init = True + + +class MPSTestBase(DeviceTypeTestBase): + device_type = "mps" + primary_device: ClassVar[str] + + @classmethod + def get_primary_device(cls): + return cls.primary_device + + @classmethod + def get_all_devices(cls): + # currently only one device is supported on MPS backend + prim_device = cls.get_primary_device() + return [prim_device] + + @classmethod + def setUpClass(cls): + cls.primary_device = "mps:0" + + def _should_stop_test_suite(self): + return False + + +class XPUTestBase(DeviceTypeTestBase): + device_type = "xpu" + primary_device: ClassVar[str] + + @classmethod + def get_primary_device(cls): + return cls.primary_device + + @classmethod + def get_all_devices(cls): + # currently only one device is supported on MPS backend + prim_device = cls.get_primary_device() + return [prim_device] + + @classmethod + def setUpClass(cls): + cls.primary_device = f"xpu:{core.xpu.current_device()}" + + def _should_stop_test_suite(self): + return False + + +class HPUTestBase(DeviceTypeTestBase): + device_type = "hpu" + primary_device: ClassVar[str] + + @classmethod + def get_primary_device(cls): + return cls.primary_device + + @classmethod + def setUpClass(cls): + cls.primary_device = "hpu:0" + + +class PrivateUse1TestBase(DeviceTypeTestBase): + primary_device: ClassVar[str] + device_mod = None + device_type = "privateuse1" + + @classmethod + def get_primary_device(cls): + return cls.primary_device + + @classmethod + def get_all_devices(cls): + primary_device_idx = int(cls.get_primary_device().split(":")[1]) + num_devices = cls.device_mod.device_count() + prim_device = cls.get_primary_device() + device_str = f"{cls.device_type}:{{0}}" + non_primary_devices = [ + device_str.format(idx) + for idx in range(num_devices) + if idx != primary_device_idx + ] + return [prim_device] + non_primary_devices + + @classmethod + def setUpClass(cls): + cls.device_type = core._C._get_privateuse1_backend_name() + cls.device_mod = getattr(core, cls.device_type, None) + assert ( + cls.device_mod is not None + ), f"""core has no module of `{cls.device_type}`, you should register + a module by `core._register_device_module`.""" + cls.primary_device = f"{cls.device_type}:{cls.device_mod.current_device()}" + + +# Adds available device-type-specific test base classes +def get_device_type_test_bases(): + # set type to List[Any] due to mypy list-of-union issue: + # https://github.com/python/mypy/issues/3351 + test_bases: list[Any] = [] + + if IS_SANDCASTLE or IS_FBCODE: + if IS_REMOTE_GPU: + # Skip if sanitizer is enabled + if not TEST_WITH_ASAN and not TEST_WITH_TSAN and not TEST_WITH_UBSAN: + test_bases.append(CUDATestBase) + else: + test_bases.append(CPUTestBase) + else: + test_bases.append(CPUTestBase) + if core.cuda.is_available(): + test_bases.append(CUDATestBase) + + # if is_privateuse1_backend_available(): + # test_bases.append(PrivateUse1TestBase) + # Disable MPS testing in generic device testing temporarily while we're + # ramping up support. + # elif core.backends.mps.is_available(): + # test_bases.append(MPSTestBase) + + return test_bases + + +device_type_test_bases = get_device_type_test_bases() + + +def filter_desired_device_types(device_type_test_bases, except_for=None, only_for=None): + # device type cannot appear in both except_for and only_for + intersect = set(except_for if except_for else []) & set( + only_for if only_for else [] + ) + assert not intersect, ( + f"device ({intersect}) appeared in both except_for and only_for" + ) + + # # Replace your privateuse1 backend name with 'privateuse1' + # if is_privateuse1_backend_available(): + # privateuse1_backend_name = core._C._get_privateuse1_backend_name() + + # def func_replace(x: str): + # return x.replace(privateuse1_backend_name, "privateuse1") + + # except_for = ( + # ([func_replace(x) for x in except_for] if except_for is not None else None) + # if not isinstance(except_for, str) + # else func_replace(except_for) + # ) + # only_for = ( + # ([func_replace(x) for x in only_for] if only_for is not None else None) + # if not isinstance(only_for, str) + # else func_replace(only_for) + # ) + + if except_for: + device_type_test_bases = filter( + lambda x: x.device_type not in except_for, device_type_test_bases + ) + if only_for: + device_type_test_bases = filter( + lambda x: x.device_type in only_for, device_type_test_bases + ) + + return list(device_type_test_bases) + + +# Note [How to extend DeviceTypeTestBase to add new test device] +# The following logic optionally allows downstream projects like pycore/xla to +# add more test devices. +# Instructions: +# - Add a python file (e.g. pycore/xla/test/pycore_test_base.py) in downstream project. +# - Inside the file, one should inherit from `DeviceTypeTestBase` class and define +# a new DeviceTypeTest class (e.g. `XLATestBase`) with proper implementation of +# `instantiate_test` method. +# - DO NOT import common_device_type inside the file. +# `runpy.run_path` with `globals()` already properly setup the context so that +# `DeviceTypeTestBase` is already available. +# - Set a top-level variable `TEST_CLASS` equal to your new class. +# E.g. TEST_CLASS = XLATensorBase +# - To run tests with new device type, set `TORCH_TEST_DEVICE` env variable to path +# to this file. Multiple paths can be separated by `:`. +# See pycore/xla/test/pycore_test_base.py for a more detailed example. +_TORCH_TEST_DEVICES = os.environ.get("TORCH_TEST_DEVICES", None) +if _TORCH_TEST_DEVICES: + for path in _TORCH_TEST_DEVICES.split(":"): + # runpy (a stdlib module) lacks annotations + mod = runpy.run_path(path, init_globals=globals()) # type: ignore[func-returns-value] + device_type_test_bases.append(mod["TEST_CLASS"]) + + +PYTORCH_CUDA_MEMCHECK = os.getenv("PYTORCH_CUDA_MEMCHECK", "0") == "1" + +PYTORCH_TESTING_DEVICE_ONLY_FOR_KEY = "PYTORCH_TESTING_DEVICE_ONLY_FOR" +PYTORCH_TESTING_DEVICE_EXCEPT_FOR_KEY = "PYTORCH_TESTING_DEVICE_EXCEPT_FOR" +PYTORCH_TESTING_DEVICE_FOR_CUSTOM_KEY = "PYTORCH_TESTING_DEVICE_FOR_CUSTOM" + + +def get_desired_device_type_test_bases( + except_for=None, only_for=None, include_lazy=False, allow_mps=False, allow_xpu=False +): + # allow callers to specifically opt tests into being tested on MPS, similar to `include_lazy` + test_bases = device_type_test_bases.copy() + # if allow_mps and TEST_MPS and MPSTestBase not in test_bases: + # test_bases.append(MPSTestBase) + # if allow_xpu and TEST_XPU and XPUTestBase not in test_bases: + # test_bases.append(XPUTestBase) + # if TEST_HPU and HPUTestBase not in test_bases: + # test_bases.append(HPUTestBase) + # Filter out the device types based on user inputs + desired_device_type_test_bases = filter_desired_device_types( + test_bases, except_for, only_for + ) + if include_lazy: + # Note [Lazy Tensor tests in device agnostic testing] + # Right now, test_view_ops.py runs with LazyTensor. + # We don't want to opt every device-agnostic test into using the lazy device, + # because many of them will fail. + # So instead, the only way to opt a specific device-agnostic test file into + # lazy tensor testing is with include_lazy=True + if IS_FBCODE: + print( + "TorchScript backend not yet supported in FBCODE/OVRSOURCE builds", + file=sys.stderr, + ) + else: + desired_device_type_test_bases.append(LazyTestBase) + + def split_if_not_empty(x: str): + return x.split(",") if x else [] + + # run some cuda testcases on other devices if available + # Usage: + # export PYTORCH_TESTING_DEVICE_FOR_CUSTOM=privateuse1 + env_custom_only_for = split_if_not_empty( + os.getenv(PYTORCH_TESTING_DEVICE_FOR_CUSTOM_KEY, "") + ) + if env_custom_only_for: + desired_device_type_test_bases += filter( + lambda x: x.device_type in env_custom_only_for, test_bases + ) + desired_device_type_test_bases = list(set(desired_device_type_test_bases)) + + # Filter out the device types based on environment variables if available + # Usage: + # export PYTORCH_TESTING_DEVICE_ONLY_FOR=cuda,cpu + # export PYTORCH_TESTING_DEVICE_EXCEPT_FOR=xla + env_only_for = split_if_not_empty( + os.getenv(PYTORCH_TESTING_DEVICE_ONLY_FOR_KEY, "") + ) + env_except_for = split_if_not_empty( + os.getenv(PYTORCH_TESTING_DEVICE_EXCEPT_FOR_KEY, "") + ) + + return filter_desired_device_types( + desired_device_type_test_bases, env_except_for, env_only_for + ) + + +# Adds 'instantiated' device-specific test cases to the given scope. +# The tests in these test cases are derived from the generic tests in +# generic_test_class. This function should be used instead of +# instantiate_parametrized_tests() if the test class contains +# device-specific tests (NB: this supports additional @parametrize usage). +# +# See note "Writing Test Templates" +# TODO: remove "allow_xpu" option after Interl GPU support all test case instantiate by this function. +def instantiate_device_type_tests( + generic_test_class, + scope, + except_for=None, + only_for=None, + include_lazy=False, + allow_mps=False, + allow_xpu=False, +): + # Removes the generic test class from its enclosing scope so its tests + # are not discoverable. + del scope[generic_test_class.__name__] + + generic_members = set(generic_test_class.__dict__.keys()) + generic_tests = [x for x in generic_members if x.startswith("test")] + + # Creates device-specific test cases + for base in get_desired_device_type_test_bases( + except_for, only_for, include_lazy, allow_mps, allow_xpu + ): + class_name = generic_test_class.__name__ + base.device_type.upper() + + # type set to Any and suppressed due to unsupported runtime class: + # https://github.com/python/mypy/wiki/Unsupported-Python-Features + device_type_test_class: Any = type(class_name, (base, generic_test_class), {}) + + # Arrange for setUpClass and tearDownClass methods defined both in the test template + # class and in the generic base to be called. This allows device-parameterized test + # classes to support setup and teardown. + # NB: This should be done before instantiate_test() is called as that invokes setup. + @classmethod + def _setUpClass(cls): + # This should always be called, whether or not the test class invokes + # super().setUpClass(), to set the primary device. + base.setUpClass() + # We want to call the @classmethod defined in the generic base, but pass + # it the device-specific class object (cls), hence the __func__ call. + generic_test_class.setUpClass.__func__(cls) + + @classmethod + def _tearDownClass(cls): + # We want to call the @classmethod defined in the generic base, but pass + # it the device-specific class object (cls), hence the __func__ call. + generic_test_class.tearDownClass.__func__(cls) + base.tearDownClass() + + device_type_test_class.setUpClass = _setUpClass + device_type_test_class.tearDownClass = _tearDownClass + + for name in generic_members: + if name in generic_tests: # Instantiates test member + test = getattr(generic_test_class, name) + # XLA-compat shim (XLA's instantiate_test takes doesn't take generic_cls) + sig = inspect.signature(device_type_test_class.instantiate_test) + if len(sig.parameters) == 3: + # Instantiates the device-specific tests + device_type_test_class.instantiate_test( + name, copy.deepcopy(test), generic_cls=generic_test_class + ) + else: + device_type_test_class.instantiate_test(name, copy.deepcopy(test)) + # Ports non-test member. Setup / teardown have already been handled above + elif name not in device_type_test_class.__dict__: + nontest = getattr(generic_test_class, name) + setattr(device_type_test_class, name, nontest) + + # Mimics defining the instantiated class in the caller's file + # by setting its module to the given class's and adding + # the module to the given scope. + # This lets the instantiated class be discovered by unittest. + device_type_test_class.__module__ = generic_test_class.__module__ + scope[class_name] = device_type_test_class + + # Delete the generic form of the test functions (e.g. TestFoo.test_bar()) so they're + # not discoverable. This mutates the original class (TestFoo), which was removed from + # scope above. At this point, device-specific tests (e.g. TestFooCUDA.test_bar_cuda) + # have already been created and the generic forms are no longer needed. + for name in generic_tests: + delattr(generic_test_class, name) + + +# Category of dtypes to run an OpInfo-based test for +# Example use: @ops(dtype=OpDTypes.supported) +# +# There are 7 categories: +# - supported: Every dtype supported by the operator. Use for exhaustive +# testing of all dtypes. +# - unsupported: Run tests on dtypes not supported by the operator. e.g. for +# testing the operator raises an error and doesn't crash. +# - supported_backward: Every dtype supported by the operator's backward pass. +# - unsupported_backward: Run tests on dtypes not supported by the operator's backward pass. +# - any_one: Runs a test for one dtype the operator supports. Prioritizes dtypes the +# operator supports in both forward and backward. +# - none: Useful for tests that are not dtype-specific. No dtype will be passed to the test +# when this is selected. +# - any_common_cpu_cuda_one: Pick a dtype that supports both CPU and CUDA. +class OpDTypes(Enum): + supported = 0 # Test all supported dtypes (default) + unsupported = 1 # Test only unsupported dtypes + supported_backward = 2 # Test all supported backward dtypes + unsupported_backward = 3 # Test only unsupported backward dtypes + any_one = 4 # Test precisely one supported dtype + none = 5 # Instantiate no dtype variants (no dtype kwarg needed) + any_common_cpu_cuda_one = ( + 6 # Test precisely one supported dtype that is common to both cuda and cpu + ) + + +# Arbitrary order +ANY_DTYPE_ORDER = ( + core.float32, + core.float64, + core.complex64, + core.complex128, + core.float16, + core.bfloat16, + core.long, + core.int32, + core.int16, + core.int8, + core.uint8, + core.bool, + core.float8_e4m3fn, + core.float8_e5m2, +) + + +def _serialize_sample(sample_input): + # NB: For OpInfos, SampleInput.summary() prints in a cleaner way. + if getattr(sample_input, "summary", None) is not None: + return sample_input.summary() + return str(sample_input) + + +# Decorator that defines the OpInfos a test template should be instantiated for. +# +# Example usage: +# +# @ops(unary_ufuncs) +# def test_numerics(self, device, dtype, op): +# +# +# This will instantiate variants of test_numerics for each given OpInfo, +# on each device the OpInfo's operator supports, and for every dtype supported by +# that operator. There are a few caveats to the dtype rule, explained below. +# +# The @ops decorator can accept two +# additional arguments, "dtypes" and "allowed_dtypes". If "dtypes" is specified +# then the test variants are instantiated for those dtypes, regardless of +# what the operator supports. If given "allowed_dtypes" then test variants +# are instantiated only for the intersection of allowed_dtypes and the dtypes +# they would otherwise be instantiated with. That is, allowed_dtypes composes +# with the options listed above and below. +# +# The "dtypes" argument can also accept additional values (see OpDTypes above): +# OpDTypes.supported - the test is instantiated for all dtypes the operator +# supports +# OpDTypes.unsupported - the test is instantiated for all dtypes the operator +# doesn't support +# OpDTypes.supported_backward - the test is instantiated for all dtypes the +# operator's gradient formula supports +# OpDTypes.unsupported_backward - the test is instantiated for all dtypes the +# operator's gradient formula doesn't support +# OpDTypes.any_one - the test is instantiated for one dtype the +# operator supports. The dtype supports forward and backward if possible. +# OpDTypes.none - the test is instantiated without any dtype. The test signature +# should not include a dtype kwarg in this case. +# OpDTypes.any_common_cpu_cuda_one - the test is instantiated for a dtype +# that supports both CPU and CUDA. +# +# These options allow tests to have considerable control over the dtypes +# they're instantiated for. + + +class ops(_TestParametrizer): + def __init__( + self, + op_list, + *, + dtypes: Union[OpDTypes, Sequence[core.dtype]] = OpDTypes.supported, + allowed_dtypes: Optional[Sequence[core.dtype]] = None, + skip_if_dynamo=True, + ): + self.op_list = list(op_list) + self.opinfo_dtypes = dtypes + self.allowed_dtypes = ( + set(allowed_dtypes) if allowed_dtypes is not None else None + ) + self.skip_if_dynamo = skip_if_dynamo + + def _parametrize_test(self, test, generic_cls, device_cls): + """Parameterizes the given test function across each op and its associated dtypes.""" + if device_cls is None: + raise RuntimeError( + "The @ops decorator is only intended to be used in a device-specific " + "context; use it with instantiate_device_type_tests() instead of " + "instantiate_parametrized_tests()" + ) + + op = check_exhausted_iterator = object() + for op in self.op_list: + # Determine the set of dtypes to use. + dtypes: Union[set[core.dtype], set[None]] + if isinstance(self.opinfo_dtypes, Sequence): + dtypes = set(self.opinfo_dtypes) + elif self.opinfo_dtypes == OpDTypes.unsupported_backward: + dtypes = set(get_all_dtypes()).difference( + op.supported_backward_dtypes(device_cls.device_type) + ) + elif self.opinfo_dtypes == OpDTypes.supported_backward: + dtypes = op.supported_backward_dtypes(device_cls.device_type) + elif self.opinfo_dtypes == OpDTypes.unsupported: + dtypes = set(get_all_dtypes()).difference( + op.supported_dtypes(device_cls.device_type) + ) + elif self.opinfo_dtypes == OpDTypes.supported: + dtypes = set(op.supported_dtypes(device_cls.device_type)) + elif self.opinfo_dtypes == OpDTypes.any_one: + # Tries to pick a dtype that supports both forward or backward + supported = op.supported_dtypes(device_cls.device_type) + supported_backward = op.supported_backward_dtypes( + device_cls.device_type + ) + supported_both = supported.intersection(supported_backward) + dtype_set = supported_both if len(supported_both) > 0 else supported + for dtype in ANY_DTYPE_ORDER: + if dtype in dtype_set: + dtypes = {dtype} + break + else: + dtypes = {} + elif self.opinfo_dtypes == OpDTypes.any_common_cpu_cuda_one: + # Tries to pick a dtype that supports both CPU and CUDA + supported = set(op.dtypes).intersection(op.dtypesIfCUDA) + if supported: + dtypes = { + next(dtype for dtype in ANY_DTYPE_ORDER if dtype in supported) + } + else: + dtypes = {} + + elif self.opinfo_dtypes == OpDTypes.none: + dtypes = {None} + else: + raise RuntimeError(f"Unknown OpDType: {self.opinfo_dtypes}") + + if self.allowed_dtypes is not None: + dtypes = dtypes.intersection(self.allowed_dtypes) + + # Construct the test name; device / dtype parts are handled outside. + # See [Note: device and dtype suffix placement] + test_name = op.formatted_name + + # Filter sample skips / xfails to only those that apply to the OpInfo. + # These are defined on the test function via decorators. + sample_skips_and_xfails = getattr(test, "sample_skips_and_xfails", None) + if sample_skips_and_xfails is not None: + sample_skips_and_xfails = [ + rule + for rule in sample_skips_and_xfails + if rule.op_match_fn(device_cls.device_type, op) + ] + + for dtype in dtypes: + # Construct parameter kwargs to pass to the test. + param_kwargs = {"op": op} + _update_param_kwargs(param_kwargs, "dtype", dtype) + + # NOTE: test_wrapper exists because we don't want to apply + # op-specific decorators to the original test. + # Test-specific decorators are applied to the original test, + # however. + try: + + @wraps(test) + def test_wrapper(*args, **kwargs): + try: + return test(*args, **kwargs) + except unittest.SkipTest as e: + raise e + except Exception as e: + tracked_input = get_tracked_input() + if PRINT_REPRO_ON_FAILURE and tracked_input is not None: + e_tracked = Exception( # noqa: TRY002 + f"Caused by {tracked_input.type_desc} " + f"at index {tracked_input.index}: " + f"{_serialize_sample(tracked_input.val)}" + ) + e_tracked._tracked_input = tracked_input # type: ignore[attr] + raise e_tracked from e + raise e + finally: + clear_tracked_input() + + if self.skip_if_dynamo and not TEST_WITH_TORCHINDUCTOR: + test_wrapper = skipIfTorchDynamo( + "Policy: we don't run OpInfo tests w/ Dynamo" + )(test_wrapper) + + # Initialize info for the last input seen. This is useful for tracking + # down which inputs caused a test failure. Note that TrackedInputIter is + # responsible for managing this. + test.tracked_input = None + + decorator_fn = partial( + op.get_decorators, + generic_cls.__name__, + test.__name__, + device_cls.device_type, + dtype, + ) + + if sample_skips_and_xfails is not None: + test_wrapper.sample_skips_and_xfails = sample_skips_and_xfails + + yield (test_wrapper, test_name, param_kwargs, decorator_fn) + except Exception as ex: + # Provides an error message for debugging before rethrowing the exception + print(f"Failed to instantiate {test_name} for op {op.name}!") + raise ex + if op is check_exhausted_iterator: + raise ValueError( + "An empty op_list was passed to @ops. " + "Note that this may result from reuse of a generator." + ) + + +# Decorator that skips a test if the given condition is true. +# Notes: +# (1) Skip conditions stack. +# (2) Skip conditions can be bools or strings. If a string the +# test base must have defined the corresponding attribute to be False +# for the test to run. If you want to use a string argument you should +# probably define a new decorator instead (see below). +# (3) Prefer the existing decorators to defining the 'device_type' kwarg. +class skipIf: + def __init__(self, dep, reason, device_type=None): + self.dep = dep + self.reason = reason + self.device_type = device_type + + def __call__(self, fn): + @wraps(fn) + def dep_fn(slf, *args, **kwargs): + if ( + self.device_type is None + or self.device_type == slf.device_type + or ( + isinstance(self.device_type, Iterable) + and slf.device_type in self.device_type + ) + ): + if (isinstance(self.dep, str) and getattr(slf, self.dep, True)) or ( + isinstance(self.dep, bool) and self.dep + ): + raise unittest.SkipTest(self.reason) + + return fn(slf, *args, **kwargs) + + return dep_fn + + +# Skips a test on CPU if the condition is true. +class skipCPUIf(skipIf): + def __init__(self, dep, reason): + super().__init__(dep, reason, device_type="cpu") + + +# Skips a test on CUDA if the condition is true. +class skipCUDAIf(skipIf): + def __init__(self, dep, reason): + super().__init__(dep, reason, device_type="cuda") + + +# Skips a test on XPU if the condition is true. +class skipXPUIf(skipIf): + def __init__(self, dep, reason): + super().__init__(dep, reason, device_type="xpu") + + +# Skips a test on XPU or CUDA if the condition is true. +class skipGPUIf(skipIf): + def __init__(self, dep, reason): + super().__init__(dep, reason, device_type=GPU_TYPES) + + +# Skips a test on Lazy if the condition is true. +class skipLazyIf(skipIf): + def __init__(self, dep, reason): + super().__init__(dep, reason, device_type="lazy") + + +# Skips a test on Meta if the condition is true. +class skipMetaIf(skipIf): + def __init__(self, dep, reason): + super().__init__(dep, reason, device_type="meta") + + +# Skips a test on MPS if the condition is true. +class skipMPSIf(skipIf): + def __init__(self, dep, reason): + super().__init__(dep, reason, device_type="mps") + + +class skipHPUIf(skipIf): + def __init__(self, dep, reason): + super().__init__(dep, reason, device_type="hpu") + + +# Skips a test on XLA if the condition is true. +class skipXLAIf(skipIf): + def __init__(self, dep, reason): + super().__init__(dep, reason, device_type="xla") + + +class skipPRIVATEUSE1If(skipIf): + def __init__(self, dep, reason): + device_type = core._C._get_privateuse1_backend_name() + super().__init__(dep, reason, device_type=device_type) + + +def _has_sufficient_memory(device, size): + if core.device(device).type == "cuda": + if not core.cuda.is_available(): + return False + gc.collect() + core.cuda.empty_cache() + # core.cuda.mem_get_info, aka cudaMemGetInfo, returns a tuple of (free memory, total memory) of a GPU + if device == "cuda": + device = "cuda:0" + return ( + core.cuda.memory.mem_get_info(device)[0] + * core.cuda.memory.get_per_process_memory_fraction(device) + ) >= size + + if device == "xla": + raise unittest.SkipTest("TODO: Memory availability checks for XLA?") + + if device == "xpu": + raise unittest.SkipTest("TODO: Memory availability checks for Intel GPU?") + + if device != "cpu": + raise unittest.SkipTest("Unknown device type") + + # CPU + if not HAS_PSUTIL: + raise unittest.SkipTest("Need psutil to determine if memory is sufficient") + + # The sanitizers have significant memory overheads + if TEST_WITH_ASAN or TEST_WITH_TSAN or TEST_WITH_UBSAN: + effective_size = size * 10 + else: + effective_size = size + + if psutil.virtual_memory().available < effective_size: + gc.collect() + return psutil.virtual_memory().available >= effective_size + + +def largeTensorTest(size, device=None, inductor=TEST_WITH_TORCHINDUCTOR): + """Skip test if the device has insufficient memory to run the test + + size may be a number of bytes, a string of the form "N GB", or a callable + + If the test is a device generic test, available memory on the primary device will be checked. + It can also be overridden by the optional `device=` argument. + In other tests, the `device=` argument needs to be specified. + """ + if isinstance(size, str): + assert size.endswith(("GB", "gb")), "only bytes or GB supported" + size = 1024**3 * int(size[:-2]) + + def inner(fn): + @wraps(fn) + def dep_fn(self, *args, **kwargs): + size_bytes: int = size(self, *args, **kwargs) if callable(size) else size + _device = device + if _device is None: + if hasattr(self, "get_primary_device"): + _device = self.get_primary_device() + else: + _device = self.device + + # If this is running with GPU cpp_wrapper, the autotuning step will generate + # an additional array of the same size as the input. + if inductor and core._inductor.config.cpp_wrapper and _device != "cpu": + size_bytes *= 2 + + if not _has_sufficient_memory(_device, size_bytes): + raise unittest.SkipTest(f"Insufficient {_device} memory") + + return fn(self, *args, **kwargs) + + return dep_fn + + return inner + + +class expectedFailure: + def __init__(self, device_type): + self.device_type = device_type + + def __call__(self, fn): + @wraps(fn) + def efail_fn(slf, *args, **kwargs): + if ( + not hasattr(slf, "device_type") + and hasattr(slf, "device") + and isinstance(slf.device, str) + ): + target_device_type = slf.device + else: + target_device_type = slf.device_type + + if self.device_type is None or self.device_type == target_device_type: + try: + fn(slf, *args, **kwargs) + except Exception: + return + else: + slf.fail("expected test to fail, but it passed") + + return fn(slf, *args, **kwargs) + + return efail_fn + + +class onlyOn: + def __init__(self, device_type): + self.device_type = device_type + + def __call__(self, fn): + @wraps(fn) + def only_fn(slf, *args, **kwargs): + if self.device_type != slf.device_type: + reason = f"Only runs on {self.device_type}" + raise unittest.SkipTest(reason) + + return fn(slf, *args, **kwargs) + + return only_fn + + +# Decorator that provides all available devices of the device type to the test +# as a list of strings instead of providing a single device string. +# Skips the test if the number of available devices of the variant's device +# type is less than the 'num_required_devices' arg. +class deviceCountAtLeast: + def __init__(self, num_required_devices): + self.num_required_devices = num_required_devices + + def __call__(self, fn): + assert not hasattr(fn, "num_required_devices"), ( + f"deviceCountAtLeast redefinition for {fn.__name__}" + ) + fn.num_required_devices = self.num_required_devices + + @wraps(fn) + def multi_fn(slf, devices, *args, **kwargs): + if len(devices) < self.num_required_devices: + reason = f"fewer than {self.num_required_devices} devices detected" + raise unittest.SkipTest(reason) + + return fn(slf, devices, *args, **kwargs) + + return multi_fn + + +# Only runs the test on the native device type (currently CPU, CUDA, Meta and PRIVATEUSE1) +def onlyNativeDeviceTypes(fn: Callable[_P, _T]) -> Callable[_P, _T]: + @wraps(fn) + def only_fn(self, *args: _P.args, **kwargs: _P.kwargs) -> _T: + if self.device_type not in NATIVE_DEVICES: + reason = f"onlyNativeDeviceTypes: doesn't run on {self.device_type}" + raise unittest.SkipTest(reason) + + return fn(self, *args, **kwargs) + + return only_fn + + +# Only runs the test on the native device types and devices specified in the devices list +def onlyNativeDeviceTypesAnd(devices=None): + def decorator(fn): + @wraps(fn) + def only_fn(self, *args, **kwargs): + if ( + self.device_type not in NATIVE_DEVICES + and self.device_type not in devices + ): + reason = f"onlyNativeDeviceTypesAnd {devices} : doesn't run on {self.device_type}" + raise unittest.SkipTest(reason) + + return fn(self, *args, **kwargs) + + return only_fn + + return decorator + + +# Specifies per-dtype precision overrides. +# Ex. +# +# @precisionOverride({core.half : 1e-2, core.float : 1e-4}) +# @dtypes(core.half, core.float, core.double) +# def test_X(self, device, dtype): +# ... +# +# When the test is instantiated its class's precision will be set to the +# corresponding override, if it exists. +# self.precision can be accessed directly, and it also controls the behavior of +# functions like self.assertEqual(). +# +# Note that self.precision is a scalar value, so if you require multiple +# precisions (or are working with multiple dtypes) they should be specified +# explicitly and computed using self.precision (e.g. +# self.precision *2, max(1, self.precision)). +class precisionOverride: + def __init__(self, d): + assert isinstance(d, dict), ( + "precisionOverride not given a dtype : precision dict!" + ) + for dtype in d.keys(): + assert isinstance(dtype, core.dtype), ( + f"precisionOverride given unknown dtype {dtype}" + ) + + self.d = d + + def __call__(self, fn): + fn.precision_overrides = self.d + return fn + + +# Specifies per-dtype tolerance overrides tol(atol, rtol). It has priority over +# precisionOverride. +# Ex. +# +# @toleranceOverride({core.float : tol(atol=1e-2, rtol=1e-3}, +# core.double : tol{atol=1e-4, rtol = 0}) +# @dtypes(core.half, core.float, core.double) +# def test_X(self, device, dtype): +# ... +# +# When the test is instantiated its class's tolerance will be set to the +# corresponding override, if it exists. +# self.rtol and self.precision can be accessed directly, and they also control +# the behavior of functions like self.assertEqual(). +# +# The above example sets atol = 1e-2 and rtol = 1e-3 for core.float and +# atol = 1e-4 and rtol = 0 for core.double. +tol = namedtuple("tol", ["atol", "rtol"]) + + +class toleranceOverride: + def __init__(self, d): + assert isinstance(d, dict), "toleranceOverride not given a dtype : tol dict!" + for dtype, prec in d.items(): + assert isinstance(dtype, core.dtype), ( + f"toleranceOverride given unknown dtype {dtype}" + ) + assert isinstance(prec, tol), ( + "toleranceOverride not given a dtype : tol dict!" + ) + + self.d = d + + def __call__(self, fn): + fn.tolerance_overrides = self.d + return fn + + +# Decorator that instantiates a variant of the test for each given dtype. +# Notes: +# (1) Tests that accept the dtype argument MUST use this decorator. +# (2) Can be overridden for CPU or CUDA, respectively, using dtypesIfCPU +# or dtypesIfCUDA. +# (3) Can accept an iterable of dtypes or an iterable of tuples +# of dtypes. +# Examples: +# @dtypes(core.float32, core.float64) +# @dtypes((core.long, core.float32), (core.int, core.float64)) +class dtypes: + def __init__(self, *args, device_type="all"): + if len(args) > 0 and isinstance(args[0], (list, tuple)): + for arg in args: + assert isinstance(arg, (list, tuple)), ( + "When one dtype variant is a tuple or list, " + "all dtype variants must be. " + f"Received non-list non-tuple dtype {str(arg)}" + ) + assert all(isinstance(dtype, core.dtype) for dtype in arg), ( + f"Unknown dtype in {str(arg)}" + ) + else: + assert all(isinstance(arg, core.dtype) for arg in args), ( + f"Unknown dtype in {str(args)}" + ) + + self.args = args + self.device_type = device_type + + def __call__(self, fn): + d = getattr(fn, "dtypes", {}) + assert self.device_type not in d, f"dtypes redefinition for {self.device_type}" + d[self.device_type] = self.args + fn.dtypes = d + return fn + + +# Overrides specified dtypes on the CPU. +class dtypesIfCPU(dtypes): + def __init__(self, *args): + super().__init__(*args, device_type="cpu") + + +# Overrides specified dtypes on CUDA. +class dtypesIfCUDA(dtypes): + def __init__(self, *args): + super().__init__(*args, device_type="cuda") + + +class dtypesIfMPS(dtypes): + def __init__(self, *args): + super().__init__(*args, device_type="mps") + + +class dtypesIfHPU(dtypes): + def __init__(self, *args): + super().__init__(*args, device_type="hpu") + + +class dtypesIfPRIVATEUSE1(dtypes): + def __init__(self, *args): + super().__init__(*args, device_type=core._C._get_privateuse1_backend_name()) + + +def onlyCPU(fn): + return onlyOn("cpu")(fn) + + +def onlyCUDA(fn): + return onlyOn("cuda")(fn) + + +def onlyMPS(fn): + return onlyOn("mps")(fn) + + +def onlyXPU(fn): + return onlyOn("xpu")(fn) + + +def onlyHPU(fn): + return onlyOn("hpu")(fn) + + +def onlyPRIVATEUSE1(fn): + device_type = core._C._get_privateuse1_backend_name() + device_mod = getattr(core, device_type, None) + if device_mod is None: + reason = f"Skip as core has no module of {device_type}" + return unittest.skip(reason)(fn) + return onlyOn(device_type)(fn) + + +def onlyCUDAAndPRIVATEUSE1(fn): + @wraps(fn) + def only_fn(self, *args, **kwargs): + if self.device_type not in ("cuda", core._C._get_privateuse1_backend_name()): + reason = f"onlyCUDAAndPRIVATEUSE1: doesn't run on {self.device_type}" + raise unittest.SkipTest(reason) + + return fn(self, *args, **kwargs) + + return only_fn + + +def disablecuDNN(fn): + @wraps(fn) + def disable_cudnn(self, *args, **kwargs): + if self.device_type == "cuda" and self.has_cudnn(): + with core.backends.cudnn.flags(enabled=False): + return fn(self, *args, **kwargs) + return fn(self, *args, **kwargs) + + return disable_cudnn + + +def disableMkldnn(fn): + @wraps(fn) + def disable_mkldnn(self, *args, **kwargs): + if core.backends.mkldnn.is_available(): + with core.backends.mkldnn.flags(enabled=False): + return fn(self, *args, **kwargs) + return fn(self, *args, **kwargs) + + return disable_mkldnn + + +def expectedFailureCPU(fn): + return expectedFailure("cpu")(fn) + + +def expectedFailureCUDA(fn): + return expectedFailure("cuda")(fn) + + +def expectedFailureXPU(fn): + return expectedFailure("xpu")(fn) + + +def expectedFailureMeta(fn): + return skipIfTorchDynamo()(expectedFailure("meta")(fn)) + + +def expectedFailureXLA(fn): + return expectedFailure("xla")(fn) + + +def expectedFailureHPU(fn): + return expectedFailure("hpu")(fn) + + +def expectedFailureMPS(fn): + return expectedFailure("mps")(fn) + + +def expectedFailureMPSPre15(fn): + import platform + + version = float(".".join(platform.mac_ver()[0].split(".")[:2]) or -1) + if not version or version < 1.0: # cpu or other unsupported device + return fn + if version < 15.0: + return expectedFailure("mps")(fn) + return fn + + +def expectedFailureMPSPre14(fn): + import platform + + version = float(".".join(platform.mac_ver()[0].split(".")[:2]) or -1) + if not version or version < 1.0: # cpu or other unsupported device + return fn + if version < 14.0: + return expectedFailure("mps")(fn) + return fn + + +# Skips a test on CPU if LAPACK is not available. +def skipCPUIfNoLapack(fn): + return skipCPUIf(not core._C.has_lapack, "PyTorch compiled without Lapack")(fn) + + +# Skips a test on CPU if FFT is not available. +def skipCPUIfNoFFT(fn): + return skipCPUIf(not core._C.has_spectral, "PyTorch is built without FFT support")( + fn + ) + + +# Skips a test on CPU if MKL is not available. +def skipCPUIfNoMkl(fn): + return skipCPUIf(not TEST_MKL, "PyTorch is built without MKL support")(fn) + + +# Skips a test on CPU if MKL Sparse is not available (it's not linked on Windows). +def skipCPUIfNoMklSparse(fn): + return skipCPUIf( + IS_WINDOWS or not TEST_MKL, "PyTorch is built without MKL support" + )(fn) + + +# Skips a test on CPU if mkldnn is not available. +def skipCPUIfNoMkldnn(fn): + return skipCPUIf( + not core.backends.mkldnn.is_available(), + "PyTorch is built without mkldnn support", + )(fn) + + +# Skips a test on CUDA if MAGMA is not available. +def skipCUDAIfNoMagma(fn): + return skipCUDAIf("no_magma", "no MAGMA library detected")( + skipCUDANonDefaultStreamIf(True)(fn) + ) + + +def has_cusolver(): + return not TEST_WITH_ROCM + + +def has_hipsolver(): + rocm_version = _get_core_rocm_version() + # hipSOLVER is disabled on ROCM < 5.3 + return rocm_version >= (5, 3) + + +# Skips a test on CUDA/ROCM if cuSOLVER/hipSOLVER is not available +def skipCUDAIfNoCusolver(fn): + return skipCUDAIf( + not has_cusolver() and not has_hipsolver(), "cuSOLVER not available" + )(fn) + + +# Skips a test if both cuSOLVER and MAGMA are not available +def skipCUDAIfNoMagmaAndNoCusolver(fn): + if has_cusolver(): + return fn + else: + # cuSolver is disabled on cuda < 10.1.243, tests depend on MAGMA + return skipCUDAIfNoMagma(fn) + + +# Skips a test if both cuSOLVER/hipSOLVER and MAGMA are not available +def skipCUDAIfNoMagmaAndNoLinalgsolver(fn): + if has_cusolver() or has_hipsolver(): + return fn + else: + # cuSolver is disabled on cuda < 10.1.243, tests depend on MAGMA + return skipCUDAIfNoMagma(fn) + + +# Skips a test on CUDA when using ROCm. +def skipCUDAIfRocm(func=None, *, msg="test doesn't currently work on the ROCm stack"): + def dec_fn(fn): + reason = f"skipCUDAIfRocm: {msg}" + return skipCUDAIf(TEST_WITH_ROCM, reason=reason)(fn) + + if func: + return dec_fn(func) + return dec_fn + + +# Skips a test on CUDA when not using ROCm. +def skipCUDAIfNotRocm(fn): + return skipCUDAIf( + not TEST_WITH_ROCM, "test doesn't currently work on the CUDA stack" + )(fn) + + +# Skips a test on CUDA if ROCm is unavailable or its version is lower than requested. +def skipCUDAIfRocmVersionLessThan(version=None): + def dec_fn(fn): + @wraps(fn) + def wrap_fn(self, *args, **kwargs): + if self.device_type == "cuda": + if not TEST_WITH_ROCM: + reason = "ROCm not available" + raise unittest.SkipTest(reason) + rocm_version_tuple = _get_core_rocm_version() + if ( + rocm_version_tuple is None + or version is None + or rocm_version_tuple < tuple(version) + ): + reason = ( + f"ROCm {rocm_version_tuple} is available but {version} required" + ) + raise unittest.SkipTest(reason) + + return fn(self, *args, **kwargs) + + return wrap_fn + + return dec_fn + + +# Skips a test on CUDA when using ROCm. +def skipCUDAIfNotMiopenSuggestNHWC(fn): + return skipCUDAIf( + not TEST_WITH_MIOPEN_SUGGEST_NHWC, + "test doesn't currently work without MIOpen NHWC activation", + )(fn) + + +# Skips a test for specified CUDA versions, given in the form of a list of [major, minor]s. +def skipCUDAVersionIn(versions: Optional[list[tuple[int, int]]] = None): + def dec_fn(fn): + @wraps(fn) + def wrap_fn(self, *args, **kwargs): + version = _get_core_cuda_version() + if version == (0, 0): # cpu or rocm + return fn(self, *args, **kwargs) + if version in (versions or []): + reason = f"test skipped for CUDA version {version}" + raise unittest.SkipTest(reason) + return fn(self, *args, **kwargs) + + return wrap_fn + + return dec_fn + + +# Skips a test for CUDA versions less than specified, given in the form of [major, minor]. +def skipCUDAIfVersionLessThan(versions: Optional[tuple[int, int]] = None): + def dec_fn(fn): + @wraps(fn) + def wrap_fn(self, *args, **kwargs): + version = _get_core_cuda_version() + if version == (0, 0): # cpu or rocm + return fn(self, *args, **kwargs) + if version < versions: + reason = f"test skipped for CUDA versions < {version}" + raise unittest.SkipTest(reason) + return fn(self, *args, **kwargs) + + return wrap_fn + + return dec_fn + + +# Skips a test on CUDA if cuDNN is unavailable or its version is lower than requested. +def skipCUDAIfCudnnVersionLessThan(version=0): + def dec_fn(fn): + @wraps(fn) + def wrap_fn(self, *args, **kwargs): + if self.device_type == "cuda": + if self.no_cudnn: + reason = "cuDNN not available" + raise unittest.SkipTest(reason) + if self.cudnn_version is None or self.cudnn_version < version: + reason = f"cuDNN version {self.cudnn_version} is available but {version} required" + raise unittest.SkipTest(reason) + + return fn(self, *args, **kwargs) + + return wrap_fn + + return dec_fn + + +# Skips a test on CUDA if cuSparse generic API is not available +def skipCUDAIfNoCusparseGeneric(fn): + return skipCUDAIf(not TEST_CUSPARSE_GENERIC, "cuSparse Generic API not available")( + fn + ) + + +def skipCUDAIfNoHipsparseGeneric(fn): + return skipCUDAIf( + not TEST_HIPSPARSE_GENERIC, "hipSparse Generic API not available" + )(fn) + + +def skipCUDAIfNoSparseGeneric(fn): + return skipCUDAIf( + not (TEST_CUSPARSE_GENERIC or TEST_HIPSPARSE_GENERIC), + "Sparse Generic API not available", + )(fn) + + +def skipCUDAIfNoCudnn(fn): + return skipCUDAIfCudnnVersionLessThan(0)(fn) + + +def skipCUDAIfMiopen(fn): + return skipCUDAIf(core.version.hip is not None, "Marked as skipped for MIOpen")(fn) + + +def skipCUDAIfNoMiopen(fn): + return skipCUDAIf(core.version.hip is None, "MIOpen is not available")( + skipCUDAIfNoCudnn(fn) + ) + + +def skipLazy(fn): + return skipLazyIf(True, "test doesn't work with lazy tensors")(fn) + + +def skipMeta(fn): + return skipMetaIf(True, "test doesn't work with meta tensors")(fn) + + +def skipXLA(fn): + return skipXLAIf(True, "Marked as skipped for XLA")(fn) + + +def skipMPS(fn): + return skipMPSIf(True, "test doesn't work on MPS backend")(fn) + + +def skipHPU(fn): + return skipHPUIf(True, "test doesn't work on HPU backend")(fn) + + +def skipPRIVATEUSE1(fn): + return skipPRIVATEUSE1If(True, "test doesn't work on privateuse1 backend")(fn) + + +# TODO: the "all" in the name isn't true anymore for quite some time as we have also have for example XLA and MPS now. +# This should probably enumerate all available device type test base classes. +def get_all_device_types() -> list[str]: + return ["cpu"] if not core.cuda.is_available() else ["cpu", "cuda"] + + +# skip since currently flex attention requires at least `avx2` support on CPU. +IS_FLEX_ATTENTION_CPU_PLATFORM_SUPPORTED = ( + not core.xpu.is_available() + and not core.cuda.is_available() + and not IS_MACOS + # and core.cpu._is_avx2_supported() + and os.getenv("ATEN_CPU_CAPABILITY") != "default" +) +flex_attention_supported_platform = unittest.skipUnless( + IS_FLEX_ATTENTION_CPU_PLATFORM_SUPPORTED + or ( + core.cuda.is_available() + and core.utils._triton.has_triton() + and core.cuda.get_device_capability() >= (8, 0) + ), + "Requires CUDA and Triton, or CPU with avx2 and later", +) +# if core.version.hip and "gfx94" in core.cuda.get_device_properties(0).gcnArchName: +# e4m3_type = core.float8_e4m3fnuz +# e5m2_type = core.float8_e5m2fnuz +# E4M3_MAX_POS = core.finfo(core.float8_e4m3fnuz).max +# E5M2_MAX_POS = core.finfo(core.float8_e5m2fnuz).max +# else: +# e4m3_type = core.float8_e4m3fn +# e5m2_type = core.float8_e5m2 +# E4M3_MAX_POS = core.finfo(core.float8_e4m3fn).max +# E5M2_MAX_POS = core.finfo(core.float8_e5m2).max \ No newline at end of file diff --git a/mindnlp/core/testing/_internal/common_dtype.py b/mindnlp/core/testing/_internal/common_dtype.py new file mode 100644 index 000000000..883d7757a --- /dev/null +++ b/mindnlp/core/testing/_internal/common_dtype.py @@ -0,0 +1,226 @@ +# mypy: ignore-errors + + +from mindnlp import core + +# Functions and classes for describing the dtypes a function supports +# NOTE: these helpers should correspond to PyTorch's C++ dispatch macros + + +# Verifies each given dtype is a core.dtype +def _validate_dtypes(*dtypes): + for dtype in dtypes: + assert isinstance(dtype, core.dtype) + return dtypes + + +# class for tuples corresponding to a PyTorch dispatch macro +class _dispatch_dtypes(tuple): + __slots__ = () + + def __add__(self, other): + assert isinstance(other, tuple) + return _dispatch_dtypes(tuple.__add__(self, other)) + + +_empty_types = _dispatch_dtypes(()) + + +def empty_types(): + return _empty_types + + +_floating_types = _dispatch_dtypes((core.float32, core.float64)) + + +def floating_types(): + return _floating_types + + +_floating_types_and_half = _floating_types + (core.half,) + + +def floating_types_and_half(): + return _floating_types_and_half + + +def floating_types_and(*dtypes): + return _floating_types + _validate_dtypes(*dtypes) + + +_floating_and_complex_types = _floating_types + (core.cfloat, core.cdouble) + + +def floating_and_complex_types(): + return _floating_and_complex_types + + +def floating_and_complex_types_and(*dtypes): + return _floating_and_complex_types + _validate_dtypes(*dtypes) + + +_double_types = _dispatch_dtypes((core.float64, core.complex128)) + + +def double_types(): + return _double_types + + +# NB: Does not contain uint16/uint32/uint64 for BC reasons +_integral_types = _dispatch_dtypes( + (core.uint8, core.int8, core.int16, core.int32, core.int64) +) + + +def integral_types(): + return _integral_types + + +def integral_types_and(*dtypes): + return _integral_types + _validate_dtypes(*dtypes) + + +_all_types = _floating_types + _integral_types + + +def all_types(): + return _all_types + + +def all_types_and(*dtypes): + return _all_types + _validate_dtypes(*dtypes) + + +_complex_types = _dispatch_dtypes((core.cfloat, core.cdouble)) + + +def complex_types(): + return _complex_types + + +def complex_types_and(*dtypes): + return _complex_types + _validate_dtypes(*dtypes) + + +_all_types_and_complex = _all_types + _complex_types + + +def all_types_and_complex(): + return _all_types_and_complex + + +def all_types_and_complex_and(*dtypes): + return _all_types_and_complex + _validate_dtypes(*dtypes) + + +_all_types_and_half = _all_types + (core.half,) + + +def all_types_and_half(): + return _all_types_and_half + + +_all_mps_types = ( + _dispatch_dtypes({core.float, core.half, core.bfloat16}) + _integral_types +) + + +def all_mps_types(): + return _all_mps_types + + +def all_mps_types_and(*dtypes): + return _all_mps_types + _validate_dtypes(*dtypes) + + +_float8_types = _dispatch_dtypes( + ( + core.float8_e4m3fn, + core.float8_e4m3fnuz, + core.float8_e5m2, + core.float8_e5m2fnuz, + ) +) + + +def float8_types(): + return _float8_types + + +def float8_types_and(*dtypes): + return _float8_types + _validate_dtypes(*dtypes) + + +def all_types_complex_float8_and(*dtypes): + return _all_types + _complex_types + _float8_types + _validate_dtypes(*dtypes) + + +def custom_types(*dtypes): + """Create a list of arbitrary dtypes""" + return _empty_types + _validate_dtypes(*dtypes) + + +# The functions below are used for convenience in our test suite and thus have no corresponding C++ dispatch macro + + +# See AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS. +def get_all_dtypes( + include_half=True, + include_bfloat16=True, + include_bool=True, + include_complex=True, + include_complex32=False, + include_qint=False, +) -> list[core.dtype]: + dtypes = get_all_int_dtypes() + get_all_fp_dtypes( + include_half=include_half, include_bfloat16=include_bfloat16 + ) + if include_bool: + dtypes.append(core.bool) + if include_complex: + dtypes += get_all_complex_dtypes(include_complex32) + if include_qint: + dtypes += get_all_qint_dtypes() + return dtypes + + +def get_all_math_dtypes(device) -> list[core.dtype]: + return ( + get_all_int_dtypes() + + get_all_fp_dtypes( + include_half=device.startswith("cuda"), include_bfloat16=False + ) + + get_all_complex_dtypes() + ) + + +def get_all_complex_dtypes(include_complex32=False) -> list[core.dtype]: + return ( + [core.complex32, core.complex64, core.complex128] + if include_complex32 + else [core.complex64, core.complex128] + ) + + +def get_all_int_dtypes() -> list[core.dtype]: + return [core.uint8, core.int8, core.int16, core.int32, core.int64] + + +def get_all_fp_dtypes(include_half=True, include_bfloat16=True) -> list[core.dtype]: + dtypes = [core.float32, core.float64] + if include_half: + dtypes.append(core.float16) + if include_bfloat16: + dtypes.append(core.bfloat16) + return dtypes + + +def get_all_qint_dtypes() -> list[core.dtype]: + return [core.qint8, core.quint8, core.qint32, core.quint4x2, core.quint2x4] + + +float_to_corresponding_complex_type_map = { + core.float16: core.complex32, + core.float32: core.complex64, + core.float64: core.complex128, +} \ No newline at end of file diff --git a/mindnlp/core/testing/_internal/common_nn.py b/mindnlp/core/testing/_internal/common_nn.py new file mode 100644 index 000000000..e89e4ef57 --- /dev/null +++ b/mindnlp/core/testing/_internal/common_nn.py @@ -0,0 +1,3991 @@ +# mypy: ignore-errors + +from abc import abstractmethod +import tempfile +import unittest + +from copy import deepcopy +from functools import reduce, partial +from itertools import product +from operator import mul + + +from mindnlp import core +import mindnlp.core.nn as nn +import mindnlp.core.nn.functional as F +from mindnlp.core.nn import _reduction as _Reduction +from mindnlp.core.testing._internal.common_utils import TestCase, to_gpu, freeze_rng_state, is_iterable, \ + gradcheck, gradgradcheck, set_default_dtype, skipIfTorchDynamo, TEST_WITH_ROCM +from mindnlp.core.testing._internal.common_cuda import TEST_CUDA, SM90OrLater +# from mindnlp.core.autograd.gradcheck import _get_numerical_jacobian, _iter_tensors +# from mindnlp.core.autograd import Variable +from mindnlp.core.types import _TensorOrTensors + +from typing import Callable, Union, Any +from collections.abc import Sequence + +TemporaryFile = tempfile.TemporaryFile +PRECISION = 1e-5 + + +def get_reduction(m): + result = getattr(m, 'reduction', None) + if result is None: + result = _Reduction.legacy_get_string(getattr(m, 'sizeAverage', None), True, emit_warning=False) + assert result is not None + return result + + +def get_weight(m): + result = getattr(m, 'weight', None) + if result is not None: + return result + return getattr(m, 'weights', None) + +# NOTE [How to check NN module / functional API parity between Python and C++ frontends] +# +# The way to check API parity is to add parity tests for the NN module / functional of interest. +# Here are the detailed steps: +# +# For NN module: +# 1. Make sure you already have a test dict with the module configuration you want to test. +# 2. Add `cpp_constructor_args` entry to the test dict, with its value exactly matching +# the Python module constructor arguments. For example, if in the test dict we pass +# `(10, 8)` to `core.nn.Linear` constructor, then we should pass `core::nn::LinearOptions(10, 8)` +# as the corresponding C++ constructor argument to `core::nn::Linear`. +# 3. If in the process of performing the above step you referenced any variables +# in the `cpp_constructor_args` entry, you must add `cpp_var_map` entry +# to the test dict to make sure that those variables are populated with the right Python values. +# For example, if the Python constructor call is +# `core.nn.FractionalMaxPool2d(2, output_ratio=0.5, _random_samples=random_samples)`, +# the corresponding C++ constructor argument is +# `core::nn::FractionalMaxPool2dOptions(2).output_ratio(0.5)._random_samples(random_samples)`, +# and the `cpp_var_map` entry must be +# `{'random_samples': random_samples}` in order to populate the C++ variable `random_samples` +# used in the C++ constructor argument with the Python tensor value `random_samples`. +# +# For NN functional: +# 1. Make sure you already have a test dict with the functional configuration you want to test. +# 2. If the test dict's `constructor` entry looks like `wrap_functional(F.some_functional_name, ...)`, +# then you must add `cpp_options_args` entry to the test dict, with its value exactly matching the Python +# functional optional arguments. For example, if the test dict's `constructor` entry is +# `wrap_functional(F.interpolate, size=12, scale_factor=None, mode='nearest')`, +# then the `cpp_options_args` entry should be +# "F::InterpolateFuncOptions().size(std::vector({12})).scale_factor(std::nullopt).mode(core::kNearest)". +# 3. Otherwise, if the test dict's `constructor` entry looks like +# `wrap_functional(lambda i: F.some_functional_name(...))`, +# then you must add `cpp_function_call` entry to the test dict, with its value exactly matching the Python +# functional function call. For example, if the test dict's `constructor` entry is +# `wrap_functional(lambda i: F.poisson_nll_loss(i, t.type_as(i), reduction='none'))`, +# then the `cpp_function_call` entry should be +# "F::poisson_nll_loss(i, t.to(i.options()), F::PoissonNLLLossFuncOptions().reduction(core::kNone))". +# 4. If in the process of performing the above two steps you referenced any variables +# in the `cpp_options_args` or `cpp_function_call` entry, you must +# add `cpp_var_map` entry to the test dict to make sure that those variables +# are populated with the right Python values. For example, if the test dict's `constructor` entry is +# `wrap_functional(lambda i: F.poisson_nll_loss(i, t.type_as(i), reduction='none'))`, +# then the `cpp_function_call` entry should be +# "F::poisson_nll_loss(i, t.to(i.options()), F::PoissonNLLLossFuncOptions().reduction(core::kNone))". +# Notice that there are two variables `i` and `t` that need to have their values provided, +# and the way to do so is to add a `cpp_var_map` entry: `cpp_var_map={'i': '_get_input()', 't': t}`. +# (Note that for `i`, since we want it to take the Python input value, we pass '_get_input()' string as value +# and the C++ parity test mechanism will populate `i` with the Python input value correctly.) +# +# There are also a few optional flags in the test dict to control the C++ parity test behavior: +# +# - `test_cpp_api_parity`: if `False`, skips the C++ parity test for this test dict. Default: True. +# - `has_parity`: if `False`, expects this test dict to fail the C++ parity test. Default: True. + + +module_tests = [ + dict( + module_name='Linear', + constructor_args=(10, 8), + cpp_constructor_args='core::nn::LinearOptions(10, 8)', + input_size=(4, 10), + reference_fn=lambda i, p, _: core.mm(i, p[0].t()) + p[1].view(1, -1).expand(4, 8), + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Linear', + constructor_args=(10, 8, False), + cpp_constructor_args='core::nn::LinearOptions(10, 8).bias(false)', + input_size=(4, 10), + desc='no_bias', + reference_fn=lambda i, p, _: core.mm(i, p[0].t()), + with_tf32=True, + tf32_precision=0.05 if TEST_WITH_ROCM else 0.005, + default_dtype=core.double, + ), + dict( + module_name='RReLU', + input_size=(1, 2, 2), + test_cuda=False, + default_dtype=core.double, + ), + dict( + module_name='RReLU', + constructor_args=(0.1, 0.9), + cpp_constructor_args='core::nn::RReLUOptions().lower(0.1).upper(0.9)', + input_size=(4, 4, 5), + desc='with_up_down', + test_cuda=False, + default_dtype=core.double, + ), + dict( + module_name='Flatten', + input_size=(2, 3, 4, 5), + reference_fn=lambda i, *_: core.flatten(i, 1), + default_dtype=core.double, + ), + # TODO: reference function + dict( + module_name='CrossMapLRN2d', + constructor_args=(5, 5e-3, 1e-3, 2), + cpp_constructor_args='core::nn::CrossMapLRN2dOptions(5).alpha(5e-3).beta(1e-3).k(2)', + input_size=(2, 3, 6, 6), + check_gradgrad=False, + # TODO(#50743): Figure out the error. "RuntimeError: Unrecognized tensor type ID: Batched" + check_batched_grad=False, + default_dtype=core.double, + ), +] + + +# Generates rand tensor with non-equal values. This ensures that duplicate +# values won't be causing test failure for modules like MaxPooling. +# size should be small, otherwise randperm fails / long overflows. +def _rand_tensor_non_equal(*size): + total = reduce(mul, size, 1) + return core.randperm(total).view(*size).double() + + +def wrap_functional(fn, **kwargs): + class FunctionalModule(nn.Module): + def forward(self, *args): + return fn(*args, **kwargs) + return FunctionalModule + + +def poissonnllloss_no_reduce_test(): + t = core.randn(10, 10) + return dict( + fullname='PoissonNLLLoss_no_reduce', + constructor=wrap_functional( + lambda i: F.poisson_nll_loss(i, t.type_as(i), reduction='none')), + cpp_function_call='F::poisson_nll_loss(' + 'i, t.to(i.options()), F::PoissonNLLLossFuncOptions().reduction(core::kNone))', + input_fn=lambda: core.rand(10, 10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: i.exp() - t.mul(i), + pickle=False, + default_dtype=core.double) + + +def bceloss_no_reduce_test(): + t = Variable(core.randn(15, 10).gt(0).to(core.double)) + return dict( + fullname='BCELoss_no_reduce', + constructor=wrap_functional( + lambda i: F.binary_cross_entropy(i, t.type_as(i), reduction='none')), + cpp_function_call='F::binary_cross_entropy(' + 'i, t.to(i.options()), F::BinaryCrossEntropyFuncOptions().reduction(core::kNone))', + input_fn=lambda: core.rand(15, 10).clamp_(2.8e-2, 1 - 2.8e-2), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: -(t * i.log() + (1 - t) * (1 - i).log()), + pickle=False, + precision=7e-4, + default_dtype=core.double) + + +def bceloss_no_reduce_scalar_test(): + t = core.randn(()).gt(0).to(core.double) + return dict( + fullname='BCELoss_no_reduce_scalar', + constructor=wrap_functional( + lambda i: F.binary_cross_entropy(i, t.type_as(i), reduction='none')), + cpp_function_call='F::binary_cross_entropy(' + 'i, t.to(i.options()), F::BinaryCrossEntropyFuncOptions().reduction(core::kNone))', + input_fn=lambda: core.rand(()).clamp_(2.8e-2, 1 - 2.8e-2), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: -(t * i.log() + (1 - t) * (1 - i).log()), + pickle=False, + default_dtype=core.double) + + +def bceloss_weights_no_reduce_test(): + t = Variable(core.randn(15, 10, dtype=core.double).gt(0).to(core.double)) + weights = core.rand(10, dtype=core.double) + return dict( + fullname='BCELoss_weights_no_reduce', + constructor=wrap_functional( + lambda i: F.binary_cross_entropy(i, t.type_as(i), + weight=weights.type_as(i), reduction='none')), + cpp_function_call='F::binary_cross_entropy(' + 'i, t.to(i.options()), ' + 'F::BinaryCrossEntropyFuncOptions().weight(weights.to(i.options())).reduction(core::kNone))', + input_fn=lambda: core.rand(15, 10).clamp_(2.8e-2, 1 - 2.8e-2), + cpp_var_map={'i': '_get_input()', 't': t, 'weights': weights}, + reference_fn=lambda i, p, m: -(t * i.log() + (1 - t) * (1 - i).log()) * weights, + pickle=False, + precision=3e-4, + default_dtype=core.double, + ) + + +def bceloss_weights_no_reduce_scalar_test(): + t = core.randn(()).gt(0).to(core.double) + weights = core.rand((), dtype=core.double) + return dict( + fullname='BCELoss_weights_no_reduce_scalar', + constructor=wrap_functional( + lambda i: F.binary_cross_entropy(i, t.type_as(i), + weight=weights.type_as(i), reduction='none')), + cpp_function_call='''F::binary_cross_entropy( + i, t.to(i.options()), + F::BinaryCrossEntropyFuncOptions().weight(weights.to(i.options())).reduction(core::kNone))''', + cpp_var_map={'i': '_get_input()', 't': t, 'weights': weights}, + input_fn=lambda: core.rand(()).clamp_(2.8e-2, 1 - 2.8e-2), + reference_fn=lambda i, *_: -(t * i.log() + (1 - t) * (1 - i).log()) * weights, + pickle=False, + default_dtype=core.double, + ) + + +def bce_with_logistic_legacy_enum_test(): + t = Variable(core.randn(15, 10).gt(0).to(core.double)) + sigmoid = nn.Sigmoid() + return dict( + fullname='BCEWithLogitsLoss_legacy_enum', + constructor=wrap_functional( + lambda i: F.binary_cross_entropy_with_logits(i, t.type_as(i), reduce=False)), + cpp_function_call='''F::binary_cross_entropy_with_logits( + i, t.to(i.options()), F::BinaryCrossEntropyWithLogitsFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.rand(15, 10).clamp_(2.8e-2, 1 - 2.8e-2), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: -(t * sigmoid(i).log() + (1 - t) * (1 - sigmoid(i)).log()), + check_gradgrad=False, + pickle=False, + default_dtype=core.double, + ) + + +def bce_with_logistic_no_reduce_test(): + t = Variable(core.randn(15, 10).gt(0).to(core.double)) + sigmoid = nn.Sigmoid() + return dict( + fullname='BCEWithLogitsLoss_no_reduce', + constructor=wrap_functional( + lambda i: F.binary_cross_entropy_with_logits(i, t.type_as(i), reduction='none')), + cpp_function_call='''F::binary_cross_entropy_with_logits( + i, t.to(i.options()), F::BinaryCrossEntropyWithLogitsFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.rand(15, 10).clamp_(2.8e-2, 1 - 2.8e-2), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: -(t * sigmoid(i).log() + (1 - t) * (1 - sigmoid(i)).log()), + check_gradgrad=False, + pickle=False, + default_dtype=core.double, + ) + + +def bce_with_logistic_no_reduce_scalar_test(): + t = core.randn(()).gt(0).to(core.double) + sigmoid = nn.Sigmoid() + return dict( + fullname='BCEWithLogitsLoss_no_reduce_scalar', + constructor=wrap_functional( + lambda i: F.binary_cross_entropy_with_logits(i, t.type_as(i), reduction='none')), + cpp_function_call='''F::binary_cross_entropy_with_logits( + i, t.to(i.options()), F::BinaryCrossEntropyWithLogitsFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.rand(()).clamp_(2.8e-2, 1 - 2.8e-2), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: -(t * sigmoid(i).log() + (1 - t) * (1 - sigmoid(i)).log()), + check_gradgrad=False, + pickle=False, + default_dtype=core.double, + ) + + +def kldivloss_with_target_no_reduce_test(): + t = core.rand(10, 10, dtype=core.double) + return dict( + fullname='KLDivLoss_with_target_no_reduce', + constructor=wrap_functional( + lambda i: F.kl_div(i, t.type_as(i), reduction='none')), + cpp_function_call='F::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(core::kNone))', + input_fn=lambda: core.rand(10, 10).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['KLDivLoss'](i, t.type_as(i), reduction='none'), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def kldivloss_no_reduce_test(): + t = core.rand(10, 10, dtype=core.double) + return dict( + fullname='KLDivLoss_no_reduce', + constructor=wrap_functional( + lambda i: F.kl_div(i, t.type_as(i), reduction='none')), + cpp_function_call='F::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(core::kNone))', + input_fn=lambda: core.rand(10, 10).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['KLDivLoss'](i, t.type_as(i), reduction='none'), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double, + ) + + +def kldivloss_no_reduce_scalar_test(): + t = core.rand((), dtype=core.double) + return dict( + fullname='KLDivLoss_no_reduce_scalar', + constructor=wrap_functional( + lambda i: F.kl_div(i, t.type_as(i), reduction='none')), + cpp_function_call='F::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(core::kNone))', + input_fn=lambda: core.rand(()).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['KLDivLoss'](i, t.type_as(i), reduction='none'), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def kldivloss_with_log_target_no_reduce_test(): + t = core.rand(10, 10, dtype=core.double).log() + return dict( + fullname='KLDivLoss_with_log_target_no_reduce', + constructor=wrap_functional( + lambda i: F.kl_div(i, t.type_as(i), reduction='none', log_target=True)), + cpp_function_call='F::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(core::kNone).log_target(true))', + input_fn=lambda: core.rand(10, 10).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['KLDivLoss_log_target'](i, t.type_as(i), reduction='none'), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def kldivloss_no_reduce_log_target_test(): + t = core.rand(10, 10, dtype=core.double).log() + return dict( + fullname='KLDivLoss_no_reduce_log_target', + constructor=wrap_functional( + lambda i: F.kl_div(i, t.type_as(i), reduction='none', log_target=True)), + cpp_function_call='F::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(core::kNone).log_target(true))', + input_fn=lambda: core.rand(10, 10).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['KLDivLoss_log_target'](i, t.type_as(i), reduction='none'), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double, + ) + + +def kldivloss_no_reduce_scalar_log_target_test(): + t = core.rand((), dtype=core.double).log() + return dict( + fullname='KLDivLoss_no_reduce_scalar_log_target', + constructor=wrap_functional( + lambda i: F.kl_div(i, t.type_as(i), reduction='none', log_target=True)), + cpp_function_call='F::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(core::kNone).log_target(true))', + input_fn=lambda: core.rand(()).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['KLDivLoss_log_target'](i, t.type_as(i), reduction='none'), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def l1loss_no_reduce_test(): + t = core.randn(2, 3, 4, dtype=core.double) + return dict( + fullname='L1Loss_no_reduce', + constructor=wrap_functional( + lambda i: F.l1_loss(i, t.type_as(i), reduction='none')), + cpp_function_call='F::l1_loss(i, t.to(i.options()), F::L1LossFuncOptions().reduction(core::kNone))', + input_fn=lambda: core.randn(2, 3, 4), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: (i - t.type_as(i)).abs(), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def l1loss_no_reduce_complex_test(): + t = core.randn(2, 3, 4, dtype=core.cdouble) + return dict( + fullname='L1Loss_no_reduce_complex', + constructor=wrap_functional( + lambda i: F.l1_loss(i, t.type_as(i), reduction='none')), + cpp_function_call='F::l1_loss(i, t.to(i.options()), F::L1LossFuncOptions().reduction(core::kNone))', + input_fn=lambda: core.randn(2, 3, 4, dtype=core.cdouble), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: (i - t.type_as(i)).abs(), + supports_forward_ad=True, + pickle=False) + + +def l1loss_no_reduce_scalar_test(): + t = core.randn((), dtype=core.double) + return dict( + fullname='L1Loss_no_reduce_scalar', + constructor=wrap_functional( + lambda i: F.l1_loss(i, t.type_as(i), reduction='none')), + cpp_function_call='F::l1_loss(i, t.to(i.options()), F::L1LossFuncOptions().reduction(core::kNone))', + input_fn=lambda: core.randn(()), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: (i - t.type_as(i)).abs(), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def mseloss_no_reduce_test(): + input_size = (2, 3, 4, 5) + target = core.randn(*input_size, dtype=core.double) + return dict( + fullname='MSELoss_no_reduce', + constructor=wrap_functional( + lambda i: F.mse_loss(i, target.type_as(i), reduction='none')), + cpp_function_call='F::mse_loss(i, target.to(i.options()), F::MSELossFuncOptions().reduction(core::kNone))', + input_size=input_size, + cpp_var_map={'i': '_get_input()', 'target': target}, + reference_fn=lambda i, *_: (i - target).pow(2), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def mseloss_no_reduce_scalar_test(): + input_size = () + target = core.randn(input_size, dtype=core.double) + return dict( + fullname='MSELoss_no_reduce_scalar', + constructor=wrap_functional( + lambda i: F.mse_loss(i, target.type_as(i), reduction='none')), + cpp_function_call='F::mse_loss(i, target.to(i.options()), F::MSELossFuncOptions().reduction(core::kNone))', + input_size=input_size, + cpp_var_map={'i': '_get_input()', 'target': target}, + reference_fn=lambda i, *_: (i - target).pow(2), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def nllloss_no_reduce_test(): + t = Variable(core.empty(15).uniform_().mul(10).floor().long()) + kwargs = {'reduction': 'none'} + return dict( + fullname='NLLLoss_no_reduce', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), reduction=kwargs['reduction'])), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), F::NLLLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.rand(15, 10).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLoss'](i, t.type_as(i).long(), **kwargs), + pickle=False, + default_dtype=core.double) + + +def nllloss_no_reduce_ignore_index_test(): + t = Variable(core.empty(15).uniform_().mul(10).floor().long()) + kwargs: dict[str, Union[int, str]] = {'ignore_index': 2, 'reduction': 'none'} + return dict( + fullname='NLLLoss_no_reduce_ignore_index', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), ignore_index=int(kwargs['ignore_index']), + reduction=str(kwargs['reduction']))), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), F::NLLLossFuncOptions().ignore_index(2).reduction(core::kNone))''', + input_fn=lambda: core.rand(15, 10).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLoss'](i, t.type_as(i).long(), **kwargs), + pickle=False, + default_dtype=core.double) + + +def nllloss_no_reduce_weights_test(): + t = Variable(core.empty(15).uniform_().mul(10).floor().long()) + weight = core.rand(10) + + def kwargs(i): + return {'weight': weight.type_as(i), 'reduction': 'none'} + + return dict( + fullname='NLLLoss_no_reduce_weights', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), **kwargs(i))), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), + F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(core::kNone))''', + input_fn=lambda: core.rand(15, 10).add(1e-2).log(), + cpp_var_map={'i': '_get_input()', 't': t, 'weight': weight}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLoss'](i, t.type_as(i).long(), **kwargs(i)), + pickle=False, + default_dtype=core.double) + + +def nllloss_no_reduce_weights_ignore_index_test(): + t = Variable(core.empty(15).uniform_().mul(10).floor().long()) + weight = core.rand(10) + + def kwargs(i): + return {'weight': weight.type_as(i), 'reduction': 'none', + 'ignore_index': 2} + + return dict( + fullname='NLLLoss_no_reduce_weights_ignore_index', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), **kwargs(i.data))), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), + F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(core::kNone).ignore_index(2))''', + input_fn=lambda: core.rand(15, 10).add(1e-2).log(), + cpp_var_map={'i': '_get_input()', 't': t, 'weight': weight}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLoss'](i, t.type_as(i).long(), **kwargs(i)), + pickle=False, + default_dtype=core.double) + + +def nllloss_no_reduce_weights_ignore_index_neg_test(): + t = Variable(core.empty(15).uniform_().mul(10).floor().long()) + weight = core.rand(10) + + def kwargs(i): + return {'weight': weight.type_as(i), 'reduction': 'none', + 'ignore_index': -1} + + return dict( + fullname='NLLLoss_no_reduce_weights_ignore_index_neg', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), **kwargs(i))), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), + F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(core::kNone).ignore_index(-1))''', + input=core.rand(15, 10, dtype=core.double).add(1e-2).log(), + cpp_var_map={'i': '_get_input()', 't': t, 'weight': weight}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLoss'](i, t.type_as(i).long(), **kwargs(i)), + pickle=False, + default_dtype=core.double) + + +def nllloss2d_no_reduce_test(): + t = Variable(core.rand(2, 5, 5).mul(3).floor().long()) + kwargs = {'reduction': 'none'} + return dict( + fullname='NLLLoss2d_no_reduce', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), reduction=kwargs['reduction'])), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), F::NLLLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.rand(2, 3, 5, 5).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLossNd'](i, t.type_as(i).long(), **kwargs), + pickle=False, + default_dtype=core.double) + + +def nllloss2d_no_reduce_ignore_index_test(): + t = Variable(core.rand(2, 5, 5).mul(3).floor().long()) + kwargs: dict[str, Union[int, str]] = {'ignore_index': 1, 'reduction': 'none'} + return dict( + fullname='NLLLoss2d_no_reduce_ignore_index', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), ignore_index=int(kwargs['ignore_index']), + reduction=str(kwargs['reduction']))), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), F::NLLLossFuncOptions().ignore_index(1).reduction(core::kNone))''', + input_fn=lambda: core.rand(2, 3, 5, 5).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLossNd'](i, t.type_as(i).long(), **kwargs), + pickle=False, + default_dtype=core.double) + + +def nllloss2d_no_reduce_weights_test(): + t = Variable(core.rand(2, 5, 5).mul(3).floor().long()) + weight = core.rand(3) + + def kwargs(i): + return {'weight': weight.type_as(i), 'reduction': 'none'} + + return dict( + fullname='NLLLoss2d_no_reduce_weights', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), **kwargs(i))), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), + F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(core::kNone))''', + input_fn=lambda: core.rand(2, 3, 5, 5).log(), + cpp_var_map={'i': '_get_input()', 't': t, 'weight': weight}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLossNd'](i, t.type_as(i).long(), **kwargs(i)), + pickle=False, + default_dtype=core.double) + + +def nlllossNd_no_reduce_test(): + t = Variable(core.rand(2, 5, 5, 2, 2).mul(3).floor().long()) + kwargs = {'reduction': 'none'} + return dict( + fullname='NLLLossNd_no_reduce', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), reduction=kwargs['reduction'])), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), F::NLLLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.rand(2, 3, 5, 5, 2, 2).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLossNd'](i, t.type_as(i).long(), **kwargs), + pickle=False, + default_dtype=core.double) + + +def nlllossNd_no_reduce_ignore_index_test(): + t = Variable(core.rand(2, 5, 5, 2, 2).mul(3).floor().long()) + kwargs: dict[str, Union[int, str]] = {'ignore_index': 1, 'reduction': 'none'} + return dict( + fullname='NLLLossNd_no_reduce_ignore_index', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), ignore_index=int(kwargs['ignore_index']), + reduction=str(kwargs['reduction']))), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), F::NLLLossFuncOptions().ignore_index(1).reduction(core::kNone))''', + input_fn=lambda: core.rand(2, 3, 5, 5, 2, 2).log(), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLossNd'](i, t.type_as(i).long(), **kwargs), + pickle=False, + default_dtype=core.double) + + +def nlllossNd_no_reduce_weights_test(): + t = Variable(core.rand(2, 5, 5, 2, 2).mul(3).floor().long()) + weight = core.rand(3) + + def kwargs(i): + return {'weight': weight.type_as(i), 'reduction': 'none'} + + return dict( + fullname='NLLLossNd_no_reduce_weights', + constructor=wrap_functional( + lambda i: F.nll_loss(i, t.type_as(i).long(), **kwargs(i))), + cpp_function_call='''F::nll_loss( + i, t.to(i.options()).to(core::kLong), + F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(core::kNone))''', + input_fn=lambda: core.rand(2, 3, 5, 5, 2, 2).log(), + cpp_var_map={'i': '_get_input()', 't': t, 'weight': weight}, + reference_fn=lambda i, *_: + loss_reference_fns['NLLLossNd'](i, t.type_as(i).long(), **kwargs(i)), + pickle=False, + default_dtype=core.double) + + +def smoothl1loss_no_reduce_test(): + t = core.randn(2, 3, 4, dtype=core.double) + return dict( + fullname='SmoothL1Loss_no_reduce', + constructor=wrap_functional( + lambda i: F.smooth_l1_loss(i, t.type_as(i), reduction='none')), + cpp_function_call='''F::smooth_l1_loss( + i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(2, 3, 4), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['SmoothL1Loss'](i, t.type_as(i), reduction='none'), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def smoothl1loss_no_reduce_scalar_test(): + t = core.randn((), dtype=core.double) + return dict( + fullname='SmoothL1Loss_no_reduce_scalar', + constructor=wrap_functional( + lambda i: F.smooth_l1_loss(i, t.type_as(i), reduction='none')), + cpp_function_call='''F::smooth_l1_loss( + i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(()), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['SmoothL1Loss'](i, t.type_as(i), reduction='none'), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def smoothl1loss_beta_test(): + t = core.randn(2, 3, 4, dtype=core.double) + return dict( + fullname='SmoothL1Loss_beta', + constructor=wrap_functional( + lambda i: F.smooth_l1_loss(i, t.type_as(i), reduction='none', beta=0.5)), + cpp_function_call='''F::smooth_l1_loss( + i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(core::kNone), 0.5)''', + input_fn=lambda: core.randn(2, 3, 4), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['SmoothL1Loss'](i, t.type_as(i), reduction='none', beta=0.5), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def smoothl1loss_zero_beta_test(): + t = core.randn(2, 3, 4, dtype=core.double) + return dict( + fullname='SmoothL1Loss_zero_beta', + constructor=wrap_functional( + lambda i: F.smooth_l1_loss(i, t.type_as(i), reduction='none', beta=0)), + cpp_function_call='''F::smooth_l1_loss( + i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(core::kNone), 0)''', + input_fn=lambda: core.randn(2, 3, 4), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['SmoothL1Loss'](i, t.type_as(i), reduction='none', beta=0), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def huberloss_delta_test(): + t = core.randn(2, 3, 4) + return dict( + fullname='HuberLoss_delta', + constructor=wrap_functional( + lambda i: F.huber_loss(i, t.type_as(i), reduction='none', delta=0.5)), + cpp_function_call='''F::huber_loss( + i, t.to(i.options()), F::HuberLossFuncOptions().reduction(core::kNone).delta(0.5))''', + input_fn=lambda: core.randn(2, 3, 4), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['HuberLoss'](i, t.type_as(i), reduction='none', delta=0.5), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def multilabelmarginloss_0d_no_reduce_test(): + t = core.zeros(()).long() + return dict( + fullname='MultiLabelMarginLoss_0d_no_reduce', + constructor=wrap_functional( + lambda i: F.multilabel_margin_loss(i, t.type_as(i).long(), reduction='none')), + cpp_function_call='''F::multilabel_margin_loss( + i, t.to(i.options()).to(core::kLong), F::MultilabelMarginLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(()), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['MultiLabelMarginLoss'](i, t.data.type_as(i).long(), reduction='none'), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False) + + +def multilabelmarginloss_1d_no_reduce_test(): + t = Variable(core.rand(10).mul(10).floor().long()) + return dict( + fullname='MultiLabelMarginLoss_1d_no_reduce', + constructor=wrap_functional( + lambda i: F.multilabel_margin_loss(i, t.type_as(i).long(), reduction='none')), + cpp_function_call='''F::multilabel_margin_loss( + i, t.to(i.options()).to(core::kLong), F::MultilabelMarginLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['MultiLabelMarginLoss'](i, t.data.type_as(i).long(), reduction='none'), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def multilabelmarginloss_index_neg_test(): + t = Variable(core.clamp(core.rand(5, 10).add(-.5).mul(20).floor().long(), min=-1)) + return dict( + fullname='MultiLabelMarginLoss_index_neg', + constructor=wrap_functional( + lambda i: F.multilabel_margin_loss(i, t.type_as(i).long(), reduction='none')), + cpp_function_call='''F::multilabel_margin_loss( + i, t.to(i.options()).to(core::kLong), F::MultilabelMarginLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(5, 10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['MultiLabelMarginLoss'](i, t.data.type_as(i).long(), reduction='none'), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def multilabelmarginloss_no_reduce_test(): + t = Variable(core.rand(5, 10).mul(10).floor().long()) + return dict( + fullname='MultiLabelMarginLoss_no_reduce', + constructor=wrap_functional( + lambda i: F.multilabel_margin_loss(i, t.type_as(i).long(), reduction='none')), + cpp_function_call='''F::multilabel_margin_loss( + i, t.to(i.options()).to(core::kLong), F::MultilabelMarginLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(5, 10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['MultiLabelMarginLoss'](i, t.data.type_as(i).long(), reduction='none'), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def hingeembeddingloss_no_reduce_test(): + t = Variable(core.randn(10).gt(0).to(core.double).mul_(2).sub(1)) + return dict( + fullname='HingeEmbeddingLoss_no_reduce', + constructor=wrap_functional( + lambda i: F.hinge_embedding_loss(i, t.type_as(i), reduction='none')), + cpp_function_call='''F::hinge_embedding_loss( + i, t.to(i.options()), F::HingeEmbeddingLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['HingeEmbeddingLoss'](i, t.type_as(i), reduction='none'), + check_sum_reduction=True, + pickle=False, + default_dtype=core.double) + + +def hingeembeddingloss_margin_no_reduce_test(): + t = Variable(core.randn(10).gt(0).to(core.double).mul_(2).sub(1)) + return dict( + fullname='HingeEmbeddingLoss_margin_no_reduce', + constructor=wrap_functional( + lambda i: F.hinge_embedding_loss(i, t.type_as(i), margin=0.5, reduction='none')), + cpp_function_call='''F::hinge_embedding_loss( + i, t.to(i.options()), F::HingeEmbeddingLossFuncOptions().margin(0.5).reduction(core::kNone))''', + input_fn=lambda: core.randn(10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['HingeEmbeddingLoss'](i, t.type_as(i), margin=0.5, reduction='none'), + check_sum_reduction=True, + pickle=False, + default_dtype=core.double) + + +def softmarginloss_no_reduce_test(): + t = core.randn(5, 5, dtype=core.double) + return dict( + fullname='SoftMarginLoss_no_reduce', + constructor=wrap_functional( + lambda i: F.soft_margin_loss(i, t.type_as(i), reduction='none')), + cpp_function_call='''F::soft_margin_loss( + i, t.to(i.options()), F::SoftMarginLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(5, 5), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['SoftMarginLoss'](i, t.type_as(i), reduction='none'), + supports_forward_ad=True, + pickle=False, + default_dtype=core.double) + + +def multilabelsoftmarginloss_no_reduce_test(): + t = core.rand(5, 10).mul(2).floor() + return dict( + fullname='MultiLabelSoftMarginLoss_no_reduce', + constructor=wrap_functional( + lambda i: F.multilabel_soft_margin_loss(i, t.type_as(i), reduction='none')), + cpp_function_call='''F::multilabel_soft_margin_loss( + i, t.to(i.options()), F::MultilabelSoftMarginLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(5, 10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + (-(t * i.sigmoid().log() + (1 - t) * (-i).sigmoid().log())).sum(dim=1) / i.size(1), + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def multilabelsoftmarginloss_weights_no_reduce_test(): + t = core.rand(5, 10).mul(2).floor() + weights = core.rand(10) + return dict( + fullname='MultiLabelSoftMarginLoss_weights_no_reduce', + constructor=wrap_functional( + lambda i: F.multilabel_soft_margin_loss(i, t.type_as(i), + weight=weights.type_as(i), reduction='none')), + cpp_function_call='''F::multilabel_soft_margin_loss( + i, t.to(i.options()), + F::MultilabelSoftMarginLossFuncOptions().weight(weights.to(i.options())).reduction(core::kNone))''', + input_fn=lambda: core.randn(5, 10), + cpp_var_map={'i': '_get_input()', 't': t, 'weights': weights}, + reference_fn=lambda i, *_: + (-(t * i.sigmoid().log() + (1 - t) * (-i).sigmoid().log()) * weights).sum(dim=1) / i.size(1), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def multimarginloss_no_reduce_test(): + t = core.rand(5).mul(8).floor().long() + return dict( + fullname='MultiMarginLoss_no_reduce', + constructor=wrap_functional( + lambda i: F.multi_margin_loss(i, t.type_as(i).long(), reduction='none')), + cpp_function_call='''F::multi_margin_loss( + i, t.to(i.options()).to(core::kLong), F::MultiMarginLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(5, 10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['MultiMarginLoss'](i, t.data.type_as(i).long(), reduction='none'), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def multimarginloss_1d_no_reduce_test(): + t = core.rand(1).mul(8).floor().long() + return dict( + fullname='MultiMarginLoss_1d_no_reduce', + constructor=wrap_functional( + lambda i: F.multi_margin_loss(i, t.type_as(i).long(), reduction='none')), + cpp_function_call='''F::multi_margin_loss( + i, t.to(i.options()).to(core::kLong), F::MultiMarginLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['MultiMarginLoss'](i, t.data.type_as(i).long(), reduction='none'), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def multimarginloss_1d_input_0d_target_no_reduce_test(): + t = core.rand(()).mul(8).floor().long() + return dict( + fullname='multimarginloss_1d_input_0d_target_no_reduce', + constructor=wrap_functional( + lambda i: F.multi_margin_loss(i, t.type_as(i).long(), reduction='none')), + cpp_function_call='''F::multi_margin_loss( + i, t.to(i.options()).to(core::kLong), F::MultiMarginLossFuncOptions().reduction(core::kNone))''', + input_fn=lambda: core.randn(10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['MultiMarginLoss'](i, t.data.type_as(i).long(), reduction='none'), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def multimarginloss_p_no_reduce_test(): + t = core.rand(5).mul(8).floor().long() + return dict( + fullname='MultiMarginLoss_p_no_reduce', + constructor=wrap_functional( + lambda i: F.multi_margin_loss(i, t.type_as(i).long(), p=2, reduction='none')), + cpp_function_call='''F::multi_margin_loss( + i, t.to(i.options()).to(core::kLong), F::MultiMarginLossFuncOptions().p(2).reduction(core::kNone))''', + input_fn=lambda: core.randn(5, 10).clamp_(1e-2, 1 - 1e-2), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['MultiMarginLoss'](i, t.data.type_as(i).long(), p=2, reduction='none'), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def multimarginloss_margin_no_reduce_test(): + t = core.rand(5).mul(8).floor().long() + return dict( + fullname='MultiMarginLoss_margin_no_reduce', + constructor=wrap_functional( + lambda i: F.multi_margin_loss(i, t.type_as(i).long(), margin=0.5, reduction='none')), + cpp_function_call='''F::multi_margin_loss( + i, t.to(i.options()).to(core::kLong), + F::MultiMarginLossFuncOptions().margin(0.5).reduction(core::kNone))''', + input_fn=lambda: core.randn(5, 10), + cpp_var_map={'i': '_get_input()', 't': t}, + reference_fn=lambda i, *_: + loss_reference_fns['MultiMarginLoss'](i, t.data.type_as(i).long(), + margin=0.5, reduction='none'), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def multimarginloss_weights_no_reduce_test(): + t = core.rand(5).mul(8).floor().long() + weights = core.rand(10, dtype=core.double) + return dict( + fullname='MultiMarginLoss_weights_no_reduce', + constructor=wrap_functional( + lambda i: F.multi_margin_loss(i, t.type_as(i).long(), weight=weights.type_as(i), + reduction='none')), + cpp_function_call='''F::multi_margin_loss( + i, t.to(i.options()).to(core::kLong), + F::MultiMarginLossFuncOptions().weight(weights.to(i.options())).reduction(core::kNone))''', + input_fn=lambda: core.randn(5, 10), + cpp_var_map={'i': '_get_input()', 't': t, 'weights': weights}, + reference_fn=lambda i, *_: + loss_reference_fns['MultiMarginLoss'](i, t.data.type_as(i).long(), + weight=weights, reduction='none'), + check_sum_reduction=True, + check_gradgrad=False, + pickle=False, + default_dtype=core.double) + + +def single_batch_reference_fn(input, parameters, module): + """Reference function for modules supporting no batch dimensions. + + The module is passed the input and target in batched form with a single item. + The output is squeezed to compare with the no-batch input. + """ + def unsqueeze_inp(inp): + if isinstance(inp, (list, tuple)): + return [t.unsqueeze(0) for t in inp] + return inp.unsqueeze(0) + + single_batch_input = unsqueeze_inp(input) + single_batch_input = [single_batch_input] if isinstance(single_batch_input, core.Tensor) else single_batch_input + with freeze_rng_state(): + return module(*single_batch_input).squeeze(0) + + +def get_new_module_tests(): + new_module_tests = [ + poissonnllloss_no_reduce_test(), + bceloss_no_reduce_test(), + bceloss_weights_no_reduce_test(), + bce_with_logistic_legacy_enum_test(), + bce_with_logistic_no_reduce_test(), + bceloss_no_reduce_scalar_test(), + bceloss_weights_no_reduce_scalar_test(), + bce_with_logistic_no_reduce_scalar_test(), + kldivloss_with_target_no_reduce_test(), + kldivloss_no_reduce_test(), + kldivloss_no_reduce_scalar_test(), + kldivloss_with_log_target_no_reduce_test(), + kldivloss_no_reduce_log_target_test(), + kldivloss_no_reduce_scalar_log_target_test(), + l1loss_no_reduce_test(), + l1loss_no_reduce_complex_test(), + l1loss_no_reduce_scalar_test(), + mseloss_no_reduce_test(), + mseloss_no_reduce_scalar_test(), + nllloss_no_reduce_test(), + nllloss_no_reduce_ignore_index_test(), + nllloss_no_reduce_weights_test(), + nllloss_no_reduce_weights_ignore_index_test(), + nllloss_no_reduce_weights_ignore_index_neg_test(), + nllloss2d_no_reduce_test(), + nllloss2d_no_reduce_weights_test(), + nllloss2d_no_reduce_ignore_index_test(), + nlllossNd_no_reduce_test(), + nlllossNd_no_reduce_weights_test(), + nlllossNd_no_reduce_ignore_index_test(), + smoothl1loss_no_reduce_test(), + smoothl1loss_no_reduce_scalar_test(), + smoothl1loss_beta_test(), + smoothl1loss_zero_beta_test(), + huberloss_delta_test(), + multilabelmarginloss_0d_no_reduce_test(), + multilabelmarginloss_1d_no_reduce_test(), + multilabelmarginloss_index_neg_test(), + multilabelmarginloss_no_reduce_test(), + hingeembeddingloss_no_reduce_test(), + hingeembeddingloss_margin_no_reduce_test(), + softmarginloss_no_reduce_test(), + multilabelsoftmarginloss_no_reduce_test(), + multilabelsoftmarginloss_weights_no_reduce_test(), + multimarginloss_no_reduce_test(), + multimarginloss_1d_no_reduce_test(), + multimarginloss_1d_input_0d_target_no_reduce_test(), + multimarginloss_p_no_reduce_test(), + multimarginloss_margin_no_reduce_test(), + multimarginloss_weights_no_reduce_test(), + dict( + module_name='Conv1d', + constructor_args=(4, 5, 3), + cpp_constructor_args='core::nn::Conv1dOptions(4, 5, 3)', + input_size=(2, 4, 10), + cudnn=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv1d', + constructor_args=(4, 5, 3, 2), + cpp_constructor_args='core::nn::Conv1dOptions(4, 5, 3).stride(2)', + input_size=(2, 4, 10), + cudnn=True, + desc='stride', + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv1d', + constructor_args=(4, 5, 3, 1, 1), + cpp_constructor_args='core::nn::Conv1dOptions(4, 5, 3).stride(1).padding(1)', + input_size=(2, 4, 10), + cudnn=True, + desc='pad1', + with_tf32=True, + tf32_precision=0.01, + default_dtype=core.double, + ), + dict( + module_name='Conv1d', + constructor_args=(4, 5, 5, 1, 2), + cpp_constructor_args='core::nn::Conv1dOptions(4, 5, 5).stride(1).padding(2)', + input_size=(2, 4, 10), + cudnn=True, + desc='pad2', + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv1d', + constructor_args=(4, 4, 3, 1, 1), + cpp_constructor_args='core::nn::Conv1dOptions(4, 4, 3).stride(1).padding(1)', + input_size=(1, 4, 1), + cudnn=True, + desc='pad1size1', + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv1d', + constructor_args=(4, 4, 5, 1, 2), + cpp_constructor_args='core::nn::Conv1dOptions(4, 4, 5).stride(1).padding(2)', + input_size=(1, 4, 1), + cudnn=True, + desc='pad2size1', + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv1d', + constructor_args=(4, 5, 3), + cpp_constructor_args='core::nn::Conv1dOptions(4, 5, 3)', + input_size=(0, 4, 10), + cudnn=True, + desc='zero_batch', + with_tf32=True, + tf32_precision=0.005, + ), + dict( + fullname='Conv1d_dilated', + constructor=lambda: nn.Conv1d(4, 5, kernel_size=3, dilation=2), + cpp_constructor_args='core::nn::Conv1dOptions(4, 5, 3).dilation(2)', + input_size=(2, 4, 10), + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv1d_groups', + constructor=lambda: nn.Conv1d(4, 6, kernel_size=3, groups=2), + cpp_constructor_args='core::nn::Conv1dOptions(4, 6, 3).groups(2)', + input_size=(2, 4, 6), + cudnn=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv1d_pad_valid', + constructor=lambda: nn.Conv1d(4, 5, 3, padding="valid"), + cpp_constructor_args='core::nn::Conv1dOptions(4, 5, 3).padding(core::kValid)', + input_size=(2, 4, 10), + cudnn=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv1d_pad_same', + constructor=lambda: nn.Conv1d(4, 5, 3, padding="same"), + cpp_constructor_args='core::nn::Conv1dOptions(4, 5, 3).padding(core::kSame)', + input_size=(2, 4, 10), + cudnn=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv1d_pad_same2', + constructor=lambda: nn.Conv1d(4, 5, 4, padding="same"), + cpp_constructor_args='core::nn::Conv1dOptions(4, 5, 4).padding(core::kSame)', + input_size=(2, 4, 10), + cudnn=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv1d_pad_same_dilated', + constructor=lambda: nn.Conv1d(4, 5, 4, padding="same", dilation=2), + cpp_constructor_args='core::nn::Conv1dOptions(4, 5, 3).padding(core::kSame).dilation(2)', + input_size=(2, 4, 10), + cudnn=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='ConvTranspose1d', + constructor=lambda: nn.ConvTranspose1d(3, 4, kernel_size=3, stride=(3,), padding=1, output_padding=(1,)), + cpp_constructor_args='core::nn::ConvTranspose1dOptions(3, 4, 3).stride(3).padding(1).output_padding(1)', + cudnn=True, + input_size=(1, 3, 7), + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='ConvTranspose1d', + constructor_args=(3, 4, 3, 2, 1, 1, 1, False), + cpp_constructor_args='''core::nn::ConvTranspose1dOptions(3, 4, 3) + .stride(2).padding(1).output_padding(1).groups(1).bias(false)''', + input_size=(1, 3, 6), + cudnn=True, + desc='no_bias', + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='ConvTranspose1d', + constructor_args=(3, 4, 3, 2, 1, 1, 1, True, 2), + cpp_constructor_args='''core::nn::ConvTranspose1dOptions(3, 4, 3) + .stride(2).padding(1).output_padding(1).groups(1).bias(true).dilation(2)''', + input_size=(1, 3, 6), + cudnn=True, + desc='dilated', + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='ConvTranspose1d_groups', + constructor=lambda: nn.ConvTranspose1d(4, 6, 3, stride=(3,), padding=1, output_padding=(1,), groups=2), + cpp_constructor_args='''core::nn::ConvTranspose1dOptions(4, 6, 3) + .stride(3).padding(1).output_padding(1).groups(2)''', + cudnn=True, + input_size=(2, 4, 7), + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv2d', + constructor_args=(3, 4, (3, 2)), + cpp_constructor_args='core::nn::Conv2dOptions(3, 4, {3, 2})', + input_size=(2, 3, 7, 5), + cudnn=True, + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv2d', + constructor_args=(3, 4, (3, 3), (2, 2)), + cpp_constructor_args='core::nn::Conv2dOptions(3, 4, {3, 3}).stride({2, 2})', + input_size=(2, 3, 6, 6), + cudnn=True, + desc='strided', + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv2d', + constructor_args=(3, 4, (3, 3), (2, 2), (1, 1)), + cpp_constructor_args='core::nn::Conv2dOptions(3, 4, {3, 3}).stride({2, 2}).padding({1, 1})', + input_size=(2, 3, 6, 6), + cudnn=True, + desc='padding', + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv2d', + constructor_args=(3, 2, (3, 3), (2, 2), (1, 1), (2, 2)), + cpp_constructor_args='core::nn::Conv2dOptions(3, 2, {3, 3}).stride({2, 2}).padding({1, 1}).dilation({2, 2})', + input_size=(2, 3, 8, 8), + cudnn=True, + desc='dilated', + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv2d', + constructor_args=(3, 4, (3, 2), 1, 0, 1, 1, False), + cpp_constructor_args='''core::nn::Conv2dOptions(3, 4, {3, 2}) + .stride(1).padding(0).dilation(1).groups(1).bias(false)''', + input_size=(2, 3, 6, 5), + cudnn=True, + desc='no_bias', + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.015, + default_dtype=core.double, + ), + dict( + module_name='Conv2d', + constructor_args=(3, 4, (3, 2)), + cpp_constructor_args='core::nn::Conv2dOptions(3, 4, {3, 2})', + input_size=(0, 3, 7, 5), + cudnn=True, + desc='zero_batch', + check_with_long_tensor=True, + with_tf32=True, + ), + dict( + fullname='Conv2d_groups', + constructor=lambda: nn.Conv2d(4, 6, (3, 2), groups=2), + cpp_constructor_args='core::nn::Conv2dOptions(4, 6, {3, 2}).groups(2)', + input_size=(2, 4, 6, 5), + cudnn=True, + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.015, + default_dtype=core.double, + ), + dict( + fullname='Conv2d_groups_thnn', + constructor=lambda: nn.Conv2d(4, 6, (3, 2), groups=2), + cpp_constructor_args='core::nn::Conv2dOptions(4, 6, {3, 2}).groups(2)', + input_size=(2, 4, 6, 5), + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.015, + default_dtype=core.double, + ), + dict( + fullname='Conv2d_pad_valid', + constructor=lambda: nn.Conv2d(2, 4, (3, 4), padding="valid"), + cpp_constructor_args='core::nn::Conv2dOptions(2, 4, {3, 4}).padding(core::kValid)', + input_size=(2, 2, 6, 5), + cudnn=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv2d_pad_same', + constructor=lambda: nn.Conv2d(2, 4, (3, 4), padding="same"), + cpp_constructor_args='core::nn::Conv2dOptions(2, 4, {3, 4}).padding(core::kSame)', + input_size=(2, 2, 6, 5), + cudnn=True, + with_tf32=True, + tf32_precision=0.01, + default_dtype=core.double, + ), + dict( + fullname='Conv2d_pad_same_dilated', + constructor=lambda: nn.Conv2d(2, 4, (3, 4), padding="same", dilation=2), + cpp_constructor_args='core::nn::Conv2dOptions(2, 4, {3, 4}).padding(core::kSame).dilation(2)', + input_size=(2, 2, 6, 5), + cudnn=True, + with_tf32=True, + tf32_precision=0.01, + default_dtype=core.double, + ), + dict( + module_name='ConvTranspose2d', + constructor_args=(3, 4, 3, (3, 2), 1, (1, 1)), + cpp_constructor_args='''core::nn::ConvTranspose2dOptions(3, 4, 3) + .stride({3, 2}).padding(1).output_padding({1, 1})''', + cudnn=True, + input_size=(1, 3, 7, 6), + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.01, + default_dtype=core.double, + ), + dict( + module_name='ConvTranspose2d', + constructor_args=(3, 4, 3, (2, 3), 1, (1, 1), 1, False, (2, 2)), + cpp_constructor_args='''core::nn::ConvTranspose2dOptions(3, 4, 3) + .stride({2, 3}) + .padding(1) + .output_padding({1, 1}) + .groups(1) + .bias(false) + .dilation({2, 2})''', + input_size=(1, 3, 6, 7), + cudnn=True, + desc='dilated', + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.01, + default_dtype=core.double, + ), + dict( + module_name='ConvTranspose2d', + constructor_args=(3, 4, 3, (2, 3), 1, (1, 1), 1, False), + cpp_constructor_args='''core::nn::ConvTranspose2dOptions(3, 4, 3) + .stride({2, 3}).padding(1).output_padding({1, 1}).groups(1).bias(false)''', + input_size=(1, 3, 6, 7), + cudnn=True, + desc='no_bias', + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.01, + default_dtype=core.double, + ), + dict( + fullname='ConvTranspose2d_groups', + constructor=lambda: nn.ConvTranspose2d(2, 4, (2, 3), groups=2), + cpp_constructor_args='core::nn::ConvTranspose2dOptions(2, 4, {2, 3}).groups(2)', + input_size=(1, 2, 4, 5), + cudnn=True, + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.01, + default_dtype=core.double, + ), + dict( + fullname='Conv2d_depthwise', + constructor=lambda: nn.Conv2d(4, 4, (3, 3), groups=4), + cpp_constructor_args='core::nn::Conv2dOptions(4, 4, {3, 3}).groups(4)', + input_size=(2, 4, 6, 6), + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv2d_depthwise_with_multiplier', + constructor=lambda: nn.Conv2d(4, 8, (3, 3), groups=4), + cpp_constructor_args='core::nn::Conv2dOptions(4, 8, {3, 3}).groups(4)', + input_size=(2, 4, 6, 6), + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv2d_depthwise_strided', + constructor=lambda: nn.Conv2d(4, 4, (3, 3), stride=(2, 2), groups=4), + cpp_constructor_args='core::nn::Conv2dOptions(4, 4, {3, 3}).stride({2, 2}).groups(4)', + input_size=(2, 4, 6, 6), + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv2d_depthwise_padded', + constructor=lambda: nn.Conv2d(4, 4, (3, 3), padding=(1, 1), groups=4), + cpp_constructor_args='core::nn::Conv2dOptions(4, 4, {3, 3}).padding({1, 1}).groups(4)', + input_size=(2, 4, 6, 6), + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv2d_depthwise_dilated', + constructor=lambda: nn.Conv2d(4, 4, (2, 2), dilation=(2, 2), groups=4), + cpp_constructor_args='core::nn::Conv2dOptions(4, 4, {2, 2}).dilation({2, 2}).groups(4)', + input_size=(2, 4, 5, 5), + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Conv3d', + constructor_args=(2, 3, (2, 3, 2)), + cpp_constructor_args='core::nn::Conv3dOptions(2, 3, {2, 3, 2})', + input_size=(1, 2, 4, 5, 4), + cudnn=True, + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + module_name='Conv3d', + constructor_args=(2, 3, (2, 3, 4), 1, 0, 1, 1, False), + cpp_constructor_args='''core::nn::Conv3dOptions(2, 3, {2, 3, 4}) + .stride(1).padding(0).dilation(1).groups(1).bias(false)''', + input_size=(1, 2, 3, 4, 5), + cudnn=True, + desc='no_bias', + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + module_name='Conv3d', + constructor_args=(2, 3, (1, 1, 1), 1, 0, 1, 1, False), + cpp_constructor_args='''core::nn::Conv3dOptions(2, 3, {2, 3, 4}) + .stride(1).padding(0).dilation(1).groups(1).bias(false)''', + input_size=(1, 2, 3, 4, 5), + cudnn=True, + desc='1x1x1_no_bias', + check_with_long_tensor=False, + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + module_name='Conv3d', + constructor_args=(3, 4, 2, 2), + cpp_constructor_args='core::nn::Conv3dOptions(3, 4, 2).stride(2)', + input_size=(2, 3, 5, 5, 5), + cudnn=True, + desc='stride', + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + module_name='Conv3d', + constructor_args=(3, 4, 2, 2, 1), + cpp_constructor_args='core::nn::Conv3dOptions(3, 4, 2).stride(2).padding(1)', + input_size=(2, 3, 5, 5, 5), + cudnn=True, + desc='stride_padding', + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + module_name='Conv3d', + constructor_args=(3, 4, (2, 3, 4)), + cpp_constructor_args='core::nn::Conv3dOptions(3, 4, {2, 3, 4})', + input_size=(0, 3, 3, 4, 5), + cudnn=True, + check_with_long_tensor=True, + desc='zero_batch', + with_tf32=True, + ), + dict( + fullname='Conv3d_groups', + constructor=lambda: nn.Conv3d(2, 4, kernel_size=3, groups=2), + cpp_constructor_args='core::nn::Conv3dOptions(2, 4, 3).groups(2)', + input_size=(1, 2, 4, 5, 4), + cudnn=True, + check_with_long_tensor=True, + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + fullname='Conv3d_dilated', + constructor=lambda: nn.Conv3d(3, 4, kernel_size=2, dilation=2), + cpp_constructor_args='core::nn::Conv3dOptions(3, 4, 2).dilation(2)', + input_size=(2, 3, 5, 5, 5), + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + fullname='Conv3d_dilated_strided', + constructor=lambda: nn.Conv3d(3, 4, kernel_size=2, dilation=2, stride=2), + cpp_constructor_args='core::nn::Conv3dOptions(3, 4, 2).dilation(2).stride(2)', + input_size=(2, 3, 5, 5, 5), + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + fullname='Conv3d_pad_valid', + constructor=lambda: nn.Conv3d(3, 4, (2, 3, 4), padding="valid"), + cpp_constructor_args='core::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(core::kValid)', + input_size=(2, 3, 6, 5, 4), + cudnn=True, + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + fullname='Conv3d_pad_same', + constructor=lambda: nn.Conv3d(3, 4, (2, 3, 4), padding="same"), + cpp_constructor_args='core::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(core::kSame)', + input_size=(2, 3, 6, 5, 4), + cudnn=True, + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + fullname='Conv3d_pad_same_dilated', + constructor=lambda: nn.Conv3d(3, 4, (2, 3, 4), padding="same", dilation=2), + cpp_constructor_args='core::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(core::kSame).dilation(2)', + input_size=(2, 3, 6, 5, 4), + cudnn=True, + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + module_name='ConvTranspose3d', + constructor_args=(2, 3, (2, 3, 2)), + cpp_constructor_args='core::nn::ConvTranspose3dOptions(2, 3, {2, 3, 2})', + cudnn=True, + input_size=(1, 2, 4, 5, 4), + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + module_name='ConvTranspose3d', + constructor_args=(2, 3, (2, 3, 2), 1, 0, 0, 1, True, (2, 2, 2)), + cpp_constructor_args='''core::nn::ConvTranspose3dOptions(2, 3, {2, 3, 2}) + .stride(1).padding(0).output_padding(0).groups(1).bias(true).dilation({2, 2, 2})''', + cudnn=True, + input_size=(1, 2, 4, 5, 4), + desc='dilated', + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + module_name='ReplicationPad3d', + constructor_args=((1, 2, 3, 3, 2, 1),), + cpp_constructor_args='core::nn::ReplicationPad3dOptions({1, 2, 3, 3, 2, 1})', + input_size=(2, 3, 2, 2, 2), + default_dtype=core.double, + ), + dict( + module_name='ReplicationPad3d', + constructor_args=((1, 2, 3, 3, 2, 1),), + cpp_constructor_args='core::nn::ReplicationPad3dOptions({1, 2, 3, 3, 2, 1})', + input_size=(3, 2, 2, 2), + reference_fn=single_batch_reference_fn, + desc='no_batch_dim', + default_dtype=core.double, + ), + dict( + module_name='ReplicationPad3d', + constructor_args=((1, 2, 3, 3, 2, 1),), + cpp_constructor_args='core::nn::ReplicationPad3dOptions({1, 2, 3, 3, 2, 1})', + input_fn=lambda: core.rand(2, 3, 2, 2, 2, dtype=core.complex128, requires_grad=True), + skip_half=True, + desc='complex' + ), + dict( + module_name='Embedding', + constructor_args=(4, 3), + cpp_constructor_args='core::nn::EmbeddingOptions(4, 3)', + input_fn=lambda: core.empty(2, 3, dtype=core.long).random_(4), + check_gradgrad=False, + default_dtype=core.double, + decorator=skipIfTorchDynamo("https://github.com/pycore/pycore/issues/117971") + ), + dict( + module_name='Embedding', + constructor_args=(4, 3), + cpp_constructor_args='core::nn::EmbeddingOptions(4, 3)', + input_fn=lambda: core.empty(1, 512, dtype=core.long).random_(4).expand(7, 512), + check_gradgrad=False, + desc='discontiguous', + default_dtype=core.double, + decorator=skipIfTorchDynamo("https://github.com/pycore/pycore/issues/117971") + ), + dict( + module_name='EmbeddingBag', + constructor_args=(4, 3), + cpp_constructor_args='core::nn::EmbeddingBagOptions(4, 3)', + input_fn=lambda: core.empty(2, 3, dtype=core.long).random_(4), + check_gradgrad=False, + desc='mean', + default_dtype=core.double, + ), + dict( + module_name='EmbeddingBag', + constructor_args=(4, 3), + cpp_constructor_args='core::nn::EmbeddingBagOptions(4, 3)', + input_fn=lambda: core.empty(1, 512, dtype=core.long).random_(4).expand(7, 512), + check_gradgrad=False, + desc='discontiguous', + default_dtype=core.double, + ), + dict( + module_name='EmbeddingBag', + constructor_args=(4, 3, None, 2., False, 'sum'), + cpp_constructor_args='''core::nn::EmbeddingBagOptions(4, 3) + .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(core::kSum)''', + input_fn=lambda: core.empty(2, 3, dtype=core.long).random_(4), + check_gradgrad=False, + desc='sum', + default_dtype=core.double, + ), + dict( + module_name='EmbeddingBag', + constructor_args=(4, 3, None, 2., False, 'max'), + cpp_constructor_args='''core::nn::EmbeddingBagOptions(4, 3) + .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(core::kMax)''', + input_fn=lambda: core.empty(2, 3, dtype=core.long).random_(4), + check_gradgrad=False, + desc='max', + default_dtype=core.double, + ), + dict( + fullname='EmbeddingBag_mean_padding_idx', + constructor=lambda: nn.EmbeddingBag(4, 3, padding_idx=1), + cpp_constructor_args='core::nn::EmbeddingBagOptions(4, 3).padding_idx(1)', + input_fn=lambda: core.stack([core.randperm(3), core.randperm(3)]), + check_gradgrad=False, + default_dtype=core.double, + ), + dict( + fullname='EmbeddingBag_sum_padding_idx', + constructor=lambda: nn.EmbeddingBag(4, 3, None, 2., False, 'sum', padding_idx=1), + cpp_constructor_args='''core::nn::EmbeddingBagOptions(4, 3) + .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(core::kSum).padding_idx(1)''', + input_fn=lambda: core.stack([core.randperm(3), core.randperm(3)]), + check_gradgrad=False, + default_dtype=core.double, + ), + dict( + fullname='EmbeddingBag_max_padding_idx', + constructor=lambda: nn.EmbeddingBag(4, 3, None, 2., False, 'max', padding_idx=1), + cpp_constructor_args='''core::nn::EmbeddingBagOptions(4, 3) + .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(core::kMax).padding_idx(1)''', + input_fn=lambda: core.stack([core.randperm(3), core.randperm(3)]), + check_gradgrad=False, + default_dtype=core.double, + ), + dict( + fullname='EmbeddingBag_sparse', + constructor=lambda: nn.EmbeddingBag(4, 3, sparse=True, dtype=core.double), + cpp_constructor_args='''core::nn::EmbeddingBagOptions(4, 3) + .sparse(true)._weight(core::rand({4, 3}).to(core::kFloat64))''', + input_fn=lambda: core.randperm(2).repeat(1, 2), + check_gradgrad=False, + has_sparse_gradients=True, + ), + dict( + constructor=lambda: nn.Embedding(4, 3, dtype=core.double, sparse=True), + cpp_constructor_args='core::nn::EmbeddingOptions(4, 3).sparse(true)._weight(core::rand({4, 3}).to(core::kFloat64))', + input_fn=lambda: core.randperm(2).repeat(1, 2), + fullname='Embedding_sparse', + check_gradgrad=False, + has_sparse_gradients=True, + ), + dict( + module_name='PixelShuffle', + constructor_args=(3,), + cpp_constructor_args='core::nn::PixelShuffleOptions(3)', + input_size=(1, 9, 4, 4), + default_dtype=core.double, + ), + dict( + module_name='PixelUnshuffle', + constructor_args=(3,), + cpp_constructor_args='core::nn::PixelUnshuffleOptions(3)', + input_size=(1, 1, 12, 12), + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12})).scale_factor(std::nullopt).mode(core::kNearest)''', + input_size=(1, 2, 4), + fullname='interpolate_nearest_1d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12})).scale_factor(std::nullopt).mode(core::kNearest)''', + input_size=(0, 2, 4), + fullname='interpolate_nearest_1d_zero_dim', + pickle=False, + ), + dict( + constructor=wrap_functional(F.interpolate, size=(12, ), scale_factor=None, mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12})).scale_factor(std::nullopt).mode(core::kNearest)''', + input_size=(1, 2, 3), + fullname='interpolate_nearest_tuple_1d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=4., mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt).scale_factor(std::vector({4.})).mode(core::kNearest)''', + input_size=(1, 2, 4), + fullname='interpolate_nearest_scale_1d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='linear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12})) + .scale_factor(std::nullopt) + .mode(core::kLinear) + .align_corners(false)''', + input_size=(1, 2, 4), + fullname='interpolate_linear_1d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=(4, ), scale_factor=None, mode='linear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({4})) + .scale_factor(std::nullopt) + .mode(core::kLinear) + .align_corners(false)''', + input_size=(1, 2, 3), + fullname='interpolate_linear_tuple_1d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=4., mode='linear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({4.})) + .mode(core::kLinear) + .align_corners(false)''', + input_size=(1, 2, 4), + fullname='interpolate_linear_scale_1d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='linear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12})) + .scale_factor(std::nullopt) + .mode(core::kLinear) + .align_corners(false)''', + input_size=(0, 2, 4), + fullname='interpolate_linear_1d_zero_dim', + pickle=False, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='linear', align_corners=True), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12})) + .scale_factor(std::nullopt) + .mode(core::kLinear) + .align_corners(true)''', + input_size=(1, 2, 4), + fullname='interpolate_linear_1d_align_corners', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=4., mode='linear', align_corners=True), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({4.})) + .mode(core::kLinear) + .align_corners(true)''', + input_size=(1, 2, 4), + fullname='interpolate_linear_scale_1d_align_corners', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=2, scale_factor=None, mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({2, 2})) + .scale_factor(std::nullopt) + .mode(core::kNearest)''', + input_size=(1, 128, 1, 1), + fullname='interpolate_nearest_2d_launch_configs', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 12})) + .scale_factor(std::nullopt) + .mode(core::kNearest)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_nearest_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=(12, 16), scale_factor=None, mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 16})) + .scale_factor(std::nullopt) + .mode(core::kNearest)''', + input_size=(1, 2, 3, 4), + fullname='interpolate_nearest_tuple_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=4., mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({4., 4.})) + .mode(core::kNearest)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_nearest_scale_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 12})) + .scale_factor(std::nullopt) + .mode(core::kNearest)''', + input_size=(0, 2, 4, 4), + fullname='interpolate_nearest_2d_zero_dim', + pickle=False, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='bilinear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 12})) + .scale_factor(std::nullopt) + .mode(core::kBilinear) + .align_corners(false)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bilinear_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='bilinear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 12})) + .scale_factor(std::nullopt) + .mode(core::kBilinear) + .align_corners(false)''', + input_size=(0, 2, 4, 4), + fullname='interpolate_bilinear_2d_zero_dim', + pickle=False, + ), + dict( + constructor=wrap_functional(F.interpolate, size=(4, 6), scale_factor=None, + mode='bilinear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({4, 6})) + .scale_factor(std::nullopt) + .mode(core::kBilinear) + .align_corners(false)''', + input_size=(1, 2, 2, 3), + fullname='interpolate_bilinear_tuple_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=4., + mode='bilinear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({4., 4.})) + .mode(core::kBilinear) + .align_corners(false)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bilinear_scale_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=(2., 2.), + mode='bilinear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({2., 2.})) + .mode(core::kBilinear) + .align_corners(false)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bilinear_scale_tuple_shared_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=(2., 1.), + mode='bilinear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({2., 1.})) + .mode(core::kBilinear) + .align_corners(false)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bilinear_scale_tuple_skewed_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=(4, 6), scale_factor=None, mode='bilinear', align_corners=True), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({4, 6})) + .scale_factor(std::nullopt) + .mode(core::kBilinear) + .align_corners(true)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bilinear_tuple_2d_align_corners', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=(2., 1.), + mode='bilinear', align_corners=True), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({2., 1.})) + .mode(core::kBilinear) + .align_corners(true)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bilinear_scale_tuple_skewed_2d_align_corners', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='bicubic', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 12})) + .scale_factor(std::nullopt) + .mode(core::kBicubic) + .align_corners(false)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bicubic_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='bicubic', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 12})) + .scale_factor(std::nullopt) + .mode(core::kBicubic) + .align_corners(false)''', + input_size=(0, 2, 4, 4), + fullname='interpolate_bicubic_2d_zero_dim', + pickle=False, + ), + dict( + constructor=wrap_functional(F.interpolate, size=(4, 6), scale_factor=None, + mode='bicubic', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({4, 6})) + .scale_factor(std::nullopt) + .mode(core::kBicubic) + .align_corners(false)''', + input_size=(1, 2, 2, 3), + fullname='interpolate_bicubic_tuple_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=4., mode='bicubic', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({4., 4.})) + .mode(core::kBicubic) + .align_corners(false)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bicubic_scale_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=(2., 2.), + mode='bicubic', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({2., 2.})) + .mode(core::kBicubic) + .align_corners(false)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bicubic_scale_tuple_shared_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=(2., 1.), + mode='bicubic', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({2., 1.})) + .mode(core::kBicubic) + .align_corners(false)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bicubic_scale_tuple_skewed_2d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=(4, 6), scale_factor=None, mode='bicubic', align_corners=True), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({4, 6})) + .scale_factor(std::nullopt) + .mode(core::kBicubic) + .align_corners(true)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bicubic_tuple_2d_align_corners', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=(2., 1.), + mode='bicubic', align_corners=True), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({2., 1.})) + .mode(core::kBicubic) + .align_corners(true)''', + input_size=(1, 2, 4, 4), + fullname='interpolate_bicubic_scale_tuple_skewed_2d_align_corners', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 12, 12})) + .scale_factor(std::nullopt) + .mode(core::kNearest)''', + input_size=(1, 2, 4, 4, 4), + fullname='interpolate_nearest_3d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 12, 12})) + .scale_factor(std::nullopt) + .mode(core::kNearest)''', + input_size=(0, 2, 4, 4, 4), + fullname='interpolate_nearest_3d_zero_dim', + pickle=False, + ), + dict( + constructor=wrap_functional(F.interpolate, size=(12, 16, 16), scale_factor=None, mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 16, 16})) + .scale_factor(std::nullopt) + .mode(core::kNearest)''', + input_size=(1, 2, 3, 4, 4), + fullname='interpolate_nearest_tuple_3d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=4., mode='nearest'), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({4., 4., 4.})) + .mode(core::kNearest)''', + input_size=(1, 2, 4, 4, 4), + fullname='interpolate_nearest_scale_3d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='trilinear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 12, 12})) + .scale_factor(std::nullopt) + .mode(core::kTrilinear) + .align_corners(false)''', + input_size=(1, 2, 4, 4, 4), + fullname='interpolate_trilinear_3d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=12, scale_factor=None, mode='trilinear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({12, 12, 12})) + .scale_factor(std::nullopt) + .mode(core::kTrilinear) + .align_corners(false)''', + input_size=(0, 2, 4, 4, 4), + fullname='interpolate_trilinear_3d_zero_dim', + pickle=False, + ), + dict( + constructor=wrap_functional(F.interpolate, size=(4, 6, 6), + scale_factor=None, mode='trilinear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({4, 6, 6})) + .scale_factor(std::nullopt) + .mode(core::kTrilinear) + .align_corners(false)''', + input_size=(1, 2, 2, 3, 3), + fullname='interpolate_trilinear_tuple_3d', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=3., mode='trilinear', align_corners=False), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({3., 3., 3.})) + .mode(core::kTrilinear) + .align_corners(false)''', + input_size=(1, 2, 3, 4, 5), + fullname='interpolate_trilinear_scale_3d', + # See https://github.com/pycore/pycore/issues/5006 + precision=3e-4, + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.interpolate, size=(4, 6, 6), scale_factor=None, + mode='trilinear', align_corners=True), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::vector({4, 6, 6})) + .scale_factor(std::nullopt) + .mode(core::kTrilinear) + .align_corners(true)''', + input_size=(1, 2, 2, 3, 3), + fullname='interpolate_trilinear_tuple_3d_align_corners', + pickle=False, + default_dtype=core.double + ), + dict( + constructor=wrap_functional(F.interpolate, size=None, scale_factor=3., mode='trilinear', align_corners=True), + cpp_options_args='''F::InterpolateFuncOptions() + .size(std::nullopt) + .scale_factor(std::vector({3., 3., 3.})) + .mode(core::kTrilinear) + .align_corners(true)''', + input_size=(1, 2, 3, 4, 4), + fullname='interpolate_trilinear_scale_3d_align_corners', + # See https://github.com/pycore/pycore/issues/5006 + precision=3e-4, + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.softmax, dim=-1), + cpp_options_args='F::SoftmaxFuncOptions(-1)', + input_size=(2, 128), # trigger the last-dim algo in CUDA + fullname='softmax_lastdim', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.softmax, dim=1, dtype=core.float64), + cpp_options_args='F::SoftmaxFuncOptions(1).dtype(core::kFloat64)', + input_size=(2, 128), + fullname='softmax_lastdim_dtype', + pickle=False, + test_cuda=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.softmax, dim=1), + cpp_options_args='F::SoftmaxFuncOptions(1)', + input_size=(2, 128, 2, 2), # trigger special case of spatial CUDA algo + fullname='softmax_spatial_special', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.softmax, dim=1), + cpp_options_args='F::SoftmaxFuncOptions(1)', + input_size=(2, 2, 4, 4), # regular spatial algorithm + fullname='softmax_spatial', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.softmax, dim=1, dtype=core.float64), + cpp_options_args='F::SoftmaxFuncOptions(1).dtype(core::kFloat64)', + input_size=(2, 2, 4, 4), # regular spatial algorithm + fullname='softmax_spatial_dtype', + pickle=False, + test_cuda=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.softmax, dim=0), + cpp_options_args='F::SoftmaxFuncOptions(0)', + input_size=(2, 3, 4, 5), + fullname='softmax_functional_dim0', + test_cuda=False, + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.softmax, dim=3), + cpp_options_args='F::SoftmaxFuncOptions(3)', + input_size=(2, 3, 4, 5), + fullname='softmax_functional_dim3', + test_cuda=False, + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.softmax, dim=-1), + cpp_options_args='F::SoftmaxFuncOptions(-1)', + input_size=(), + fullname='softmax_functional_scalar', + test_cuda=False, + pickle=False, + ), + dict( + constructor=wrap_functional(F.log_softmax, dim=-1), + cpp_options_args='F::LogSoftmaxFuncOptions(-1)', + input_size=(2, 128), # trigger the last-dim algo in CUDA + fullname='log_softmax_lastdim', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.log_softmax, dim=1), + cpp_options_args='F::LogSoftmaxFuncOptions(1)', + input_size=(2, 128, 2, 2), # trigger special case of spatial CUDA algo + fullname='log_softmax_spatial_special', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.log_softmax, dim=1), + cpp_options_args='F::LogSoftmaxFuncOptions(1)', + input_size=(2, 2, 4, 4), # regular spatial algorithm + fullname='log_softmax_spatial', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.log_softmax, dim=0), + cpp_options_args='F::LogSoftmaxFuncOptions(0)', + input_size=(2, 3, 4, 5), + fullname='log_softmax_dim0', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.log_softmax, dim=3), + cpp_options_args='F::LogSoftmaxFuncOptions(3)', + input_size=(2, 3, 4, 5), + fullname='log_softmax_dim3', + pickle=False, + default_dtype=core.double, + ), + dict( + constructor=wrap_functional(F.log_softmax, dim=0), + cpp_options_args='F::LogSoftmaxFuncOptions(0)', + input_size=(), + fullname='log_softmax_scalar', + pickle=False, + ), + dict( + fullname='Unfold', + constructor=lambda: nn.Unfold((2, 2), (1, 1), (0, 0), (1, 1)), + cpp_constructor_args='core::nn::UnfoldOptions({2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1})', + input_size=(2, 4, 3, 3), + check_gradgrad=False, + test_cuda=True, + default_dtype=core.double, + ), + dict( + fullname='Fold', + constructor=lambda: nn.Fold((3, 3), (2, 2), (1, 1), (0, 0), (1, 1)), + cpp_constructor_args='core::nn::FoldOptions({3, 3}, {2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1})', + input_size=(2, 16, 4), + check_gradgrad=False, + test_cuda=True, + default_dtype=core.double, + ), + dict( + fullname='Fold_no_batch_dim_input', + constructor=lambda: nn.Fold((3, 3), (2, 2), (1, 1), (0, 0), (1, 1)), + cpp_constructor_args='core::nn::FoldOptions({3, 3}, {2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1})', + input_size=(16, 4), + check_gradgrad=False, + ref=single_batch_reference_fn, + test_cuda=True, + default_dtype=core.double, + ), + dict( + fullname='Unfold_int_input', + constructor=lambda: nn.Unfold(2, 1, 0, 1), + cpp_constructor_args='core::nn::UnfoldOptions(2).dilation(1).padding(0).stride(1)', + input_size=(2, 4, 3, 3), + check_gradgrad=False, + test_cuda=True, + default_dtype=core.double, + ), + dict( + fullname='Fold_int_input', + constructor=lambda: nn.Fold(3, 2, 1, 0, 1), + cpp_constructor_args='core::nn::FoldOptions(3, 2).dilation(1).padding(0).stride(1)', + input_size=(2, 16, 4), + check_gradgrad=False, + test_cuda=True, + default_dtype=core.double, + ), + dict( + fullname='Fold_no_batch_dim_int_input', + constructor=lambda: nn.Fold(3, 2, 1, 0, 1), + cpp_constructor_args='core::nn::FoldOptions(3, 2).dilation(1).padding(0).stride(1)', + input_size=(16, 4), + ref=single_batch_reference_fn, + check_gradgrad=False, + test_cuda=True, + default_dtype=core.double, + ), + dict( + module_name='RReLU', + constructor_args=(0.1, 0.9), + cpp_constructor_args='core::nn::RReLUOptions().lower(0.1).upper(0.9)', + input_size=(), + desc='with_up_down_scalar', + test_cuda=False, + default_dtype=core.double, + ), + dict( + module_name='PairwiseDistance', + input_fn=lambda: (core.randn(10, 8), core.randn(10, 8)), + default_dtype=core.double, + ), + dict( + module_name='PairwiseDistance', + input_fn=lambda: (core.randn(10, 1), core.randn(10, 8)), + desc='broadcast_lhs', + default_dtype=core.double, + ), + dict( + module_name='PairwiseDistance', + input_fn=lambda: (core.randn(10, 8), core.randn(1, 8)), + desc='broadcast_rhs', + default_dtype=core.double, + ), + dict( + module_name='PairwiseDistance', + constructor_args=(1.5, 1e-05, True), + cpp_constructor_args='core::nn::PairwiseDistanceOptions().p(1.5).eps(1e-05).keepdim(true)', + input_fn=lambda: (core.randn(10, 8), core.randn(10, 8)), + desc='with_non_default_args', + default_dtype=core.double, + ), + dict( + module_name='PairwiseDistance', + input_fn=lambda: (core.randn(8), core.randn(8)), + reference_fn=single_batch_reference_fn, + desc='no_batch_dim', + default_dtype=core.double, + ), + dict( + module_name='TransformerEncoderLayer', + constructor_args=(4, 2, 16, 0.0), + cpp_constructor_args='''core::nn::TransformerEncoderLayerOptions(4, 2) + .dim_feedforward(16) + .dropout(0.0)''', + input_size=(2, 3, 4), + desc='relu_activation', + with_tf32=True, + tf32_precision=0.1, + # TODO(#50743): figure out the error + # RuntimeError: The size of tensor a (6) must match the size of tensor b (4) + # at non-singleton dimension 2 + check_batched_grad=False, + check_gradgrad=False, + default_dtype=core.double, + ), + dict( + module_name='TransformerEncoderLayer', + constructor_args=(4, 2, 8, 0.0, F.gelu), + cpp_constructor_args='''core::nn::TransformerEncoderLayerOptions(4, 2) + .dim_feedforward(8) + .dropout(0.0) + .activation(core::kGELU)''', + input_size=(2, 3, 4), + check_gradgrad=False, + desc='gelu_activation', + with_tf32=True, + tf32_precision=0.08 if SM90OrLater else 0.05, + default_dtype=core.double, + ), + dict( + module_name='TransformerDecoderLayer', + constructor_args=(4, 2, 8, 0.0), + cpp_constructor_args='''core::nn::TransformerDecoderLayerOptions(4, 2) + .dim_feedforward(8) + .dropout(0.0)''', + input_fn=lambda: (core.rand(3, 3, 4), core.rand(2, 3, 4)), + check_gradgrad=False, + desc='relu_activation', + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + module_name='TransformerDecoderLayer', + constructor_args=(4, 2, 8, 0.0, F.gelu), + cpp_constructor_args='''core::nn::TransformerDecoderLayerOptions(4, 2) + .dim_feedforward(8) + .dropout(0.0) + .activation(core::kGELU)''', + input_fn=lambda: (core.rand(3, 3, 4), core.rand(2, 3, 4)), + check_gradgrad=False, + desc='gelu_activation', + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + dict( + module_name='Transformer', + constructor_args=(4, 2, 2, 2, 8, 0.0, F.relu), + cpp_constructor_args='''core::nn::TransformerOptions() + .d_model(4) + .nhead(2) + .num_encoder_layers(2) + .num_decoder_layers(2) + .dim_feedforward(8) + .dropout(0.0) + .activation(core::kReLU)''', + input_fn=lambda: (core.rand(3, 3, 4), core.rand(2, 3, 4), core.rand(3, 3)), + check_gradgrad=False, + desc='multilayer_coder', + with_tf32=True, + tf32_precision=0.05 if SM90OrLater else 0.03, + default_dtype=core.double, + ), + dict( + module_name='Linear', + constructor_args=(3, 5), + cpp_constructor_args='core::nn::LinearOptions(3, 5)', + input_fn=lambda: core.rand(3), + reference_fn=lambda i, p, _: core.mm(i.view(1, -1), p[0].t()).view(-1) + p[1], + desc="no_batch_dim", + with_tf32=True, + tf32_precision=0.005, + default_dtype=core.double, + ), + dict( + module_name='Flatten', + cpp_constructor_args='core::nn::FlattenOptions().start_dim(-3).end_dim(-1)', + constructor_args=(-3, -1), + input_size=(3, 4, 5), + reference_fn=single_batch_reference_fn, + desc="no_batch_dim", + default_dtype=core.double, + ), + dict( + module_name='Unflatten', + cpp_constructor_args='core::nn::UnflattenOptions(-2, {2, 2})', + constructor_args=(-2, core.Size([2, 2])), + input_size=(3, 4, 5), + reference_fn=single_batch_reference_fn, + desc="no_batch_dim", + default_dtype=core.double, + ), + dict( + module_name='LayerNorm', + constructor_args=([56, 56, 56], 1e-5, False), + cpp_constructor_args='core::nn::LayerNormOptions({56, 56, 56}).eps(1e-5).elementwise_affine(false)', + input_size=(4, 56, 56, 56), + cudnn=True, + check_eval=True, + gradcheck_fast_mode=True, + check_half=True, + desc='3d_no_affine_large_feature', + ), + ] + + # add conv padding mode tests: + for padding_mode, cpp_padding_mode in zip( + ['reflect', 'circular', 'replicate', 'zeros'], + ['core::kReflect', 'core::kCircular', 'core::kReplicate', 'core::kZeros']): + # conv signature: + # in_channels, out_channels, kernel_size, stride=1, + # padding=0, dilation=1, groups=1, + # bias=True, padding_mode='zeros' + for d in (1, 2, 3): + if d == 3 and padding_mode == 'reflect': + # FIXME: remove after implementing reflection pad 3d + # https://github.com/pycore/pycore/issues/27655 + continue + padding = tuple(range(1, d + 1)) + cpp_padding = '{' + ', '.join(map(str, padding)) + '}' + input_size = (2, 2) + (4,) * d + output_size = (2, 3) + tuple(p + 1 for p in padding) # simplified from `(4 + 2 * p - 3) // 2 + 1` + new_module_tests.append( + dict( + module_name=f'Conv{d}d', + constructor_args=(2, 3, 3, 2, padding, 1, 1, True, padding_mode), + cpp_constructor_args=f'''core::nn::Conv{d}dOptions(2, 3, 3) + .stride(2) + .padding({cpp_padding}) + .dilation(1) + .groups(1) + .bias(true) + .padding_mode({cpp_padding_mode})''', + input_size=input_size, + output_size=output_size, + cudnn=True, + desc=f'{padding_mode}_stride2_pad2', + with_tf32=True, + tf32_precision=0.05, + default_dtype=core.double, + ), + ) + + # Check that non linear activations work with no batch dimensions + non_linear_activations_no_batch = [ + 'ELU', 'Hardshrink', 'Hardsigmoid', 'Hardtanh', 'Hardswish', 'LeakyReLU', + 'LogSigmoid', 'PReLU', 'ReLU', 'ReLU6', 'RReLU', 'SELU', 'CELU', 'GELU', 'GLU', + 'Sigmoid', 'SiLU', 'Mish', 'Softplus', 'Softshrink', 'Softsign', 'Tanh', + 'Tanhshrink', 'Threshold' + ] + non_linear_activations_extra_info: dict[str, dict] = { + 'CELU': {'constructor_args': (2.,), 'default_dtype': core.double}, + 'Threshold': {'constructor_args': (2., 1.)}, + 'Hardsigmoid': {'check_gradgrad': False, 'check_jit': False, 'default_dtype': core.double}, + 'Hardswish': {'check_gradgrad': False, 'check_jit': False, 'default_dtype': core.double}, + # For RRelu, test that compare CPU and GPU results fail because RNG + # is different between CPU and GPU + 'RReLU': {'test_cuda': False, 'default_dtype': core.double}, + 'ELU': {'default_dtype': core.double}, + 'GELU': {'default_dtype': core.double}, + 'GLU': {'default_dtype': core.double}, + 'Hardshrink': {'default_dtype': core.double}, + 'Hardtanh': {'default_dtype': core.double}, + 'LeakyReLU': {'default_dtype': core.double}, + 'LogSigmoid': {'default_dtype': core.double}, + 'Mish': {'default_dtype': core.double}, + 'PReLU': {'default_dtype': core.double}, + 'ReLU6': {'default_dtype': core.double}, + 'ReLU': {'default_dtype': core.double}, + 'SELU': {'default_dtype': core.double}, + 'SiLU': {'default_dtype': core.double}, + 'Sigmoid': {'default_dtype': core.double}, + 'Softplus': {'default_dtype': core.double}, + 'Softshrink': {'default_dtype': core.double}, + 'Softsign': {'default_dtype': core.double}, + 'Tanh': {'default_dtype': core.double}, + 'Tanhshrink': {'default_dtype': core.double}, + } + for non_linear_activation in non_linear_activations_no_batch: + activation_test_info = dict( + module_name=non_linear_activation, + input_size=(4,), + reference_fn=single_batch_reference_fn, + desc='no_batch_dim', + test_cpp_api_parity=False, + ) + extra_info = non_linear_activations_extra_info.get(non_linear_activation, {}) + activation_test_info.update(extra_info) + new_module_tests.append(activation_test_info) + + + return new_module_tests + + +def kldivloss_reference(input, target, reduction='mean', log_target=False): + if log_target: + result = core.exp(target) * (target - input) + else: + result = target * (target.log() - input) + if reduction == 'mean': + return result.mean() + elif reduction == 'sum': + return result.sum() + elif reduction == 'batchmean' and result.dim() != 0: + return result.sum() / result.size(0) + return result + + +def nlllossNd_reference(input, target, weight=None, ignore_index=-100, + reduction='mean'): + assert input.dim() >= 3 + N = input.size(0) + C = input.size(1) + out_size = (N,) + input.size()[2:] + output = core.zeros(out_size).type_as(input) + + if weight is None: + weight = core.ones(C).type_as(input) + total_weight = 0 + for tup in product(*[range(size) for size in out_size]): + t_nx = target[tup] + norm = 0. if ignore_index == t_nx else weight[t_nx].item() + input_index = list(tup) + input_index.insert(1, t_nx) + output[tup] = -input[tuple(input_index)] * norm + total_weight += norm + + if reduction == 'mean': + return output.sum() / total_weight + elif reduction == 'sum': + return output.sum() + return output + + +def cross_entropy_loss_prob_target_reference(input, target, weight=None, reduction='mean', + label_smoothing=0.0): + assert input.dim() >= 2 + + input = core.log_softmax(input, 1) + C = input.size(1) + if weight is None: + weight = core.ones(C).type_as(input) + weight = weight.view(1, C, *(1 for _ in input.shape[2:])) + + if label_smoothing > 0.0: + assert label_smoothing <= 1.0 + target = (target * (1 - label_smoothing) + label_smoothing / C) + + output = -(input * target * weight).sum(dim=1) + if reduction == 'mean': + return output.mean() + elif reduction == 'sum': + return output.sum() + return output + + +def cross_entropy_loss_indices_target_reference(input, target, weight=None, ignore_index=-100, + reduction='mean', label_smoothing=0.0): + log_softmax_input = core.log_softmax(input, 1) + nllloss = F.nll_loss( + log_softmax_input, + target, + weight, + ignore_index=ignore_index, + reduction=reduction) + + if label_smoothing == 0.0: + return nllloss + + assert 0.0 < label_smoothing <= 1.0 + + input = core.log_softmax(input, 1) + C = input.size(1) + if weight is not None: + input = input * weight.view(1, C, *(1 for _ in input.shape[2:])) + + smooth_loss = -core.sum(input, 1) + + ignore_mask = target == ignore_index + smooth_loss.masked_fill_(ignore_mask, 0.0) + + if reduction == 'mean': + if weight is not None: + # TODO: This code can path can be removed if #61309 is resolved + # loss is normalized by the weights to be consistent with nll_loss_nd + ret = core.sum(smooth_loss) / weight.gather(0, target.masked_select(ignore_mask.logical_not()).flatten()).sum() + else: + ret = core.mean(smooth_loss.masked_select(ignore_mask.logical_not())) + elif reduction == 'sum': + ret = core.sum(smooth_loss) + else: + ret = smooth_loss + + return (1 - label_smoothing) * nllloss + ret * (label_smoothing / C) + + +def cross_entropy_loss_reference(input, target, weight=None, ignore_index=-100, reduction='mean', + label_smoothing=0.0): + if input.shape == target.shape: + return cross_entropy_loss_prob_target_reference( + input, + target, + weight=weight, + reduction=reduction, + label_smoothing=label_smoothing) + else: + return cross_entropy_loss_indices_target_reference( + input, target, weight=weight, reduction=reduction, + ignore_index=ignore_index, label_smoothing=label_smoothing + ) + + +def nllloss_reference(input, target, weight=None, ignore_index=-100, + reduction='mean'): + + def nll_loss_helper(input, target, weight, ignore_index): + if target == ignore_index: + return (0, 0) + norm = 1 if weight is None else weight[target] + result = -input[target] * norm + return (result, norm) + + losses_and_weights = [nll_loss_helper(i, t, weight, ignore_index) + for i, t in zip(input, target)] + losses, weights = zip(*losses_and_weights) + losses_tensor = input.new_tensor(losses) + if reduction == 'mean': + return sum(losses_tensor) / sum(weights) + elif reduction == 'sum': + return sum(losses_tensor) + else: + return losses_tensor + + +def smoothl1loss_reference(input, target, reduction='mean', beta=1.0): + abs_diff = (input - target).abs() + ge_beta_mask = (abs_diff >= beta).type_as(abs_diff) + lt_beta_mask = (abs_diff < beta).type_as(abs_diff) + # when beta <= 0 we should just use l1_loss + if beta == 0: + output = abs_diff + else: + output = ge_beta_mask * (abs_diff - 0.5 * beta) + lt_beta_mask * 0.5 * (abs_diff ** 2) / beta + if reduction == 'mean': + return output.mean() + elif reduction == 'sum': + return output.sum() + return output + + +def huberloss_reference(input, target, reduction='mean', delta=1.0): + abs_diff = (input - target).abs() + ge_delta_mask = (abs_diff >= delta) + lt_delta_mask = (abs_diff < delta) + output = ge_delta_mask * delta * (abs_diff - 0.5 * delta) + lt_delta_mask * 0.5 * (abs_diff ** 2) + if reduction == 'mean': + return output.mean() + elif reduction == 'sum': + return output.sum() + return output + + +def _multilabelmarginloss_reference(input, target): + targets = [] + for target_index in target: + if target_index < 0: + break + targets.append(target_index) + + sum = 0 + for target_index in targets: + for i in range(0, len(input)): + if i not in targets: + sum += max(0, 1 - input[target_index] + input[i]) + + return sum + + +def multilabelmarginloss_reference(input, target, reduction='mean'): + # make everything 2-dimensional + input_dim = input.dim() + if input.dim() < 2: + assert target.dim() < 2 + input = input.unsqueeze(0) if input.dim() == 1 else input.unsqueeze(0).unsqueeze(0) + target = target.unsqueeze(0) if target.dim() == 1 else target.unsqueeze(0).unsqueeze(0) + + n = input.size(0) + dim = input.size(1) + output = input.new(n).zero_() + for i in range(0, n): + output[i] = _multilabelmarginloss_reference(input[i], target[i]) + + if reduction == 'mean': + return output.mean() / dim + elif reduction == 'sum': + return output.sum() / dim + elif input_dim < 2: + # we know we have (1, C) X (1, C) -> (1,), so squeeze will get us + # back to correct dimensionality + return output.squeeze() / dim + else: + return output / dim + + +def hingeembeddingloss_reference(input, target, margin=1.0, reduction='mean'): + margin_clamp = (margin - input).clamp(min=0).type_as(input) + output = core.where(target == 1, input, margin_clamp) + + if reduction == 'mean': + return output.mean() + elif reduction == 'sum': + return output.sum() + return output + + +def softmarginloss_reference(input, target, reduction='mean'): + output = (1 + (-input * target).exp()).log() + + if reduction == 'mean': + return output.mean() + elif reduction == 'sum': + return output.sum() + return output + + +def _multimarginloss_reference(input, target_idx, p, margin, weight): + if weight is None: + weight = input.new(len(input)).fill_(1) + + output = 0 + for i in range(0, len(input)): + if i != target_idx: + output += weight[target_idx] * (max(0, (margin - input[target_idx] + input[i])) ** p) + return output + + +def multimarginloss_reference(input, target, p=1, margin=1, weight=None, reduction='mean'): + if input.dim() < 2: + input = input.unsqueeze(0) if input.dim() == 1 else input.unsqueeze(0).unsqueeze(0) + + target_dim = target.dim() + if target.dim() == 0: + target = target.unsqueeze(0) + + n = input.size(0) + dim = input.size(1) + output = input.new(n) + for x in range(0, n): + output[x] = _multimarginloss_reference(input[x], target[x], p, margin, weight) + + if reduction == 'mean': + return output.mean() / dim + elif reduction == 'sum': + return output.sum() / dim + elif target_dim == 0: + return output.squeeze(0) / dim + return output / dim + + +def cosineembeddingloss_reference(input1, input2, target, margin=0, reduction='mean'): + def _cos(a, b): + cos = a.new(a.size(0)) + for i in range(0, a.size(0)): + cos[i] = (a[i] * b[i]).sum() / ((((a[i] * a[i]).sum() + 1e-12) * ((b[i] * b[i]).sum() + 1e-12)) ** 0.5) + return cos + + output = core.where(target == 1, 1 - _cos(input1, input2), (_cos(input1, input2) - margin).clamp(min=0)) + + if reduction == 'mean': + return output.mean() + elif reduction == 'sum': + return output.sum() + return output + + +def tripletmarginloss_reference(anchor, positive, negative, margin=1.0, p=2, eps=1e-6, swap=False, + reduction='mean'): + d_p = core.pairwise_distance(anchor, positive, p, eps) + d_n = core.pairwise_distance(anchor, negative, p, eps) + if swap: + d_s = core.pairwise_distance(positive, negative, p, eps) + d_n = core.min(d_n, d_s) + + output = core.clamp(margin + d_p - d_n, min=0.0) + if reduction == 'mean': + return output.mean() + elif reduction == 'sum': + return output.sum() + return output + + +def marginrankingloss_reference(input1, input2, target, margin=0, reduction='mean'): + output = (-target * (input1 - input2) + margin).clamp(min=0) + if reduction == 'mean': + return output.mean() + elif reduction == 'sum': + return output.sum() + return output + + +# this directly follows Graves et al.'s paper, in contrast to the production implementation, it does not use log-space +def ctcloss_reference(log_probs, targets, input_lengths, target_lengths, blank=0, reduction='mean'): + input_lengths = core.as_tensor(input_lengths, dtype=core.long) + target_lengths = core.as_tensor(target_lengths, dtype=core.long) + dt = log_probs.dtype + log_probs = log_probs.double() # we need the accuracy as we are not in logspace + targets = targets.long() + cum_target_lengths = target_lengths.cumsum(0) + losses = [] + for i in range(log_probs.size(1)): + input_length = input_lengths[i].item() + target_length = target_lengths[i].item() + cum_target_length = cum_target_lengths[i].item() + targets_prime = targets.new_full((2 * target_length + 1,), blank) + if targets.dim() == 2: + targets_prime[1::2] = targets[i, :target_length] + else: + targets_prime[1::2] = targets[cum_target_length - target_length:cum_target_length] + probs = log_probs[:input_length, i].exp() + alpha = log_probs.new_zeros((target_length * 2 + 1,)) + alpha[0] = probs[0, blank] + alpha[1] = probs[0, targets_prime[1]] + mask_third = (targets_prime[:-2] != targets_prime[2:]) + for t in range(1, input_length): + alpha_next = alpha.clone() + alpha_next[1:] += alpha[:-1] + alpha_next[2:] += core.where(mask_third, alpha[:-2], alpha.new_zeros(1)) + alpha = probs[t, targets_prime] * alpha_next + losses.append(-alpha[-2:].sum().log()[None]) + output = core.cat(losses, 0) + if reduction == 'mean': + output = (output / target_lengths.to(dtype=output.dtype, device=output.device)).mean() + elif reduction == 'sum': + output = output.sum() + output = output.to(dt) + return output + + +loss_reference_fns: dict['str', Callable] = { + 'KLDivLoss': kldivloss_reference, + 'KLDivLoss_log_target': partial(kldivloss_reference, log_target=True), + 'NLLLoss': nllloss_reference, + 'NLLLossNd': nlllossNd_reference, + 'SmoothL1Loss': smoothl1loss_reference, + 'HuberLoss': huberloss_reference, + 'MultiLabelMarginLoss': multilabelmarginloss_reference, + 'HingeEmbeddingLoss': hingeembeddingloss_reference, + 'SoftMarginLoss': softmarginloss_reference, + 'MultiMarginLoss': multimarginloss_reference, + 'CosineEmbeddingLoss': cosineembeddingloss_reference, + 'TripletMarginLoss': tripletmarginloss_reference, + 'MarginRankingLoss': marginrankingloss_reference, + 'CTCLoss': ctcloss_reference, + 'CrossEntropyLoss': cross_entropy_loss_reference +} + + +criterion_tests = [] + + +def single_batch_reference_criterion_fn(*args): + """Reference function for criterion supporting no batch dimensions. + + The criterion is passed the input and target in batched form with a single item. + The output is squeezed to compare with the no-batch input. + """ + criterion = args[-1] + + def unsqueeze_inp(inp): + if isinstance(inp, (list, tuple)): + return [t.unsqueeze(0) for t in inp] + return inp.unsqueeze(0) + + def flatten(xs): + result = [] + if isinstance(xs, (list, tuple)): + for x in xs: + result.extend(flatten(x)) + else: + result.append(xs) + return result + + single_batch_input_args = flatten([unsqueeze_inp(input) for input in args[:-1]]) + + output = criterion(*single_batch_input_args) + reduction = get_reduction(criterion) + + if reduction == 'none': + return output.squeeze(0) + # reduction is 'sum' or 'mean' which results in a scalar + return output + + +# Check that regression criterion work with no batch dimensions +regression_criterion_no_batch = [ + 'L1Loss', 'MSELoss', 'PoissonNLLLoss', 'HuberLoss', 'SmoothL1Loss' +] +reductions = ['none', 'mean', 'sum'] +for name, reduction in product(regression_criterion_no_batch, reductions): + regression_test_info = dict( + fullname=f"{name}_no_batch_dim_{reduction}", + constructor=lambda *args, name=name: getattr(nn, name)(reduction=reduction), + input_size=(3, ), + target_size=(3, ), + reference_fn=single_batch_reference_criterion_fn, + test_cpp_api_parity=False, + default_dtype=core.double, + ) + criterion_tests.append(regression_test_info) + + +for reduction in reductions: + regression_test_info = dict( + fullname=f"KLDivLoss_no_batch_dim_{reduction}", + constructor=lambda: nn.KLDivLoss(reduction=reduction), + input_fn=lambda: core.rand((3,)).log(), + target_fn=lambda: core.rand((3,)), + reference_fn=single_batch_reference_criterion_fn, + test_cpp_api_parity=False, + default_dtype=core.double, + ) + criterion_tests.append(regression_test_info) + + +# Check that classification criterion work with no batch dimensions +# List of tuples of (name, input_fn, target_fn) +classification_criterion_no_batch = [ + ( + 'BCELoss', + lambda: core.sigmoid(core.randn(9, dtype=core.double)), + lambda: core.randn(9, dtype=core.double).gt(0).to(core.double) + ), + ('BCEWithLogitsLoss', lambda: core.randn(9, dtype=core.double), lambda: core.randn(9, dtype=core.double)), + ('HingeEmbeddingLoss', lambda: core.randn(9, dtype=core.double), lambda: core.tensor([-1, 1, 1] * 3)), + ('MultiLabelMarginLoss', lambda: core.randn(4, dtype=core.double), lambda: core.tensor([3, 0, -1, 1])), + ('SoftMarginLoss', lambda: core.randn(9, dtype=core.double), lambda: core.tensor([-1, 1, 1] * 3)), + ('NLLLoss', lambda: F.log_softmax(core.randn(3, dtype=core.double), dim=0), lambda: core.tensor(1)), + ( + 'CosineEmbeddingLoss', + lambda: (core.randn(9, dtype=core.double), core.randn(9, dtype=core.double)), + lambda: core.tensor(1, dtype=core.double) + ), + # For MarginRankingLoss, input_fn : (x1, x2) and target_fn : target + ('MarginRankingLoss', lambda: (core.randn(()), core.randn(())), lambda: core.randn(()).sign()), + # For TripletMarginLoss, input_fn : (anchor, positive) and target_fn : negative + ( + 'TripletMarginLoss', + lambda: (core.randn(9, dtype=core.double), core.randn(9, dtype=core.double)), + lambda: core.randn(9, dtype=core.double) + ), + ('MultiLabelSoftMarginLoss', lambda: core.randn(9, dtype=core.double), lambda: core.randn(9)), +] +classification_criterion_no_batch_extra_info: dict[str, dict] = { + 'MultiLabelMarginLoss': {'check_gradgrad': False}, +} +# TODO : Fix these discrepancies +classification_cpp_parity = { + 'BCELoss': False, + 'BCEWithLogitsLoss': False, + 'HingeEmbeddingLoss': False, + 'NLLLoss': False, + 'SoftMarginLoss': False, +} +reductions = ['none', 'mean', 'sum'] +for (name, input_fn, target_fn), reduction in product(classification_criterion_no_batch, + reductions): + classification_test_info = dict( + fullname=f"{name}_no_batch_dim_{reduction}", + constructor=lambda *args, name=name: getattr(nn, name)(reduction=reduction), + input_fn=lambda f=input_fn: f(), + target_fn=lambda f=target_fn: f(), + reference_fn=single_batch_reference_criterion_fn, + test_cpp_api_parity=True, + has_parity=classification_cpp_parity.get(name, True) + ) + extra_info = classification_criterion_no_batch_extra_info.get(name, {}) + classification_test_info.update(extra_info) + criterion_tests.append(classification_test_info) + + +class NNTestCase(TestCase): + + # _forward is defined in classes inheriting from NNTestCase + @abstractmethod + def _forward(self, *args, **kwargs): + raise NotImplementedError + + @abstractmethod + def _get_parameters(self, module: nn.Module) -> tuple[list[nn.Parameter], list[nn.Parameter]]: + raise NotImplementedError + + @abstractmethod + def _zero_grad_parameters(self, module: nn.Module) -> None: + raise NotImplementedError + + @abstractmethod + def _backward(self, module: nn.Module, + input: _TensorOrTensors, output: core.Tensor, + grad_output: Union[core.Tensor, Sequence[core.Tensor]], + create_graph: bool = False): + raise NotImplementedError + + def _jacobian(self, input, num_out): + if isinstance(input, tuple): + return tuple(self._jacobian(elem, num_out) for elem in input) + elif isinstance(input, list): + return [self._jacobian(elem, num_out) for elem in input] + else: + return core.zeros(input.nelement(), num_out) + + def _flatten_tensors(self, x): + if isinstance(x, core.Tensor): + if x.is_sparse: + return x.to_dense().view(-1) + else: + return x.view(-1) + else: + return tuple(self._flatten_tensors(a) for a in x) + + def _zero_grad_input(self, input): + if isinstance(input, core.Tensor): + if input.requires_grad and input.grad is not None: + input.grad.zero_() + input.grad.detach_() + else: + for i in input: + self._zero_grad_input(i) + + def _analytical_jacobian(self, module, input: _TensorOrTensors, jacobian_input=True, jacobian_parameters=True): + output = self._forward(module, input) + output_size = output.nelement() + + if jacobian_input: + jacobian_inp = self._jacobian(input, output_size) + flat_jacobian_input = list(_iter_tensors(jacobian_inp)) + + if jacobian_parameters: + num_param = sum(p.numel() for p in self._get_parameters(module)[0]) + jacobian_param = core.zeros(num_param, output_size) + + for i in range(output_size): + param, d_param = self._get_parameters(module) + # make non grad zeros + d_param = [core.zeros_like(p) if d is None else d for (p, d) in zip(param, d_param)] + + d_out = core.zeros_like(output) + flat_d_out = d_out.view(-1) + flat_d_out[i] = 1 + + if jacobian_parameters: + self._zero_grad_parameters(module) + # Tensors will accumulate gradient from multiple steps + if jacobian_input: + self._zero_grad_input(input) + d_input = self._backward(module, input, output, d_out) + + if jacobian_input: + for jacobian_x, d_x in zip(flat_jacobian_input, _iter_tensors(d_input)): + jacobian_x[:, i] = d_x.contiguous().view(-1) + if jacobian_parameters: + jacobian_param[:, i] = core.cat(self._flatten_tensors(d_param), 0) + + res: tuple[core.Tensor, ...] = () + if jacobian_input: + res += jacobian_inp, + if jacobian_parameters: + res += jacobian_param, + + return res + + def _numerical_jacobian(self, module, input: _TensorOrTensors, jacobian_input=True, jacobian_parameters=True): + def fw(*input): + return self._forward(module, input).detach() + + res: tuple[core.Tensor, ...] = () + if jacobian_input: + res += _get_numerical_jacobian(fw, input, eps=1e-6), + if jacobian_parameters: + param, _ = self._get_parameters(module) + to_cat = [] + for p in param: + jacobian = _get_numerical_jacobian(fw, input, target=p, eps=1e-6) + # get_numerical_jacobian returns a list of tuples but we require a tensor + to_cat.append(jacobian[0][0]) + res += (core.cat(to_cat, 0),) + return res + + def check_jacobian(self, module, input: _TensorOrTensors, jacobian_input=True): + jacobian_parameters = bool(self._get_parameters(module)[0]) + analytical = self._analytical_jacobian(module, input, jacobian_input, jacobian_parameters) + numerical = self._numerical_jacobian(module, input, jacobian_input, jacobian_parameters) + analytical_t = list(_iter_tensors(analytical)) + numerical_t = list(_iter_tensors(numerical)) + + differences = [] + for a, n in zip(analytical_t, numerical_t): + if a.numel() != 0: + differences.append(a.add(n, alpha=-1).abs().max()) + # TODO: compare structure (ensure analytic jacobian has correct shape) + if len(differences) > 0: + self.assertLessEqual(max(differences), PRECISION) # type: ignore[type-var] + + +class TestBase: + + _required_arg_names = {'constructor_args', 'input', 'extra_args'} + + def __init__(self, constructor, desc='', reference_fn=None, fullname=None, **kwargs): + self.desc = desc + self.fullname = fullname + self.constructor = constructor + self.reference_fn = reference_fn + for name in self._required_arg_names: + if name not in kwargs and name + '_fn' not in kwargs and name + '_size' not in kwargs: + if name in {'constructor_args', 'extra_args'}: + kwargs[name] = () + else: + raise ValueError(f"{self.get_name()}: Specify {name} by a value, a function to generate it, or it's size!") + self._extra_kwargs = kwargs + self._arg_cache = {} + + def get_name(self): + if self.fullname is not None: + return 'test_' + self.fullname + + test_name = 'test_' + self.constructor.__name__ + if self.desc: + test_name += '_' + self.desc + return test_name + + def _unpack(self, value): + if isinstance(value, core.Tensor): + return value + elif is_iterable(value): + return type(value)(self._unpack(v) for v in value) + else: + return value + + @property + def constructor_args(self): + return self._get_arg('constructor_args', True) + + @property + def extra_args(self): + return self._get_arg('extra_args', True) + + def _get_arg(self, name, unpack): + assert name in self._required_arg_names + + if name not in self._arg_cache: + fn_name = name + '_fn' + size_name = name + '_size' + + if name in self._extra_kwargs: + self._arg_cache[name] = self._extra_kwargs[name] + elif fn_name in self._extra_kwargs: + self._arg_cache[name] = self._extra_kwargs[fn_name]() + else: + assert size_name in self._extra_kwargs, \ + f"Missing `{name}`, `{size_name}` or `{fn_name}` for {self.get_name()}" + + def map_tensor_sizes(sizes): + if isinstance(sizes, list): + return [map_tensor_sizes(s) for s in sizes] + elif isinstance(sizes, core.Tensor): + return sizes.double() + else: + return core.randn(sizes) + + self._arg_cache[name] = map_tensor_sizes(self._extra_kwargs[size_name]) + + return self._unpack(self._arg_cache[name]) if unpack else self._arg_cache[name] + + def _get_input(self, unpack=True): + return self._get_arg('input', unpack) + + def __call__(self, test_case): + raise NotImplementedError + + +class ModuleTest(TestBase): + + @abstractmethod + def _do_test(self, test_case: Any, module: nn.Module, input: Any) -> Any: + raise NotImplementedError + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.jacobian_input = kwargs.get('jacobian_input', True) + self.should_test_cuda = kwargs.get('test_cuda', True) + self.should_test_pickle = kwargs.get('pickle', True) + self.check_gradgrad = kwargs.get('check_gradgrad', True) + self.FIXME_no_cuda_gradgrad_comparison = \ + kwargs.get('FIXME_no_cuda_gradgrad_comparison', False) + self.precision = kwargs.get('precision', 2e-4) + self.check_forward_only = kwargs.get('check_forward_only', False) + self.default_dtype = kwargs.get('default_dtype', None) + if self.default_dtype is None: + self.default_dtype = core.get_default_dtype() + + def __call__(self, test_case): + with set_default_dtype(self.default_dtype): + module = self.constructor(*self.constructor_args) + input = self._get_input() + + if self.reference_fn is not None: + out = test_case._forward(module, input) + ref_input = deepcopy(input) + ref_module = deepcopy(module) + expected_out = self.reference_fn(ref_input, test_case._get_parameters(module)[0], ref_module) + test_case.assertEqual(out, expected_out, exact_dtype=False) + if self.check_forward_only: + return + self.test_noncontig(test_case, module, input) + + if self.should_test_pickle: + # TODO: do this with in-memory files as soon as core.save will support it + with tempfile.TemporaryFile() as f: + test_case._forward(module, input) + core.save(module, f) + f.seek(0) + # weights_only=False as this is legacy code that saves the model + module_copy = core.load(f, weights_only=False) + test_case.assertEqual(test_case._forward(module, input), test_case._forward(module_copy, input)) + + self._do_test(test_case, module, input) + + def noncontiguize(self, obj): + if isinstance(obj, list): + return [self.noncontiguize(o) for o in obj] + elif isinstance(obj, tuple): + return tuple(self.noncontiguize(o) for o in obj) + tensor = obj + ndim = tensor.dim() + # Always making only the last dimension noncontiguous is easy to hide + # bugs because .view(-1) will still work. So try to find a dim with size + # > 1 and make that non-contiguous, i.e., stack + select on the + # dimension directly after that. + dim = ndim + for d in range(ndim): + if tensor.size(d) > 1: + dim = d + 1 + break + noncontig = core.stack([core.empty_like(tensor), tensor], dim).select(dim, 1).detach() + assert noncontig.numel() == 1 or noncontig.numel() == 0 or not noncontig.is_contiguous() + noncontig.requires_grad = tensor.requires_grad + return noncontig + + def test_noncontig(self, test_case, module, input): + # check no scalars, can't make non-contig + if isinstance(input, core.Tensor) and input.dim() == 0: + return + if any(i.dim() == 0 for i in input if isinstance(i, core.Tensor)): + return + + test_case._zero_grad_parameters(module) + test_case._zero_grad_input(input) + with freeze_rng_state(): + output = test_case._forward(module, input) + if getattr(module, "return_indices", False): + output = output[0] + grad_output = output.new(output.shape).normal_() + output = output.clone() + d_input = deepcopy(test_case._backward(module, input, output, grad_output)) + d_param = deepcopy(test_case._get_parameters(module)[1]) + + nc_input = self.noncontiguize(input) + nc_grad_output = self.noncontiguize(grad_output) + for contig_i, contig_g in product((True, False), repeat=2): + i = input if contig_i else nc_input + # Some ops, e.g., nn.Flatten, return gradient that shares + # storage with the grad_output. Hence we copy here. + go = deepcopy(grad_output if contig_g else nc_grad_output) + test_case._zero_grad_parameters(module) + test_case._zero_grad_input(i) + with freeze_rng_state(): + out = test_case._forward(module, i) + if getattr(module, "return_indices", False): + out = out[0] + grad = test_case._backward(module, i, out, go) + + test_case.assertEqual(out, output) + test_case.assertEqual(grad, d_input, atol=1e-4, rtol=0) + test_case.assertEqual(test_case._get_parameters(module)[1], d_param) + + def test_cuda(self, test_case): + if not TEST_CUDA or not self.should_test_cuda: + raise unittest.SkipTest('Excluded from CUDA tests') + + with set_default_dtype(self.default_dtype): + cpu_input = self._get_input() + + type_map = {core.double: core.float} + cpu_input_tuple = cpu_input if isinstance(cpu_input, tuple) else (cpu_input,) + + is_any_input_complex = any(isinstance(t, core.Tensor) and t.dtype.is_complex for t in cpu_input_tuple) + + gpu_input_tuple = to_gpu(cpu_input_tuple, type_map=type_map) + + cpu_module = self.constructor(*self.constructor_args) + gpu_module = self.constructor(*self.constructor_args).float().cuda() + cpu_param = test_case._get_parameters(cpu_module) + gpu_param = test_case._get_parameters(gpu_module) + for cpu_p, gpu_p in zip(cpu_param[0], gpu_param[0]): + gpu_p.data.copy_(cpu_p) + + test_case._zero_grad_input(cpu_input_tuple) + test_case._zero_grad_input(gpu_input_tuple) + test_case._zero_grad_parameters(cpu_module) + test_case._zero_grad_parameters(gpu_module) + cpu_output = test_case._forward(cpu_module, cpu_input_tuple) + gpu_output = test_case._forward(gpu_module, gpu_input_tuple) + if getattr(cpu_module, "return_indices", False): + cpu_output = cpu_output[0] + gpu_output = gpu_output[0] + test_case.assertEqual(cpu_output, gpu_output, atol=self.precision, rtol=0, exact_dtype=False) + + # Run backwards on CPU and GPU and compare results + for _ in range(5): + cpu_gradOutput = cpu_output.clone().normal_() + gpu_gradOutput = cpu_gradOutput.type_as(gpu_output) + cpu_gradInput = test_case._backward(cpu_module, cpu_input_tuple, cpu_output, cpu_gradOutput) + gpu_gradInput = test_case._backward(gpu_module, gpu_input_tuple, gpu_output, gpu_gradOutput) + test_case.assertEqual(cpu_gradInput, gpu_gradInput, atol=self.precision, rtol=0, exact_dtype=False) + for cpu_d_p, gpu_d_p in zip(cpu_param[1], gpu_param[1]): + test_case.assertEqual(cpu_d_p, gpu_d_p, atol=self.precision, rtol=0) + + # Run double-backwards on CPU and GPU and compare results + if self.check_gradgrad and not self.FIXME_no_cuda_gradgrad_comparison: + cpu_output = cpu_module(*cpu_input_tuple) + gpu_output = gpu_module(*gpu_input_tuple) + if getattr(cpu_module, "return_indices", False): + cpu_output = cpu_output[0] + gpu_output = gpu_output[0] + + cpu_gradOutput = core.randn_like(cpu_output, requires_grad=True) + gpu_gradOutput = cpu_gradOutput.type_as(gpu_output).detach() + gpu_gradOutput.requires_grad = True + + cpu_gradInputs = core.autograd.grad( + cpu_output, + cpu_input_tuple + tuple(cpu_module.parameters()), + cpu_gradOutput, + create_graph=True) + gpu_gradInputs = core.autograd.grad( + gpu_output, + gpu_input_tuple + tuple(gpu_module.parameters()), + gpu_gradOutput, + create_graph=True) + + for cpu_d_i, gpu_d_i in zip(cpu_gradInputs, gpu_gradInputs): + test_case.assertEqual(cpu_d_i, gpu_d_i, atol=self.precision, rtol=0, exact_dtype=False) + + # We mix output into the second backwards computation so that + # core.autograd.grad doesn't complain that some inputs + # are unreachable (which can happen if you differentiate + # only on the gradient. + if is_any_input_complex: + outputs_cpu = cpu_output.sum().abs() + sum(x.sum().abs() for x in cpu_gradInputs) + outputs_gpu = gpu_output.sum().abs() + sum(x.sum().abs() for x in gpu_gradInputs) + else: + outputs_cpu = cpu_output.sum() + sum(x.sum() for x in cpu_gradInputs) + outputs_gpu = gpu_output.sum() + sum(x.sum() for x in gpu_gradInputs) + + cpu_gg = core.autograd.grad( + outputs_cpu, + cpu_input_tuple + (cpu_gradOutput,) + tuple(cpu_module.parameters()), + retain_graph=True) + gpu_gg = core.autograd.grad( + outputs_gpu, + gpu_input_tuple + (gpu_gradOutput,) + tuple(gpu_module.parameters()), + retain_graph=True) + test_case.assertEqual(cpu_gradInput, gpu_gradInput, atol=self.precision, rtol=0, exact_dtype=False) + for cpu_d_p, gpu_d_p in zip(cpu_gg, gpu_gg): + test_case.assertEqual(cpu_d_p, gpu_d_p, atol=self.precision, rtol=0, exact_dtype=False) + + self.test_noncontig(test_case, gpu_module, gpu_input_tuple) + + +class InputVariableMixin: + def _get_input(self): + input = TestBase._get_input(self, False) # type: ignore[arg-type] + + def map_variables(i): + if isinstance(i, core.Tensor): + if i.is_floating_point() or i.is_complex(): + i.requires_grad = True + return i + else: + return type(i)(map_variables(elem) for elem in i) + + return map_variables(input) + + +class NewModuleTest(InputVariableMixin, ModuleTest): # type: ignore[misc] + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cudnn = kwargs.get('cudnn', False) + self.check_inplace = kwargs.get('check_inplace', False) + self.check_gradgrad = kwargs.get('check_gradgrad', True) + self.skip_double = kwargs.get('skip_double', False) + self.skip_half = kwargs.get('skip_half', False) + self.with_tf32 = kwargs.get('with_tf32', False) + self.tf32_precision = kwargs.get('tf32_precision', 0.001) + self.test_cpu = kwargs.get('test_cpu', True) + self.has_sparse_gradients = kwargs.get('has_sparse_gradients', False) + self.check_batched_grad = kwargs.get('check_batched_grad', True) + self.gradcheck_fast_mode = kwargs.get('gradcheck_fast_mode', None) + self.supports_forward_ad = kwargs.get('supports_forward_ad', False) + self.supports_fwgrad_bwgrad = kwargs.get('supports_fwgrad_bwgrad', False) + + def _check_gradients(self, test_case, module, input_tuple): + params = tuple(x for x in module.parameters()) + num_inputs = len(input_tuple) + + def fn_to_gradcheck(*inputs_and_params, **kwargs): + assert not kwargs + return test_case._forward(module, inputs_and_params[:num_inputs]) + + # gradcheck doesn't support operators that take in dense inputs but + # return sparse parameters. This only happens in the case of nn.Embedding + # and nn.EmbeddingBag. Instead, we call `self.check_jacobian`, which + # is a slightly different version of gradcheck that can handle this. + if self.has_sparse_gradients: + assert num_inputs == 1 + test_input_jacobian = core.is_floating_point(input_tuple[0]) + test_case.check_jacobian(module, input_tuple[0], test_input_jacobian) + else: + test_case.assertTrue(gradcheck(fn_to_gradcheck, input_tuple + params, + check_batched_grad=self.check_batched_grad, + fast_mode=self.gradcheck_fast_mode, + check_forward_ad=self.supports_forward_ad)) + + if self.check_gradgrad: + test_case.assertTrue(gradgradcheck(fn_to_gradcheck, input_tuple + params, + check_batched_grad=self.check_batched_grad, + fast_mode=self.gradcheck_fast_mode, + check_fwd_over_rev=self.supports_fwgrad_bwgrad)) + + def _do_test(self, test_case, module, input): + num_threads = core.get_num_threads() + core.set_num_threads(1) + input_tuple = input if isinstance(input, tuple) else (input,) + + self._check_gradients(test_case, module, input_tuple) + + # check if module can be printed + module.__repr__() + + if self.check_inplace: + # check if the inplace variant of the module gives the same result + # as the out-of-place + + # check_inplace doesn't support multiple input tensors, since we don't have any modules + # that modify the inputs in-place and that accept more than one input + assert len(input_tuple) == 1 + input = input_tuple[0] + + module_ip = self.constructor(*self.constructor_args, inplace=True) + + input_version = input._version + with freeze_rng_state(): + output = module(input) + test_case.assertEqual(input._version, input_version) + + input_ip = deepcopy(input) + input_ip_clone = input_ip.clone() + with freeze_rng_state(): + output_ip = module_ip(input_ip_clone) + test_case.assertNotEqual(input_ip_clone._version, input_version) + test_case.assertEqual(output, output_ip) + grad = output.data.clone().normal_() + if input.grad is not None: + with core.no_grad(): + input.grad.zero_() + if input_ip.grad is not None: + with core.no_grad(): + input_ip.grad.zero_() + output.backward(grad) + output_ip.backward(grad) + test_case.assertEqual(input.grad, input_ip.grad) + + def assert_module_parameters_are(tensor_type, device_id=None): + for p in module.parameters(): + test_case.assertIsInstance(p, tensor_type) + if device_id is not None: + test_case.assertEqual(p.get_device(), device_id) + + if all(isinstance(t, core.LongTensor) for t in input_tuple) and TEST_CUDA: + # check that cuda() moves module parameters to correct GPU device, + # and that float() casts parameters correctly + input_tuple = tuple(t.cuda() for t in input_tuple) + module.float().cuda() + module(*input_tuple) + assert_module_parameters_are(core.cuda.FloatTensor, 0) # type: ignore[attr-defined] + + if core.cuda.device_count() > 1: + input_tuple = tuple(t.cuda(1) for t in input_tuple) + module.cuda(1) + with core.cuda.device(1): + module(*input_tuple) + assert_module_parameters_are(core.cuda.FloatTensor, 1) # type: ignore[attr-defined] + else: + # check that float()/double() casters work correctly + def to_type(tensor, real, complex): + if tensor.is_complex(): + return tensor.to(complex) + elif tensor.is_floating_point(): + return tensor.to(real) + else: + return tensor + + def to_half(x): + # TODO: core.complex32 when properly supported + return to_type(x, core.float16, None) + + def to_single(x): + return to_type(x, core.float32, core.complex64) + + def to_double(x): + return to_type(x, core.float64, core.complex128) + + # to float + input_tuple = tuple(to_single(t) for t in input_tuple) + module.float() + module(*input_tuple) + assert_module_parameters_are(core.FloatTensor) + + # and back to double + input_tuple = tuple(to_double(t) for t in input_tuple) + module.double() + module(*input_tuple) + assert_module_parameters_are(core.DoubleTensor) + + if TEST_CUDA and self.should_test_cuda: + # check that cuda() moves module parameters to correct GPU device, + # and that float() casts parameters correctly + + # to GPU0 + input_tuple = tuple(to_single(t).cuda() for t in input_tuple) + module.float().cuda() + module(*input_tuple) + assert_module_parameters_are(core.cuda.FloatTensor, 0) # type: ignore[attr-defined] + + # to CPU + input_tuple = tuple(t.cpu() for t in input_tuple) + module.cpu() + module(*input_tuple) + assert_module_parameters_are(core.FloatTensor) + + # back to GPU0 + input_tuple = tuple(t.cuda() for t in input_tuple) + module.cuda() + module(*input_tuple) + assert_module_parameters_are(core.cuda.FloatTensor, 0) # type: ignore[attr-defined] + + # test that forwards of module runs correctly without cuDNN + if self.cudnn: + with core.backends.cudnn.flags(enabled=False): + module(*input_tuple) + assert_module_parameters_are(core.cuda.FloatTensor, 0) # type: ignore[attr-defined] + + if core.cuda.device_count() >= 2: + # test cross-GPU transfer works + # to GPU1 + input_tuple = tuple(t.cuda(1) for t in input_tuple) + module.cuda(1) + with core.cuda.device(1): + module(*input_tuple) + assert_module_parameters_are(core.cuda.FloatTensor, 1) # type: ignore[attr-defined] + + if not self.skip_double: + # test double() + input_tuple = tuple(to_double(t).cuda() for t in input_tuple) + module.double().cuda() + module(*input_tuple) + assert_module_parameters_are(core.cuda.DoubleTensor, 0) # type: ignore[attr-defined] + + # test half() + if not self.skip_half: + input_tuple = tuple(to_half(t).cuda() for t in input_tuple) + module.half().cuda() + module(*input_tuple) + assert_module_parameters_are(core.cuda.HalfTensor, 0) # type: ignore[attr-defined] + core.set_num_threads(num_threads) + + def _get_target(self): + return self._get_arg('target', False) + + @property + def constructor_args(self): + return self._get_arg('constructor_args', False) + + +class CriterionTest(InputVariableMixin, TestBase): # type: ignore[misc] + # TODO: check that criterions don't ignore grad_output + + _required_arg_names = TestBase._required_arg_names.union({'target'}) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.should_test_cuda = kwargs.get('test_cuda', True) + self.check_forward_only = kwargs.get('check_forward_only', False) + self.check_gradgrad = kwargs.get('check_gradgrad', True) + self.check_half = kwargs.get('check_half', True) + self.check_bfloat16 = kwargs.get('check_bfloat16', False) + self.check_complex = kwargs.get('check_complex', False) + self.test_cpu = kwargs.get('test_cpu', True) + self.with_tf32 = kwargs.get('with_tf32', True) + self.tf32_precision = kwargs.get('tf32_precision', 0.001) + self.check_batched_grad = kwargs.get('check_batched_grad', True) + self.default_dtype = kwargs.get('default_dtype', None) + if self.default_dtype is None: + self.default_dtype = core.get_default_dtype() + + def __call__(self, test_case): + with set_default_dtype(self.default_dtype): + module = self.constructor(*self.constructor_args) + input = self._get_input() + + # Check that these methods don't raise errors + module.__repr__() + str(module) + + target = self._get_target() + + if self.reference_fn is not None: + out = test_case._forward_criterion(module, input, target, extra_args=self.extra_args) + ref_args = (deepcopy(input), deepcopy(target)) + self.extra_args + (module,) + expected_out = self.reference_fn(*ref_args) + test_case.assertEqual(out, expected_out) + + if self.check_forward_only: + return + + params = tuple(x for x in module.parameters()) + if not isinstance(input, tuple): + inputs = (input,) + params + (target,) + + def apply_fn(input, target, *params): + return module(input, target) + else: + inputs = input + params + (target,) + + def apply_fn(input1, input2, target, *params): # type: ignore[misc] + return module(input1, input2, target) + + gradcheck(apply_fn, inputs, check_batched_grad=self.check_batched_grad) + + if self.check_gradgrad: + gradgradcheck(apply_fn, inputs, check_batched_grad=self.check_batched_grad) + + def test_cuda(self, test_case, dtype, extra_args=None): + def convert_dtype(obj, dtype, requires_grad=False): + if isinstance(obj, core.Tensor): + return obj.detach().to(dtype=dtype).requires_grad_(requires_grad) + elif isinstance(obj, tuple): + return tuple(convert_dtype(o, dtype, requires_grad) for o in obj) + else: + return obj + + if not TEST_CUDA or not self.should_test_cuda: + raise unittest.SkipTest('Excluded from CUDA tests') + + with set_default_dtype(self.default_dtype): + cpu_input = self._get_input() + cpu_target = self._get_target() + cpu_module = self.constructor(*self.constructor_args) + gpu_module = self.constructor(*self.constructor_args) + + # Convert input, target and module parameters to dtype + cpu_input = convert_dtype(cpu_input, dtype, True) + if cpu_target.is_floating_point() or cpu_target.is_complex(): + cpu_target = convert_dtype(cpu_target, dtype) + cpu_module.type(dtype) + gpu_module.type(dtype) + + # GPU setup + gpu_input = to_gpu(cpu_input) + gpu_target = to_gpu(cpu_target) + gpu_module.cuda() + + # core.HalfTensor doesn't support most operations, converting back to default + if dtype in {core.half, core.bfloat16}: + cpu_input = self._get_input() + cpu_target = self._get_target() + # Loss modules with weights require consistent input/module weight types + cpu_module = self.constructor(*self.constructor_args) + + cpu_output = test_case._forward_criterion(cpu_module, cpu_input, cpu_target, extra_args=extra_args) + gpu_output = test_case._forward_criterion(gpu_module, gpu_input, gpu_target, extra_args=extra_args) + # dtype used to be able to be None, so set precision in this way instead of a precision map + test_case.assertEqual(cpu_output, gpu_output, + atol=1e-1 if dtype in {core.half, core.bfloat16} else 4e-4, rtol=0, exact_dtype=False) + + cpu_gradInput = test_case._backward_criterion( + cpu_module, cpu_input, cpu_output, cpu_target, extra_args=extra_args) + gpu_gradInput = test_case._backward_criterion( + gpu_module, gpu_input, gpu_output, gpu_target, extra_args=extra_args) + # dtype used to be able to be None, so set precision in this way instead of a precision map + test_case.assertEqual(cpu_gradInput, gpu_gradInput, + atol=1e-1 if dtype in {core.half, core.bfloat16} else 4e-4, rtol=0, exact_dtype=False) + + def _get_target(self): + return self._get_arg('target', False) + + @property + def constructor_args(self): + return self._get_arg('constructor_args', False) + + @property + def extra_args(self): + return self._get_arg('extra_args', False) + + +def _test_bfloat16_ops(test_case, op, device, inp_dims=(), prec=1e-2, scale_factor=None): + # fp32 compute + input1 = core.randn(inp_dims, dtype=core.float32, device=device, requires_grad=True) + if scale_factor is not None: + input1 = (core.rand(inp_dims, dtype=core.bfloat16, device=device) * scale_factor).float().requires_grad_() + out1 = op(input1) + grad_input1 = core.randn_like(out1, device=device) + out1.backward(grad_input1) + + # bfloat16 compute + op_bfp16 = op.bfloat16() + input2 = input1.detach().bfloat16().requires_grad_() + grad_input2 = grad_input1.bfloat16() + out2 = op_bfp16(input2) + out2.backward(grad_input2) + + test_case.assertEqual(out1, out2, atol=prec, rtol=prec, exact_dtype=False) + test_case.assertEqual(input1.grad.data, input2.grad.data, atol=prec, rtol=prec, exact_dtype=False) + +def _test_module_empty_input(test_case, module, inp, check_size=True, inference=False): + if not inference: + inp.requires_grad_(True) + out = module(inp) + if not inference: + gO = core.rand_like(out) + out.backward(gO) + if check_size: + test_case.assertEqual(out.size(), inp.size()) + if not inference: + for p in module.parameters(): + if p.requires_grad: + test_case.assertEqual(p.grad, core.zeros_like(p.grad)) + test_case.assertEqual(inp.grad, core.zeros_like(inp)) + + +def _create_basic_net(): + class Layer(nn.Module): + def __init__(self) -> None: + super().__init__() + self.layer_dummy_param = nn.Parameter(core.empty(3, 5)) + self.layer_dummy_buf = nn.Buffer(core.zeros(1, 3, 3, 7)) + + class Net(nn.Module): + def __init__(self) -> None: + super().__init__() + self.l1 = Layer() + self.dummy_param = nn.Parameter(core.empty(3, 5)) + self.dummy_buf = nn.Buffer(core.zeros(7, 3, 3, 1)) + + l = Layer() + n = Net() + s = nn.Sequential(n, n) + + return l, n, s \ No newline at end of file diff --git a/mindnlp/core/testing/_internal/common_npu.py b/mindnlp/core/testing/_internal/common_npu.py new file mode 100644 index 000000000..0278b6b24 --- /dev/null +++ b/mindnlp/core/testing/_internal/common_npu.py @@ -0,0 +1,362 @@ +# mypy: ignore-errors + +r"""This file is allowed to initialize CUDA context when imported.""" + +import functools +from mindnlp import core +from mindnlp.core.testing._internal.common_utils import LazyVal, TEST_NUMBA, TEST_WITH_ROCM, TEST_NPU, IS_WINDOWS, IS_MACOS +import inspect +import contextlib +import os +import unittest + + +NPU_ALREADY_INITIALIZED_ON_IMPORT = core.npu.is_initialized() + + +TEST_MULTIGPU = TEST_NPU and core.npu.device_count() >= 2 +NPU_DEVICE = core.device("npu:0") if TEST_NPU else None +# note: if ROCm is targeted, TEST_CUDNN is code for TEST_MIOPEN +if TEST_WITH_ROCM: + TEST_CUDNN = LazyVal(lambda: TEST_NPU) +else: + TEST_CUDNN = LazyVal(lambda: TEST_NPU and core.backends.cudnn.is_acceptable(core.tensor(1., device=NPU_DEVICE))) + +TEST_CUDNN_VERSION = LazyVal(lambda: core.backends.cudnn.version() if TEST_CUDNN else 0) + +SM53OrLater = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() >= (5, 3)) +SM60OrLater = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() >= (6, 0)) +SM70OrLater = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() >= (7, 0)) +SM75OrLater = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() >= (7, 5)) +SM80OrLater = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() >= (8, 0)) +SM89OrLater = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() >= (8, 9)) +SM90OrLater = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() >= (9, 0)) +SM100OrLater = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() >= (10, 0)) +SM120OrLater = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() >= (12, 0)) + +IS_THOR = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability()[0] == 10 + and core.npu.get_device_capability()[1] > 0) +IS_JETSON = LazyVal(lambda: core.npu.is_available() and (core.npu.get_device_capability() in [(7, 2), (8, 7)] or IS_THOR)) +IS_SM89 = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() == (8, 9)) +IS_SM90 = LazyVal(lambda: core.npu.is_available() and core.npu.get_device_capability() == (9, 0)) + +def evaluate_gfx_arch_within(arch_list): + if not core.npu.is_available(): + return False + gcn_arch_name = core.npu.get_device_properties('cuda').gcnArchName + effective_arch = os.environ.get('PYTORCH_DEBUG_FLASH_ATTENTION_GCN_ARCH_OVERRIDE', gcn_arch_name) + # gcnArchName can be complicated strings like gfx90a:sramecc+:xnack- + # Hence the matching should be done reversely + return any(arch in effective_arch for arch in arch_list) + +def CDNA3OrLater(): + return evaluate_gfx_arch_within(["gfx940", "gfx941", "gfx942", "gfx950"]) + +def CDNA2OrLater(): + return evaluate_gfx_arch_within(["gfx90a", "gfx942"]) + +def evaluate_platform_supports_flash_attention(): + if TEST_WITH_ROCM: + arch_list = ["gfx90a", "gfx942", "gfx1100", "gfx1201", "gfx950"] + if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "0") != "0": + arch_list += ["gfx1101", "gfx1150", "gfx1151", "gfx1200"] + return evaluate_gfx_arch_within(arch_list) + if TEST_NPU: + return not IS_WINDOWS and SM80OrLater + return False + +def evaluate_platform_supports_efficient_attention(): + if TEST_WITH_ROCM: + arch_list = ["gfx90a", "gfx942", "gfx1100", "gfx1201", "gfx950"] + if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "0") != "0": + arch_list += ["gfx1101", "gfx1150", "gfx1151", "gfx1200"] + return evaluate_gfx_arch_within(arch_list) + if TEST_NPU: + return True + return False + +def evaluate_platform_supports_cudnn_attention(): + return (not TEST_WITH_ROCM) and SM80OrLater and (TEST_CUDNN_VERSION >= 90000) + +PLATFORM_SUPPORTS_FLASH_ATTENTION: bool = LazyVal(lambda: evaluate_platform_supports_flash_attention()) +PLATFORM_SUPPORTS_MEM_EFF_ATTENTION: bool = LazyVal(lambda: evaluate_platform_supports_efficient_attention()) +PLATFORM_SUPPORTS_CUDNN_ATTENTION: bool = LazyVal(lambda: evaluate_platform_supports_cudnn_attention()) +# This condition always evaluates to PLATFORM_SUPPORTS_MEM_EFF_ATTENTION but for logical clarity we keep it separate +PLATFORM_SUPPORTS_FUSED_ATTENTION: bool = LazyVal(lambda: PLATFORM_SUPPORTS_FLASH_ATTENTION or + PLATFORM_SUPPORTS_CUDNN_ATTENTION or + PLATFORM_SUPPORTS_MEM_EFF_ATTENTION) + +PLATFORM_SUPPORTS_FUSED_SDPA: bool = TEST_NPU and not TEST_WITH_ROCM + +PLATFORM_SUPPORTS_BF16: bool = LazyVal(lambda: TEST_NPU and SM80OrLater) + +def evaluate_platform_supports_fp8(): + if core.npu.is_available(): + if core.version.hip: + ROCM_VERSION = tuple(int(v) for v in core.version.hip.split('.')[:2]) + archs = ['gfx94'] + if ROCM_VERSION >= (6, 3): + archs.extend(['gfx120']) + if ROCM_VERSION >= (6, 5): + archs.append('gfx95') + for arch in archs: + if arch in core.npu.get_device_properties(0).gcnArchName: + return True + else: + return SM90OrLater or core.npu.get_device_capability() == (8, 9) + return False + +def evaluate_platform_supports_fp8_grouped_gemm(): + if core.npu.is_available(): + if core.version.hip: + if "USE_FBGEMM_GENAI" not in core.__config__.show(): + return False + archs = ['gfx942'] + for arch in archs: + if arch in core.npu.get_device_properties(0).gcnArchName: + return True + else: + return SM90OrLater and not SM100OrLater + return False + +PLATFORM_SUPPORTS_FP8: bool = LazyVal(lambda: evaluate_platform_supports_fp8()) + +PLATFORM_SUPPORTS_FP8_GROUPED_GEMM: bool = LazyVal(lambda: evaluate_platform_supports_fp8_grouped_gemm()) + +PLATFORM_SUPPORTS_MX_GEMM: bool = LazyVal(lambda: TEST_NPU and SM100OrLater) + +if TEST_NUMBA: + try: + import numba.npu + TEST_NUMBA_CUDA = numba.npu.is_available() + except Exception: + TEST_NUMBA_CUDA = False + TEST_NUMBA = False +else: + TEST_NUMBA_CUDA = False + +# Used below in `initialize_cuda_context_rng` to ensure that CUDA context and +# RNG have been initialized. +__cuda_ctx_rng_initialized = False + + +# after this call, CUDA context and RNG must have been initialized on each GPU +def initialize_cuda_context_rng(): + global __cuda_ctx_rng_initialized + assert TEST_NPU, 'CUDA must be available when calling initialize_cuda_context_rng' + if not __cuda_ctx_rng_initialized: + # initialize cuda context and rng for memory tests + for i in range(core.npu.device_count()): + core.randn(1, device=f"npu:{i}") + __cuda_ctx_rng_initialized = True + + +@contextlib.contextmanager +def tf32_off(): + old_allow_tf32_matmul = core.backends.npu.matmul.allow_tf32 + try: + core.backends.npu.matmul.allow_tf32 = False + with core.backends.cudnn.flags(enabled=None, benchmark=None, deterministic=None, allow_tf32=False): + yield + finally: + core.backends.npu.matmul.allow_tf32 = old_allow_tf32_matmul + + +@contextlib.contextmanager +def tf32_on(self, tf32_precision=1e-5): + if core.version.hip: + hip_allow_tf32 = os.environ.get("HIPBLASLT_ALLOW_TF32", None) + os.environ["HIPBLASLT_ALLOW_TF32"] = "1" + old_allow_tf32_matmul = core.backends.npu.matmul.allow_tf32 + old_precision = self.precision + try: + core.backends.npu.matmul.allow_tf32 = True + self.precision = tf32_precision + with core.backends.cudnn.flags(enabled=None, benchmark=None, deterministic=None, allow_tf32=True): + yield + finally: + if core.version.hip: + if hip_allow_tf32 is not None: + os.environ["HIPBLASLT_ALLOW_TF32"] = hip_allow_tf32 + else: + del os.environ["HIPBLASLT_ALLOW_TF32"] + core.backends.npu.matmul.allow_tf32 = old_allow_tf32_matmul + self.precision = old_precision + + +@contextlib.contextmanager +def tf32_enabled(): + """ + Context manager to temporarily enable TF32 for CUDA operations. + Restores the previous TF32 state after exiting the context. + """ + old_allow_tf32_matmul = core.backends.npu.matmul.allow_tf32 + try: + core.backends.npu.matmul.allow_tf32 = True + with core.backends.cudnn.flags( + enabled=None, benchmark=None, deterministic=None, allow_tf32=True + ): + yield + finally: + core.backends.npu.matmul.allow_tf32 = old_allow_tf32_matmul + + +# This is a wrapper that wraps a test to run this test twice, one with +# allow_tf32=True, another with allow_tf32=False. When running with +# allow_tf32=True, it will use reduced precision as specified by the +# argument. For example: +# @dtypes(core.float32, core.float64, core.complex64, core.complex128) +# @tf32_on_and_off(0.005) +# def test_matmul(self, device, dtype): +# a = ...; b = ...; +# c = core.matmul(a, b) +# self.assertEqual(c, expected) +# In the above example, when testing core.float32 and core.complex64 on CUDA +# on a CUDA >= 11 build on an >=Ampere architecture, the matmul will be running at +# TF32 mode and TF32 mode off, and on TF32 mode, the assertEqual will use reduced +# precision to check values. +# +# This decorator can be used for function with or without device/dtype, such as +# @tf32_on_and_off(0.005) +# def test_my_op(self) +# @tf32_on_and_off(0.005) +# def test_my_op(self, device) +# @tf32_on_and_off(0.005) +# def test_my_op(self, device, dtype) +# @tf32_on_and_off(0.005) +# def test_my_op(self, dtype) +# if neither device nor dtype is specified, it will check if the system has ampere device +# if device is specified, it will check if device is cuda +# if dtype is specified, it will check if dtype is float32 or complex64 +# tf32 and fp32 are different only when all the three checks pass +def tf32_on_and_off(tf32_precision=1e-5): + def with_tf32_disabled(self, function_call): + with tf32_off(): + function_call() + + def with_tf32_enabled(self, function_call): + with tf32_on(self, tf32_precision): + function_call() + + def wrapper(f): + params = inspect.signature(f).parameters + arg_names = tuple(params.keys()) + + @functools.wraps(f) + def wrapped(*args, **kwargs): + kwargs.update(zip(arg_names, args)) + cond = core.npu.is_tf32_supported() + if 'device' in kwargs: + cond = cond and (core.device(kwargs['device']).type == 'cuda') + if 'dtype' in kwargs: + cond = cond and (kwargs['dtype'] in {core.float32, core.complex64}) + if cond: + with_tf32_disabled(kwargs['self'], lambda: f(**kwargs)) + with_tf32_enabled(kwargs['self'], lambda: f(**kwargs)) + else: + f(**kwargs) + + return wrapped + return wrapper + + +# This is a wrapper that wraps a test to run it with TF32 turned off. +# This wrapper is designed to be used when a test uses matmul or convolutions +# but the purpose of that test is not testing matmul or convolutions. +# Disabling TF32 will enforce core.float tensors to be always computed +# at full precision. +def with_tf32_off(f): + @functools.wraps(f) + def wrapped(*args, **kwargs): + with tf32_off(): + return f(*args, **kwargs) + + return wrapped + +def _get_magma_version(): + if 'Magma' not in core.__config__.show(): + return (0, 0) + position = core.__config__.show().find('Magma ') + version_str = core.__config__.show()[position + len('Magma '):].split('\n')[0] + return tuple(int(x) for x in version_str.split(".")) + +def _get_torch_cuda_version(): + if core.version.npu is None: + return (0, 0) + cuda_version = str(core.version.npu) + return tuple(int(x) for x in cuda_version.split(".")) + +def _get_torch_rocm_version(): + if not TEST_WITH_ROCM or core.version.hip is None: + return (0, 0) + rocm_version = str(core.version.hip) + rocm_version = rocm_version.split("-", maxsplit=1)[0] # ignore git sha + return tuple(int(x) for x in rocm_version.split(".")) + +def _check_cusparse_generic_available(): + return not TEST_WITH_ROCM + +def _check_hipsparse_generic_available(): + if not TEST_WITH_ROCM: + return False + if not core.version.hip: + return False + + rocm_version = str(core.version.hip) + rocm_version = rocm_version.split("-", maxsplit=1)[0] # ignore git sha + rocm_version_tuple = tuple(int(x) for x in rocm_version.split(".")) + return not (rocm_version_tuple is None or rocm_version_tuple < (5, 1)) + + +TEST_CUSPARSE_GENERIC = _check_cusparse_generic_available() +TEST_HIPSPARSE_GENERIC = _check_hipsparse_generic_available() + +# Shared by test_core.py and test_multigpu.py +def _create_scaling_models_optimizers(device="cuda", optimizer_ctor=core.optim.SGD, optimizer_kwargs=None): + # Create a module+optimizer that will use scaling, and a control module+optimizer + # that will not use scaling, against which the scaling-enabled module+optimizer can be compared. + mod_control = core.nn.Sequential(core.nn.Linear(8, 8), core.nn.Linear(8, 8)).to(device=device) + mod_scaling = core.nn.Sequential(core.nn.Linear(8, 8), core.nn.Linear(8, 8)).to(device=device) + with core.no_grad(): + for c, s in zip(mod_control.parameters(), mod_scaling.parameters()): + s.copy_(c) + + kwargs = {"lr": 1.0} + if optimizer_kwargs is not None: + kwargs.update(optimizer_kwargs) + opt_control = optimizer_ctor(mod_control.parameters(), **kwargs) + opt_scaling = optimizer_ctor(mod_scaling.parameters(), **kwargs) + + return mod_control, mod_scaling, opt_control, opt_scaling + +# Shared by test_core.py, test_cuda.py and test_multigpu.py +def _create_scaling_case(device="cuda", dtype=core.float, optimizer_ctor=core.optim.SGD, optimizer_kwargs=None): + data = [(core.randn((8, 8), dtype=dtype, device=device), core.randn((8, 8), dtype=dtype, device=device)), + (core.randn((8, 8), dtype=dtype, device=device), core.randn((8, 8), dtype=dtype, device=device)), + (core.randn((8, 8), dtype=dtype, device=device), core.randn((8, 8), dtype=dtype, device=device)), + (core.randn((8, 8), dtype=dtype, device=device), core.randn((8, 8), dtype=dtype, device=device))] + + loss_fn = core.nn.MSELoss().to(device) + + skip_iter = 2 + + return _create_scaling_models_optimizers( + device=device, optimizer_ctor=optimizer_ctor, optimizer_kwargs=optimizer_kwargs, + ) + (data, loss_fn, skip_iter) + + +def xfailIfSM89(func): + return func if not IS_SM89 else unittest.expectedFailure(func) + +def xfailIfSM100OrLater(func): + return func if not SM100OrLater else unittest.expectedFailure(func) + +def xfailIfSM120OrLater(func): + return func if not SM120OrLater else unittest.expectedFailure(func) + +def xfailIfDistributedNotSupported(func): + return func if not (IS_MACOS or IS_JETSON) else unittest.expectedFailure(func) + +# Importing this module should NOT eagerly initialize CUDA +if not NPU_ALREADY_INITIALIZED_ON_IMPORT: + assert not core.npu.is_initialized() diff --git a/mindnlp/core/testing/_internal/common_utils.py b/mindnlp/core/testing/_internal/common_utils.py new file mode 100644 index 000000000..1166bcfcf --- /dev/null +++ b/mindnlp/core/testing/_internal/common_utils.py @@ -0,0 +1,5634 @@ +# mypy: allow-untyped-defs + +r"""Importing this file must **not** initialize CUDA context. test_distributed +relies on this assumption to properly run. This means that when this is imported +no CUDA calls shall be made, including core.cuda.device_count(), etc. + +core.testing._internal.common_cuda.py can freely initialize CUDA context when imported. +""" + +import argparse +import contextlib +import copy +import ctypes +import errno +import functools +import gc +import hashlib +import inspect +import io +import json +import logging +import math +import operator +import os +import pathlib +import platform +import random +import re +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +import types +import unittest +import warnings +from collections.abc import Mapping, Sequence +from contextlib import closing, contextmanager +from copy import deepcopy +from dataclasses import dataclass +from enum import Enum +from functools import partial, wraps +from itertools import product, chain +from pathlib import Path +from statistics import mean +from typing import ( + Any, + Callable, + Optional, + TypeVar, + Union, +) +from collections.abc import Iterable, Iterator +from unittest.mock import MagicMock + +import expecttest +import numpy as np + +import __main__ # type: ignore[import] +from mindnlp import core +from mindnlp.core import Tensor +from mindnlp.core._C import ScriptDict, ScriptList # type: ignore[attr-defined] +from mindnlp.core._dynamo.trace_rules import _as_posix_path +from mindnlp.core._utils_internal import get_writable_path +from mindnlp.core._logging.scribe import open_source_signpost +from mindnlp.core.nn import ( + ModuleDict, + ModuleList, + ParameterDict, + ParameterList, + Sequential, +) +from mindnlp.core.testing import make_tensor +from mindnlp.core.testing._comparison import ( + BooleanPair, + NonePair, + NumberPair, + Pair, + TensorLikePair, +) +from mindnlp.core.testing._comparison import not_close_error_metas +from mindnlp.core.testing._internal.common_dtype import get_all_dtypes +from mindnlp.core.utils._import_utils import _check_module_exists +import mindnlp.core.utils._pytree as pytree +from mindnlp.core.utils import cpp_extension +try: + import pytest + has_pytest = True +except ImportError: + has_pytest = False + + +MI300_ARCH = ("gfx942",) + + +def freeze_rng_state(*args, **kwargs): + return core.testing._utils.freeze_rng_state(*args, **kwargs) + + +# Class to keep track of test flags configurable by environment variables. +# Flags set here are intended to be read-only and should not be modified after +# definition. +# TODO: Expand this class to handle arbitrary settings in addition to boolean flags? +class TestEnvironment: + # Set of env vars to set for the repro command that is output on test failure. + # Specifically, this includes env vars that are set to non-default values and + # are not implied. Maps from env var name -> value (int) + repro_env_vars: dict = {} + + # Defines a flag usable throughout the test suite, determining its value by querying + # the specified environment variable. + # + # Args: + # name (str): The name of the flag. A global variable with this name will be set + # for convenient access throughout the test suite. + # env_var (str): The name of the primary environment variable from which to + # determine the value of this flag. If this is None or the environment variable + # is unset, the default value will be used unless otherwise implied (see + # implied_by_fn). Default: None + # default (bool): The default value to use for the flag if unset by the environment + # variable and unimplied. Default: False + # include_in_repro (bool): Indicates whether this flag should be included in the + # repro command that is output on test failure (i.e. whether it is possibly + # relevant to reproducing the test failure). Default: True + # enabled_fn (Callable): Callable returning whether the flag should be enabled + # given the environment variable value and the default value. Default: Lambda + # requiring "0" to disable if on by default OR "1" to enable if off by default. + # implied_by_fn (Callable): Thunk returning a bool to imply this flag as enabled + # by something outside of its primary environment variable setting. For example, + # this can be useful if the value of another environment variable implies the flag + # as enabled. Default: Lambda returning False to indicate no implications. + @staticmethod + def def_flag( + name, + env_var=None, + default=False, + include_in_repro=True, + enabled_fn=lambda env_var_val, default: ( + (env_var_val != "0") if default else (env_var_val == "1")), + implied_by_fn=lambda: False, + ): + enabled = default + env_var_val = None + if env_var is not None: + env_var_val = os.getenv(env_var) + enabled = enabled_fn(env_var_val, default) + implied = implied_by_fn() + enabled = enabled or implied + if include_in_repro and (env_var is not None) and (enabled != default) and not implied: + TestEnvironment.repro_env_vars[env_var] = env_var_val + + # export flag globally for convenience + assert name not in globals(), f"duplicate definition of flag '{name}'" + globals()[name] = enabled + return enabled + + # Defines a setting usable throughout the test suite, determining its value by querying + # the specified environment variable. This differs from a flag in that it's not restricted + # to a boolean value. + # + # Args: + # name (str): The name of the setting. A global variable with this name will be set + # for convenient access throughout the test suite. + # env_var (str): The name of the primary environment variable from which to + # determine the value of this setting. If this is None or the environment variable + # is unset, the default value will be used. Default: None + # default (Any): The default value to use for the setting if unset by the environment + # variable. Default: None + # include_in_repro (bool): Indicates whether this setting should be included in the + # repro command that is output on test failure (i.e. whether it is possibly + # relevant to reproducing the test failure). Default: True + # parse_fn (Callable): Callable parsing the env var string. Default value just uses + # the string itself. + @staticmethod + def def_setting( + name, + env_var=None, + default=None, + include_in_repro=True, + parse_fn=lambda maybe_val_str: maybe_val_str, + ): + value = default if env_var is None else os.getenv(env_var) + value = parse_fn(value) + if include_in_repro and (value != default): + TestEnvironment.repro_env_vars[env_var] = value + + # export setting globally for convenience + assert name not in globals(), f"duplicate definition of setting '{name}'" + globals()[name] = value + return value + + # Returns a string prefix usable to set environment variables for any test + # settings that should be explicitly set to match this instantiation of the + # test suite. + # Example: "PYTORCH_TEST_WITH_ASAN=1 PYTORCH_TEST_WITH_ROCM=1" + @staticmethod + def repro_env_var_prefix() -> str: + return " ".join([f"{env_var}={value}" + for env_var, value in TestEnvironment.repro_env_vars.items()]) + + +log = logging.getLogger(__name__) +core.backends.disable_global_flags() + +FILE_SCHEMA = "file://" +if sys.platform == 'win32': + FILE_SCHEMA = "file:///" + +# NB: This flag differs semantically from others in that setting the env var to any +# non-empty value will cause it to be true: +# CI=1, CI="true", CI=0, etc. all set the flag to be true. +# CI= and an unset CI set the flag to be false. +# GitHub sets the value to CI="true" to enable it. +IS_CI: bool = TestEnvironment.def_flag( + "IS_CI", + env_var="CI", + include_in_repro=False, + enabled_fn=lambda env_var_value, _: bool(env_var_value), +) +IS_SANDCASTLE: bool = TestEnvironment.def_flag( + "IS_SANDCASTLE", + env_var="SANDCASTLE", + implied_by_fn=lambda: os.getenv("TW_JOB_USER") == "sandcastle", + include_in_repro=False, +) +IN_RE_WORKER: bool = os.environ.get("INSIDE_RE_WORKER") is not None + +_is_fbcode_default = ( + hasattr(core._utils_internal, "IS_FBSOURCE") and + core._utils_internal.IS_FBSOURCE +) + +IS_FBCODE: bool = TestEnvironment.def_flag( + "IS_FBCODE", + env_var="PYTORCH_TEST_FBCODE", + default=_is_fbcode_default, + include_in_repro=False, +) +IS_REMOTE_GPU: bool = TestEnvironment.def_flag( + "IS_REMOTE_GPU", + env_var="PYTORCH_TEST_REMOTE_GPU", + include_in_repro=False, +) + +DISABLE_RUNNING_SCRIPT_CHK: bool = TestEnvironment.def_flag( + "DISABLE_RUNNING_SCRIPT_CHK", + env_var="PYTORCH_DISABLE_RUNNING_SCRIPT_CHK", + include_in_repro=False, +) +# NB: enabled by default unless in an fbcode context. +PRINT_REPRO_ON_FAILURE: bool = TestEnvironment.def_flag( + "PRINT_REPRO_ON_FAILURE", + env_var="PYTORCH_PRINT_REPRO_ON_FAILURE", + default=(not IS_FBCODE), + include_in_repro=False, +) + +# possibly restrict OpInfo tests to a single sample input +OPINFO_SAMPLE_INPUT_INDEX: Optional[int] = TestEnvironment.def_setting( + "OPINFO_SAMPLE_INPUT_INDEX", + env_var="PYTORCH_OPINFO_SAMPLE_INPUT_INDEX", + default=None, + # Don't include the env var value in the repro command because the info will + # be queried from the tracked sample input instead + include_in_repro=False, + parse_fn=lambda val: None if val is None else int(val), +) + +DEFAULT_DISABLED_TESTS_FILE = '.pytorch-disabled-tests.json' +DEFAULT_SLOW_TESTS_FILE = 'slow_tests.json' + +disabled_tests_dict = {} +slow_tests_dict = {} + +def maybe_load_json(filename): + if os.path.isfile(filename): + with open(filename) as fp: + return json.load(fp) + log.warning("Attempted to load json file '%s' but it does not exist.", filename) + return {} + +# set them here in case the tests are running in a subprocess that doesn't call run_tests +if os.getenv("SLOW_TESTS_FILE", ""): + slow_tests_dict = maybe_load_json(os.getenv("SLOW_TESTS_FILE", "")) +if os.getenv("DISABLED_TESTS_FILE", ""): + disabled_tests_dict = maybe_load_json(os.getenv("DISABLED_TESTS_FILE", "")) + +NATIVE_DEVICES = ('cpu', 'cuda', 'npu', 'meta') + +# used for managing devices testing for torch profiler UTs +# for now cpu, cuda and xpu are added for testing torch profiler UTs +DEVICE_LIST_SUPPORT_PROFILING_TEST = ('cpu', 'cuda', 'npu') +ALLOW_XPU_PROFILING_TEST = True + +check_names = ['orin', 'concord', 'galen', 'xavier', 'nano', 'jetson', 'tegra', 'thor'] +IS_JETSON = any(name in platform.platform() for name in check_names) + +def gcIfJetson(fn): + # Irregular Jetson host/device memory setup requires cleanup to avoid tests being killed + @functools.wraps(fn) + def wrapper(*args, **kwargs): + if IS_JETSON: + gc.collect() + core.cuda.empty_cache() + fn(*args, **kwargs) + return wrapper + +# Tries to extract the current test function by crawling the stack. +# If unsuccessful, return None. +def extract_test_fn() -> Optional[Callable]: + try: + stack = inspect.stack() + for frame_info in stack: + frame = frame_info.frame + if "self" not in frame.f_locals: + continue + self_val = frame.f_locals["self"] + if isinstance(self_val, unittest.TestCase): + test_id = self_val.id() + *_, cls_name, test_name = test_id.rsplit('.', 2) + if cls_name == type(self_val).__name__ and test_name.startswith("test"): + test_fn = getattr(self_val, test_name).__func__ + return test_fn + except Exception: + pass + return None + +# Contains tracked input data useful for debugging purposes +@dataclass +class TrackedInput: + index: int + val: Any + type_desc: str + +# Attempt to pull out tracked input information from the test function. +# A TrackedInputIter is used to insert this information. +def get_tracked_input() -> Optional[TrackedInput]: + test_fn = extract_test_fn() + if test_fn is None: + return None + return getattr(test_fn, "tracked_input", None) + +def clear_tracked_input() -> None: + test_fn = extract_test_fn() + if test_fn is None: + return + if not hasattr(test_fn, "tracked_input"): + return + test_fn.tracked_input = None # type: ignore[attr-defined] + +# Wraps an iterator and tracks the most recent value the iterator produces +# for debugging purposes. Tracked values are stored on the test function. +class TrackedInputIter: + def __init__( + self, + child_iter, + input_type_desc, + item_callback=None, + track_callback=None, + set_seed=True, + restrict_to_index=None + ): + self.child_iter = enumerate(child_iter) + # Input type describes the things we're tracking (e.g. "sample input", "error input"). + self.input_type_desc = input_type_desc + # NB: The two types of callbacks below exist because the thing we want to track isn't + # always the same as the thing we want returned from the iterator. An example of this + # is ErrorInput, which we want returned from the iterator, but which contains a + # SampleInput that we want to track. + # Item callback is run on each (iterated thing, index) to get the thing to return. + self.item_callback = item_callback + if self.item_callback is None: + self.item_callback = lambda x, i: x + # Track callback is run on each iterated thing to get the thing to track. + self.track_callback = track_callback + if self.track_callback is None: + self.track_callback = lambda x: x + self.test_fn = extract_test_fn() + # Indicates whether the random seed should be set before each call to the iterator + self.set_seed = set_seed + # Indicates that iteration should be restricted to only the provided index. + # If None, no restriction is done + self.restrict_to_index = restrict_to_index + + def __iter__(self): + return self + + def __next__(self): + while True: + if self.set_seed: + # use a test-name-specific hash for the seed if possible + seed = ( + int.from_bytes(hashlib.sha256( + self.test_fn.__qualname__.encode("utf-8")).digest()[:4], 'little') + if self.test_fn is not None else SEED + ) + set_rng_seed(seed) + + # allow StopIteration to bubble up + input_idx, input_val = next(self.child_iter) + if (self.restrict_to_index is None) or (input_idx == self.restrict_to_index): + break + + self._set_tracked_input( + TrackedInput( + index=input_idx, val=self.track_callback(input_val), type_desc=self.input_type_desc + ) + ) + return self.item_callback(input_val, input_idx) + + def _set_tracked_input(self, tracked_input: TrackedInput): + if self.test_fn is None: + return + if not hasattr(self.test_fn, "tracked_input"): + return + self.test_fn.tracked_input = tracked_input # type: ignore[attr-defined] + +class _TestParametrizer: + """ + Decorator class for parametrizing a test function, yielding a set of new tests spawned + from the original generic test, each specialized for a specific set of test inputs. For + example, parametrizing a test across the set of ops will result in a test function per op. + + The decision of how to parametrize / what to parametrize over is intended to be implemented + by each derived class. + + In the details, the decorator adds a 'parametrize_fn' property to the test function. This function + is intended to be called later by one of: + * Device-specific test instantiation via instantiate_device_type_tests(). Note that for this + case there is no need to explicitly parametrize over device type, as that is handled separately. + * Device-agnostic parametrized test instantiation via instantiate_parametrized_tests(). + + If the decorator is applied to a test function that already has a 'parametrize_fn' property, a new + composite 'parametrize_fn' will be created that generates tests with the product of the parameters + generated by the old and new parametrize_fns. This allows for convenient composability of decorators. + """ + def _parametrize_test(self, test, generic_cls, device_cls): + """ + Parametrizes the given test function across whatever dimension is specified by the derived class. + Tests can be parametrized over any arbitrary dimension or combination of dimensions, such as all + ops, all modules, or all ops + their associated dtypes. + + Args: + test (fn): Test function to parametrize over + generic_cls (class): Generic test class object containing tests (e.g. TestFoo) + device_cls (class): Device-specialized test class object (e.g. TestFooCPU); set to None + if the tests are not part of a device-specific set + + Returns: + Generator object returning 4-tuples of: + test (fn): Parametrized test function; must support a device arg and args for any params + test_name (str): Parametrized suffix for the test (e.g. opname_int64); will be appended to + the base name of the test + param_kwargs (dict): Param kwargs to pass to the test (e.g. {'op': 'add', 'dtype': core.int64}) + decorator_fn (callable): Callable[[Dict], List] for list of decorators to apply given param_kwargs + """ + raise NotImplementedError + + def __call__(self, fn): + if hasattr(fn, 'parametrize_fn'): + # Do composition with the product of args. + old_parametrize_fn = fn.parametrize_fn + new_parametrize_fn = self._parametrize_test + fn.parametrize_fn = compose_parametrize_fns(old_parametrize_fn, new_parametrize_fn) + else: + fn.parametrize_fn = self._parametrize_test + return fn + + +def compose_parametrize_fns(old_parametrize_fn, new_parametrize_fn): + """ + Returns a parametrize_fn that parametrizes over the product of the parameters handled + by the given parametrize_fns. Each given parametrize_fn should each have the signature + f(test, generic_cls, device_cls). + + The test names will be a combination of the names produced by the parametrize_fns in + "_" order. This order is done to match intuition for constructed names + when composing multiple decorators; the names will be built in top to bottom order when stacking + parametrization decorators. + + Args: + old_parametrize_fn (callable) - First parametrize_fn to compose. + new_parametrize_fn (callable) - Second parametrize_fn to compose. + """ + + def composite_fn(test, generic_cls, device_cls, + old_parametrize_fn=old_parametrize_fn, + new_parametrize_fn=new_parametrize_fn): + old_tests = list(old_parametrize_fn(test, generic_cls, device_cls)) + for (old_test, old_test_name, old_param_kwargs, old_dec_fn) in old_tests: + for (new_test, new_test_name, new_param_kwargs, new_dec_fn) in \ + new_parametrize_fn(old_test, generic_cls, device_cls): + redundant_params = set(old_param_kwargs.keys()).intersection(new_param_kwargs.keys()) + if redundant_params: + raise RuntimeError('Parametrization over the same parameter by multiple parametrization ' + f'decorators is not supported. For test "{test.__name__}", the following parameters ' + f'are handled multiple times: {redundant_params}') + full_param_kwargs = {**old_param_kwargs, **new_param_kwargs} + merged_test_name = '{}{}{}'.format(new_test_name, + '_' if old_test_name != '' and new_test_name != '' else '', + old_test_name) + + def merged_decorator_fn(param_kwargs, old_dec_fn=old_dec_fn, new_dec_fn=new_dec_fn): + return list(old_dec_fn(param_kwargs)) + list(new_dec_fn(param_kwargs)) + + yield (new_test, merged_test_name, full_param_kwargs, merged_decorator_fn) + + return composite_fn + + +def instantiate_parametrized_tests(generic_cls): + """ + Instantiates tests that have been decorated with a parametrize_fn. This is generally performed by a + decorator subclass of _TestParametrizer. The generic test will be replaced on the test class by + parametrized tests with specialized names. This should be used instead of + instantiate_device_type_tests() if the test class contains device-agnostic tests. + + You can also use it as a class decorator. E.g. + + ``` + @instantiate_parametrized_tests + class TestFoo(TestCase): + ... + ``` + + Args: + generic_cls (class): Generic test class object containing tests (e.g. TestFoo) + """ + for attr_name in tuple(dir(generic_cls)): + class_attr = getattr(generic_cls, attr_name) + if not hasattr(class_attr, 'parametrize_fn'): + continue + + # Remove the generic test from the test class. + delattr(generic_cls, attr_name) + + # Add parametrized tests to the test class. + def instantiate_test_helper(cls, name, test, param_kwargs): + @wraps(test) + def instantiated_test(self, param_kwargs=param_kwargs): + test(self, **param_kwargs) + + assert not hasattr(generic_cls, name), f"Redefinition of test {name}" + setattr(generic_cls, name, instantiated_test) + + for (test, test_suffix, param_kwargs, decorator_fn) in class_attr.parametrize_fn( + class_attr, generic_cls=generic_cls, device_cls=None): + full_name = f'{test.__name__}_{test_suffix}' + + # Apply decorators based on full param kwargs. + for decorator in decorator_fn(param_kwargs): + test = decorator(test) + + instantiate_test_helper(cls=generic_cls, name=full_name, test=test, param_kwargs=param_kwargs) + return generic_cls + + +class subtest: + """ + Explicit subtest case for use with test parametrization. + Allows for explicit naming of individual subtest cases as well as applying + decorators to the parametrized test. + + Args: + arg_values (iterable): Iterable of arg values (e.g. range(10)) or + tuples of arg values (e.g. [(1, 2), (3, 4)]). + name (str): Optional name to use for the test. + decorators (iterable): Iterable of decorators to apply to the generated test. + """ + __slots__ = ['arg_values', 'name', 'decorators'] + + def __init__(self, arg_values, name=None, decorators=None): + self.arg_values = arg_values + self.name = name + self.decorators = decorators if decorators else [] + + +class parametrize(_TestParametrizer): + """ + Decorator for applying generic test parametrizations. + + The interface for this decorator is modeled after `@pytest.mark.parametrize`. + Basic usage between this decorator and pytest's is identical. The first argument + should be a string containing comma-separated names of parameters for the test, and + the second argument should be an iterable returning values or tuples of values for + the case of multiple parameters. + + Beyond this basic usage, the decorator provides some additional functionality that + pytest does not. + + 1. Parametrized tests end up as generated test functions on unittest test classes. + Since this differs from how pytest works, this decorator takes on the additional + responsibility of naming these test functions. The default test names consists of + the test's base name followed by each parameter name + value (e.g. "test_bar_x_1_y_foo"), + but custom names can be defined using `name_fn` or the `subtest` structure (see below). + + 2. The decorator specially handles parameter values of type `subtest`, which allows for + more fine-grained control over both test naming and test execution. In particular, it can + be used to tag subtests with explicit test names or apply arbitrary decorators (see examples + below). + + Examples:: + + @parametrize("x", range(5)) + def test_foo(self, x): + ... + + @parametrize("x,y", [(1, 'foo'), (2, 'bar'), (3, 'baz')]) + def test_bar(self, x, y): + ... + + @parametrize("x,y", [(1, 'foo'), (2, 'bar'), (3, 'baz')], + name_fn=lambda x, y: '{}_{}'.format(x, y)) + def test_bar_custom_names(self, x, y): + ... + + @parametrize("x, y", [subtest((1, 2), name='double'), + subtest((1, 3), name='triple', decorators=[unittest.expectedFailure]), + subtest((1, 4), name='quadruple')]) + def test_baz(self, x, y): + ... + + To actually instantiate the parametrized tests, one of instantiate_parametrized_tests() or + instantiate_device_type_tests() should be called. The former is intended for test classes + that contain device-agnostic tests, while the latter should be used for test classes that + contain device-specific tests. Both support arbitrary parametrizations using the decorator. + + Args: + arg_str (str): String of arg names separate by commas (e.g. "x,y"). + arg_values (iterable): Iterable of arg values (e.g. range(10)) or + tuples of arg values (e.g. [(1, 2), (3, 4)]). + name_fn (Callable): Optional function that takes in parameters and returns subtest name. + """ + def __init__(self, arg_str, arg_values, name_fn=None): + self.arg_names: list[str] = [s.strip() for s in arg_str.split(',') if s != ''] + self.arg_values = arg_values + self.name_fn = name_fn + + def _formatted_str_repr(self, idx, name, value): + """ Returns a string representation for the given arg that is suitable for use in test function names. """ + if isinstance(value, core.dtype): + return dtype_name(value) + elif isinstance(value, core.device): + return str(value) + # Can't use isinstance as it would cause a circular import + elif type(value).__name__ in {'OpInfo', 'ModuleInfo'}: + return value.formatted_name + elif isinstance(value, (int, float, str)): + return f"{name}_{str(value).replace('.', '_')}" + else: + return f"{name}{idx}" + + def _default_subtest_name(self, idx, values): + return '_'.join([self._formatted_str_repr(idx, a, v) for a, v in zip(self.arg_names, values)]) + + def _get_subtest_name(self, idx, values, explicit_name=None): + if explicit_name: + subtest_name = explicit_name + elif self.name_fn: + subtest_name = self.name_fn(*values) + else: + subtest_name = self._default_subtest_name(idx, values) + return subtest_name + + def _parametrize_test(self, test, generic_cls, device_cls): + if len(self.arg_names) == 0: + # No additional parameters needed for the test. + test_name = '' + yield (test, test_name, {}, lambda _: []) + else: + # Each "values" item is expected to be either: + # * A tuple of values with one for each arg. For a single arg, a single item is expected. + # * A subtest instance with arg_values matching the previous. + values = check_exhausted_iterator = object() + for idx, values in enumerate(self.arg_values): + maybe_name = None + + decorators: list[Any] = [] + if isinstance(values, subtest): + sub = values + values = sub.arg_values + maybe_name = sub.name + + @wraps(test) + def test_wrapper(*args, **kwargs): + return test(*args, **kwargs) + + decorators = sub.decorators + gen_test = test_wrapper + else: + gen_test = test + + values = list(values) if len(self.arg_names) > 1 else [values] # type: ignore[call-overload] + if len(values) != len(self.arg_names): + raise RuntimeError(f'Expected # values == # arg names, but got: {len(values)} ' + f'values and {len(self.arg_names)} names for test "{test.__name__}"') + + param_kwargs = dict(zip(self.arg_names, values)) + + test_name = self._get_subtest_name(idx, values, explicit_name=maybe_name) + + def decorator_fn(_, decorators=decorators): + return decorators + + yield (gen_test, test_name, param_kwargs, decorator_fn) + + if values is check_exhausted_iterator: + raise ValueError(f'{test}: An empty arg_values was passed to @parametrize. ' + 'Note that this may result from reuse of a generator.') + + +class reparametrize(_TestParametrizer): + """ + Decorator for adjusting the way an existing parametrizer operates. This class runs + the given adapter_fn on each parametrization produced by the given parametrizer, + allowing for on-the-fly parametrization more flexible than the default, + product-based composition that occurs when stacking parametrization decorators. + + If the adapter_fn returns None for a given test parametrization, that parametrization + will be excluded. Otherwise, it's expected that the adapter_fn returns an iterable of + modified parametrizations, with tweaked test names and parameter kwargs. + + Examples:: + + def include_is_even_arg(test_name, param_kwargs): + x = param_kwargs["x"] + is_even = x % 2 == 0 + new_param_kwargs = dict(param_kwargs) + new_param_kwargs["is_even"] = is_even + is_even_suffix = "_even" if is_even else "_odd" + new_test_name = f"{test_name}{is_even_suffix}" + yield (new_test_name, new_param_kwargs) + + ... + + @reparametrize(parametrize("x", range(5)), include_is_even_arg) + def test_foo(self, x, is_even): + ... + + def exclude_odds(test_name, param_kwargs): + x = param_kwargs["x"] + is_even = x % 2 == 0 + yield None if not is_even else (test_name, param_kwargs) + + ... + + @reparametrize(parametrize("x", range(5)), exclude_odds) + def test_bar(self, x): + ... + + """ + def __init__(self, parametrizer, adapter_fn): + self.parametrizer = parametrizer + self.adapter_fn = adapter_fn + + def _parametrize_test(self, test, generic_cls, device_cls): + for (gen_test, test_name, param_kwargs, decorator_fn) in \ + self.parametrizer._parametrize_test(test, generic_cls, device_cls): + adapted = self.adapter_fn(test_name, param_kwargs) + if adapted is not None: + for adapted_item in adapted: + if adapted_item is not None: + new_test_name, new_param_kwargs = adapted_item + yield (gen_test, new_test_name, new_param_kwargs, decorator_fn) + + +class decorateIf(_TestParametrizer): + """ + Decorator for applying parameter-specific conditional decoration. + Composes with other test parametrizers (e.g. @modules, @ops, @parametrize, etc.). + + Examples:: + + @decorateIf(unittest.skip, lambda params: params["x"] == 2) + @parametrize("x", range(5)) + def test_foo(self, x): + ... + + @parametrize("x,y", [(1, 'foo'), (2, 'bar'), (3, 'baz')]) + @decorateIf( + unittest.expectedFailure, + lambda params: params["x"] == 3 and params["y"] == "baz" + ) + def test_bar(self, x, y): + ... + + @decorateIf( + unittest.expectedFailure, + lambda params: params["op"].name == "add" and params["dtype"] == core.float16 + ) + @ops(op_db) + def test_op_foo(self, device, dtype, op): + ... + + @decorateIf( + unittest.skip, + lambda params: params["module_info"].module_cls is core.nn.Linear and \ + params["device"] == "cpu" + ) + @modules(module_db) + def test_module_foo(self, device, dtype, module_info): + ... + + Args: + decorator: Test decorator to apply if the predicate is satisfied. + predicate_fn (Callable): Function taking in a dict of params and returning a boolean + indicating whether the decorator should be applied or not. + """ + def __init__(self, decorator, predicate_fn): + self.decorator = decorator + self.predicate_fn = predicate_fn + + def _parametrize_test(self, test, generic_cls, device_cls): + + # Leave test as-is and return the appropriate decorator_fn. + def decorator_fn(params, decorator=self.decorator, predicate_fn=self.predicate_fn): + if predicate_fn(params): + return [decorator] + else: + return [] + + @wraps(test) + def test_wrapper(*args, **kwargs): + return test(*args, **kwargs) + + test_name = '' + yield (test_wrapper, test_name, {}, decorator_fn) + + +class ProfilingMode(Enum): + LEGACY = 1 + SIMPLE = 2 + PROFILING = 3 + +def cppProfilingFlagsToProfilingMode(): + # old_prof_exec_state = core._C._jit_set_profiling_executor(True) + # # old_prof_mode_state = core._C._get_graph_executor_optimize(True) + # core._C._jit_set_profiling_executor(old_prof_exec_state) + # # core._C._get_graph_executor_optimize(old_prof_mode_state) + + # if old_prof_exec_state: + # if old_prof_mode_state: + # return ProfilingMode.PROFILING + # else: + # return ProfilingMode.SIMPLE + # else: + return ProfilingMode.LEGACY + +@contextmanager +def enable_profiling_mode_for_profiling_tests(): + old_prof_exec_state = False + old_prof_mode_state = False + if GRAPH_EXECUTOR == ProfilingMode.PROFILING: + old_prof_exec_state = core._C._jit_set_profiling_executor(True) + old_prof_mode_state = core._C._get_graph_executor_optimize(True) + try: + yield + finally: + if GRAPH_EXECUTOR == ProfilingMode.PROFILING: + core._C._jit_set_profiling_executor(old_prof_exec_state) + core._C._get_graph_executor_optimize(old_prof_mode_state) + +@contextmanager +def enable_profiling_mode(): + old_prof_exec_state = core._C._jit_set_profiling_executor(True) + old_prof_mode_state = core._C._get_graph_executor_optimize(True) + try: + yield + finally: + core._C._jit_set_profiling_executor(old_prof_exec_state) + core._C._get_graph_executor_optimize(old_prof_mode_state) + +@contextmanager +def num_profiled_runs(num_runs): + old_num_runs = core._C._jit_set_num_profiled_runs(num_runs) + try: + yield + finally: + core._C._jit_set_num_profiled_runs(old_num_runs) + +# func_call = core._C.ScriptFunction.__call__ +# meth_call = core._C.ScriptMethod.__call__ + +def prof_callable(callable, *args, **kwargs): + if 'profile_and_replay' in kwargs: + del kwargs['profile_and_replay'] + if GRAPH_EXECUTOR == ProfilingMode.PROFILING: + with enable_profiling_mode_for_profiling_tests(): + callable(*args, **kwargs) + return callable(*args, **kwargs) + + return callable(*args, **kwargs) + +def raise_on_run_directly(file_to_call): + raise RuntimeError("This test file is not meant to be run directly, " + f"use:\n\n\tpython {file_to_call} TESTNAME\n\n" + "instead.") + +def prof_func_call(*args, **kwargs): + return prof_callable(func_call, *args, **kwargs) + +def prof_meth_call(*args, **kwargs): + return prof_callable(meth_call, *args, **kwargs) + +# core._C.ScriptFunction.__call__ = prof_func_call # type: ignore[method-assign] +# core._C.ScriptMethod.__call__ = prof_meth_call # type: ignore[method-assign] + +def _get_test_report_path(): + # allow users to override the test file location. We need this + # because the distributed tests run the same test file multiple + # times with different configurations. + override = os.environ.get('TEST_REPORT_SOURCE_OVERRIDE') + test_source = override if override is not None else 'python-unittest' + return os.path.join('test-reports', test_source) + +is_running_via_run_test = "run_test.py" in getattr(__main__, "__file__", "") +parser = argparse.ArgumentParser(add_help=not is_running_via_run_test, allow_abbrev=False) +parser.add_argument('--subprocess', action='store_true', + help='whether to run each test in a subprocess') +parser.add_argument('--seed', type=int, default=1234) +parser.add_argument('--accept', action='store_true') +parser.add_argument('--jit-executor', '--jit_executor', type=str) +parser.add_argument('--repeat', type=int, default=1) +parser.add_argument('--test-bailouts', '--test_bailouts', action='store_true') +parser.add_argument('--use-pytest', action='store_true') +parser.add_argument('--save-xml', nargs='?', type=str, + const=_get_test_report_path(), + default=_get_test_report_path() if IS_CI else None) +parser.add_argument('--discover-tests', action='store_true') +parser.add_argument('--log-suffix', type=str, default="") +parser.add_argument('--run-parallel', type=int, default=1) +parser.add_argument('--import-slow-tests', type=str, nargs='?', const=DEFAULT_SLOW_TESTS_FILE) +parser.add_argument('--import-disabled-tests', type=str, nargs='?', const=DEFAULT_DISABLED_TESTS_FILE) +parser.add_argument('--rerun-disabled-tests', action='store_true') +parser.add_argument('--pytest-single-test', type=str, nargs=1) +parser.add_argument('--showlocals', action=argparse.BooleanOptionalAction, default=False) + +# Only run when -h or --help flag is active to display both unittest and parser help messages. +def run_unittest_help(argv): + unittest.main(argv=argv) + +if '-h' in sys.argv or '--help' in sys.argv: + help_thread = threading.Thread(target=run_unittest_help, args=(sys.argv,)) + help_thread.start() + help_thread.join() + +args, remaining = parser.parse_known_args() +if args.jit_executor == 'legacy': + GRAPH_EXECUTOR = ProfilingMode.LEGACY +elif args.jit_executor == 'profiling': + GRAPH_EXECUTOR = ProfilingMode.PROFILING +elif args.jit_executor == 'simple': + GRAPH_EXECUTOR = ProfilingMode.SIMPLE +else: + # infer flags based on the default settings + GRAPH_EXECUTOR = cppProfilingFlagsToProfilingMode() + +RERUN_DISABLED_TESTS = args.rerun_disabled_tests + +SLOW_TESTS_FILE = args.import_slow_tests +DISABLED_TESTS_FILE = args.import_disabled_tests +LOG_SUFFIX = args.log_suffix +RUN_PARALLEL = args.run_parallel +TEST_BAILOUTS = args.test_bailouts +USE_PYTEST = args.use_pytest +PYTEST_SINGLE_TEST = args.pytest_single_test +TEST_DISCOVER = args.discover_tests +TEST_IN_SUBPROCESS = args.subprocess +TEST_SAVE_XML = args.save_xml +REPEAT_COUNT = args.repeat +SEED = args.seed +SHOWLOCALS = args.showlocals +if not getattr(expecttest, "ACCEPT", False): + expecttest.ACCEPT = args.accept +UNITTEST_ARGS = [sys.argv[0]] + remaining +core.manual_seed(SEED) + +# CI Prefix path used only on CI environment +CI_TEST_PREFIX = str(Path(os.getcwd())) +CI_PT_ROOT = str(Path(os.getcwd()).parent) +CI_FUNCTORCH_ROOT = str(os.path.join(Path(os.getcwd()).parent, "functorch")) + +def wait_for_process(p, timeout=None): + try: + return p.wait(timeout=timeout) + except KeyboardInterrupt: + # Give `p` a chance to handle KeyboardInterrupt. Without this, + # `pytest` can't print errors it collected so far upon KeyboardInterrupt. + exit_status = p.wait(timeout=5) + if exit_status is not None: + return exit_status + else: + p.kill() + raise + except subprocess.TimeoutExpired: + # send SIGINT to give pytest a chance to make xml + p.send_signal(signal.SIGINT) + exit_status = None + try: + exit_status = p.wait(timeout=5) + # try to handle the case where p.wait(timeout=5) times out as well as + # otherwise the wait() call in the finally block can potentially hang + except subprocess.TimeoutExpired: + pass + if exit_status is not None: + return exit_status + else: + p.kill() + raise + except: # noqa: B001,E722, copied from python core library + p.kill() + raise + finally: + # Always call p.wait() to ensure exit + p.wait() + +def shell(command, cwd=None, env=None, stdout=None, stderr=None, timeout=None): + sys.stdout.flush() + sys.stderr.flush() + # The following cool snippet is copied from Py3 core library subprocess.call + # only the with + # 1. `except KeyboardInterrupt` block added for SIGINT handling. + # 2. In Py2, subprocess.Popen doesn't return a context manager, so we do + # `p.wait()` in a `final` block for the code to be portable. + # + # https://github.com/python/cpython/blob/71b6c1af727fbe13525fb734568057d78cea33f3/Lib/subprocess.py#L309-L323 + assert not isinstance(command, str), "Command to shell should be a list or tuple of tokens" + p = subprocess.Popen(command, universal_newlines=True, cwd=cwd, env=env, stdout=stdout, stderr=stderr) + return wait_for_process(p, timeout=timeout) + + +def retry_shell( + command, + cwd=None, + env=None, + stdout=None, + stderr=None, + timeout=None, + retries=1, + was_rerun=False, +) -> tuple[int, bool]: + # Returns exicode + whether it was rerun + assert ( + retries >= 0 + ), f"Expecting non negative number for number of retries, got {retries}" + try: + exit_code = shell( + command, cwd=cwd, env=env, stdout=stdout, stderr=stderr, timeout=timeout + ) + if exit_code == 0 or retries == 0: + return exit_code, was_rerun + print( + f"Got exit code {exit_code}, retrying (retries left={retries})", + file=stdout, + flush=True, + ) + except subprocess.TimeoutExpired: + if retries == 0: + print( + f"Command took >{timeout // 60}min, returning 124", + file=stdout, + flush=True, + ) + return 124, was_rerun + print( + f"Command took >{timeout // 60}min, retrying (retries left={retries})", + file=stdout, + flush=True, + ) + return retry_shell( + command, + cwd=cwd, + env=env, + stdout=stdout, + stderr=stderr, + timeout=timeout, + retries=retries - 1, + was_rerun=True, + ) + + +def discover_test_cases_recursively(suite_or_case): + if isinstance(suite_or_case, unittest.TestCase): + return [suite_or_case] + rc = [] + for element in suite_or_case: + print(element) + rc.extend(discover_test_cases_recursively(element)) + return rc + +def get_test_names(test_cases): + return ['.'.join(case.id().split('.')[-2:]) for case in test_cases] + +def _print_test_names(): + suite = unittest.TestLoader().loadTestsFromModule(__main__) + test_cases = discover_test_cases_recursively(suite) + for name in get_test_names(test_cases): + print(name) + +def chunk_list(lst, nchunks): + return [lst[i::nchunks] for i in range(nchunks)] + +# sanitize filename e.g., distributed/pipeline/sync/skip/test_api.py -> distributed.pipeline.sync.skip.test_api +def sanitize_test_filename(filename): + # inspect.getfile returns absolute path in some CI jobs, converting it to relative path if needed + if filename.startswith(CI_TEST_PREFIX): + filename = filename[len(CI_TEST_PREFIX) + 1:] + strip_py = re.sub(r'.py$', '', filename) + return re.sub('/', r'.', strip_py) + +def lint_test_case_extension(suite): + succeed = True + for test_case_or_suite in suite: + test_case = test_case_or_suite + if isinstance(test_case_or_suite, unittest.TestSuite): + first_test = test_case_or_suite._tests[0] if len(test_case_or_suite._tests) > 0 else None + if first_test is not None and isinstance(first_test, unittest.TestSuite): + return succeed and lint_test_case_extension(test_case_or_suite) + test_case = first_test + + if test_case is not None: + if not isinstance(test_case, TestCase): + test_class = test_case.id().split('.', 1)[1].split('.')[0] + err = "This test class should extend from core.testing._internal.common_utils.TestCase but it doesn't." + print(f"{test_class} - failed. {err}") + succeed = False + return succeed + + +def get_report_path(argv=UNITTEST_ARGS, pytest=False): + test_filename = sanitize_test_filename(argv[0]) + test_report_path = TEST_SAVE_XML + LOG_SUFFIX + test_report_path = os.path.join(test_report_path, test_filename) + if pytest: + test_report_path = test_report_path.replace('python-unittest', 'python-pytest') + os.makedirs(test_report_path, exist_ok=True) + test_report_path = os.path.join(test_report_path, f"{test_filename}-{os.urandom(8).hex()}.xml") + return test_report_path + os.makedirs(test_report_path, exist_ok=True) + return test_report_path + + +def sanitize_pytest_xml(xml_file: str): + # pytext xml is different from unittext xml, this function makes pytest xml more similar to unittest xml + # consider somehow modifying the XML logger in conftest to do this instead + import xml.etree.ElementTree as ET + tree = ET.parse(xml_file) + for testcase in tree.iter('testcase'): + full_classname = testcase.attrib.get("classname") + if full_classname is None: + continue + # The test prefix is optional + regex_result = re.search(r"^(test\.)?(?P.*)\.(?P[^\.]*)$", full_classname) + if regex_result is None: + continue + classname = regex_result.group("classname") + file = regex_result.group("file").replace(".", "/") + testcase.set("classname", classname) + testcase.set("file", f"{file}.py") + tree.write(xml_file) + + +def get_pytest_test_cases(argv: list[str]) -> list[str]: + class TestCollectorPlugin: + def __init__(self) -> None: + self.tests: list[Any] = [] + + def pytest_collection_finish(self, session): + for item in session.items: + self.tests.append(session.config.cwd_relative_nodeid(item.nodeid)) + + test_collector_plugin = TestCollectorPlugin() + import pytest + pytest.main( + [arg for arg in argv if arg != '-vv'] + ['--collect-only', '-qq', '--use-main-module'], + plugins=[test_collector_plugin] + ) + return test_collector_plugin.tests + + +def run_tests(argv=UNITTEST_ARGS): + # import test files. + if SLOW_TESTS_FILE: + if os.path.exists(SLOW_TESTS_FILE): + with open(SLOW_TESTS_FILE) as fp: + global slow_tests_dict + slow_tests_dict = json.load(fp) + # use env vars so pytest-xdist subprocesses can still access them + os.environ['SLOW_TESTS_FILE'] = SLOW_TESTS_FILE + else: + warnings.warn(f'slow test file provided but not found: {SLOW_TESTS_FILE}') + if DISABLED_TESTS_FILE: + if os.path.exists(DISABLED_TESTS_FILE): + with open(DISABLED_TESTS_FILE) as fp: + global disabled_tests_dict + disabled_tests_dict = json.load(fp) + os.environ['DISABLED_TESTS_FILE'] = DISABLED_TESTS_FILE + else: + warnings.warn(f'disabled test file provided but not found: {DISABLED_TESTS_FILE}') + # Determine the test launch mechanism + if TEST_DISCOVER: + _print_test_names() + return + + # Before running the tests, lint to check that every test class extends from TestCase + suite = unittest.TestLoader().loadTestsFromModule(__main__) + if not lint_test_case_extension(suite): + sys.exit(1) + + if SHOWLOCALS: + argv = [ + argv[0], + *(["--showlocals", "--tb=long", "--color=yes"] if USE_PYTEST else ["--locals"]), + *argv[1:], + ] + + if TEST_IN_SUBPROCESS: + other_args = [] + if DISABLED_TESTS_FILE: + other_args.append("--import-disabled-tests") + if SLOW_TESTS_FILE: + other_args.append("--import-slow-tests") + if USE_PYTEST: + other_args.append("--use-pytest") + if RERUN_DISABLED_TESTS: + other_args.append("--rerun-disabled-tests") + if TEST_SAVE_XML: + other_args += ['--save-xml', TEST_SAVE_XML] + + test_cases = ( + get_pytest_test_cases(argv) if USE_PYTEST else + [case.id().split('.', 1)[1] for case in discover_test_cases_recursively(suite)] + ) + + failed_tests = [] + + for test_case_full_name in test_cases: + + cmd = ( + [sys.executable] + [argv[0]] + other_args + argv[1:] + + (["--pytest-single-test"] if USE_PYTEST else []) + + [test_case_full_name] + ) + string_cmd = " ".join(cmd) + + timeout = None if RERUN_DISABLED_TESTS else 15 * 60 + + exitcode, _ = retry_shell(cmd, timeout=timeout, retries=0 if RERUN_DISABLED_TESTS else 1) + + if exitcode != 0: + # This is sort of hacky, but add on relevant env variables for distributed tests. + if 'TestDistBackendWithSpawn' in test_case_full_name: + backend = os.environ.get("BACKEND", "") + world_size = os.environ.get("WORLD_SIZE", "") + env_prefix = f"BACKEND={backend} WORLD_SIZE={world_size}" + string_cmd = env_prefix + " " + string_cmd + # Log the command to reproduce the failure. + print(f"Test exited with non-zero exitcode {exitcode}. Command to reproduce: {string_cmd}") + failed_tests.append(test_case_full_name) + + assert len(failed_tests) == 0, "{} unit test(s) failed:\n\t{}".format( + len(failed_tests), '\n\t'.join(failed_tests)) + + elif RUN_PARALLEL > 1: + test_cases = discover_test_cases_recursively(suite) + test_batches = chunk_list(get_test_names(test_cases), RUN_PARALLEL) + processes = [] + for i in range(RUN_PARALLEL): + command = [sys.executable] + argv + [f'--log-suffix=-shard-{i + 1}'] + test_batches[i] + processes.append(subprocess.Popen(command, universal_newlines=True)) + failed = False + for p in processes: + failed |= wait_for_process(p) != 0 + assert not failed, "Some test shards have failed" + elif USE_PYTEST: + pytest_args = argv + ["--use-main-module"] + test_report_path = "" + if TEST_SAVE_XML: + test_report_path = get_report_path(pytest=True) + print(f'Test results will be stored in {test_report_path}') + pytest_args.append(f'--junit-xml-reruns={test_report_path}') + if PYTEST_SINGLE_TEST: + pytest_args = PYTEST_SINGLE_TEST + pytest_args[1:] + + import pytest + os.environ["NO_COLOR"] = "1" + exit_code = pytest.main(args=pytest_args) + if TEST_SAVE_XML: + sanitize_pytest_xml(test_report_path) + + # exitcode of 5 means no tests were found, which happens since some test configs don't + # run tests from certain files + sys.exit(0 if exit_code == 5 else exit_code) + elif TEST_SAVE_XML: + # import here so that non-CI doesn't need xmlrunner installed + import xmlrunner # type: ignore[import] + from xmlrunner.result import _XMLTestResult # type: ignore[import] + + class XMLTestResultVerbose(_XMLTestResult): + """ + Adding verbosity to test outputs: + by default test summary prints 'skip', + but we want to also print the skip reason. + GH issue: https://github.com/pytorch/pytorch/issues/69014 + + This works with unittest_xml_reporting<=3.2.0,>=2.0.0 + (3.2.0 is latest at the moment) + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def addSkip(self, test, reason): + super().addSkip(test, reason) + for c in self.callback.__closure__: + if isinstance(c.cell_contents, str) and c.cell_contents == 'skip': + # this message is printed in test summary; + # it stands for `verbose_str` captured in the closure + c.cell_contents = f"skip: {reason}" + + def printErrors(self) -> None: + super().printErrors() + self.printErrorList("XPASS", self.unexpectedSuccesses) + test_report_path = get_report_path() + verbose = '--verbose' in argv or '-v' in argv + if verbose: + print(f'Test results will be stored in {test_report_path}') + unittest.main(argv=argv, testRunner=xmlrunner.XMLTestRunner( + output=test_report_path, + verbosity=2 if verbose else 1, + resultclass=XMLTestResultVerbose)) + elif REPEAT_COUNT > 1: + for _ in range(REPEAT_COUNT): + if not unittest.main(exit=False, argv=argv).result.wasSuccessful(): + sys.exit(-1) + else: + unittest.main(argv=argv) + +IS_LINUX = sys.platform == "linux" +IS_WINDOWS = sys.platform == "win32" +IS_MACOS = sys.platform == "darwin" +IS_PPC = platform.machine() == "ppc64le" +IS_X86 = platform.machine() in ('x86_64', 'i386') +IS_ARM64 = platform.machine() in ('arm64', 'aarch64') +IS_S390X = platform.machine() == "s390x" + +def is_avx512_vnni_supported(): + if sys.platform != 'linux': + return False + with open("/proc/cpuinfo", encoding="ascii") as f: + lines = f.read() + return "vnni" in lines + +IS_AVX512_VNNI_SUPPORTED = is_avx512_vnni_supported() + +if IS_WINDOWS: + @contextmanager + def TemporaryFileName(*args, **kwargs): + # Ideally we would like to not have to manually delete the file, but NamedTemporaryFile + # opens the file, and it cannot be opened multiple times in Windows. To support Windows, + # close the file after creation and try to remove it manually + if 'delete' in kwargs: + if kwargs['delete'] is not False: + raise UserWarning("only TemporaryFileName with delete=False is supported on Windows.") + else: + kwargs['delete'] = False + f = tempfile.NamedTemporaryFile(*args, **kwargs) + try: + f.close() + yield f.name + finally: + os.unlink(f.name) +else: + @contextmanager # noqa: T484 + def TemporaryFileName(*args, **kwargs): + with tempfile.NamedTemporaryFile(*args, **kwargs) as f: + yield f.name + +if IS_WINDOWS: + @contextmanager + def TemporaryDirectoryName(suffix=None): + # On Windows the directory created by TemporaryDirectory is likely to be removed prematurely, + # so we first create the directory using mkdtemp and then remove it manually + try: + dir_name = tempfile.mkdtemp(suffix=suffix) + yield dir_name + finally: + shutil.rmtree(dir_name) +else: + @contextmanager # noqa: T484 + def TemporaryDirectoryName(suffix=None): + with tempfile.TemporaryDirectory(suffix=suffix) as d: + yield d + + +def is_privateuse1_backend_available(): + privateuse1_backend_name = core._C._get_privateuse1_backend_name() + privateuse1_backend_module = getattr(torch, privateuse1_backend_name, None) + return (is_available := getattr(privateuse1_backend_module, "is_available", None)) and is_available() + + +IS_FILESYSTEM_UTF8_ENCODING = sys.getfilesystemencoding() == 'utf-8' + +TEST_NUMPY = _check_module_exists('numpy') +TEST_FAIRSEQ = _check_module_exists('fairseq') +TEST_SCIPY = _check_module_exists('scipy') +# TEST_MKL = core.backends.mkl.is_available() +# TEST_ACL = core.backends.mkldnn.is_available() and core.ops.mkldnn._is_mkldnn_acl_supported() +TEST_MPS = core.backends.mps.is_available() +MACOS_VERSION = float('.'.join(platform.mac_ver()[0].split('.')[:2]) or -1) +# TEST_XPU = core.xpu.is_available() +# TEST_HPU = True if (hasattr(torch, "hpu") and core.hpu.is_available()) else False +TEST_CUDA = core.cuda.is_available() +TEST_NPU = core.npu.is_available() +# custom_device_mod = getattr(torch, core._C._get_privateuse1_backend_name(), None) +# TEST_PRIVATEUSE1 = is_privateuse1_backend_available() +# TEST_PRIVATEUSE1_DEVICE_TYPE = core._C._get_privateuse1_backend_name() +TEST_NUMBA = _check_module_exists('numba') +TEST_TRANSFORMERS = _check_module_exists('transformers') +TEST_DILL = _check_module_exists('dill') + +TEST_LIBROSA = _check_module_exists('librosa') and not IS_ARM64 + +TEST_OPT_EINSUM = _check_module_exists('opt_einsum') + +TEST_Z3 = _check_module_exists('z3') + +def split_if_not_empty(x: str): + return x.split(",") if len(x) != 0 else [] + +NOTEST_CPU = "cpu" in split_if_not_empty(os.getenv('PYTORCH_TESTING_DEVICE_EXCEPT_FOR', '')) + +skipIfNoDill = unittest.skipIf(not TEST_DILL, "no dill") + + +NO_MULTIPROCESSING_SPAWN: bool = False +TEST_WITH_ASAN: bool = TestEnvironment.def_flag( + "TEST_WITH_ASAN", + env_var="PYTORCH_TEST_WITH_ASAN", +) +TEST_WITH_DEV_DBG_ASAN: bool = TestEnvironment.def_flag( + "TEST_WITH_DEV_DBG_ASAN", + env_var="PYTORCH_TEST_WITH_DEV_DBG_ASAN", +) +TEST_WITH_TSAN: bool = TestEnvironment.def_flag( + "TEST_WITH_TSAN", + env_var="PYTORCH_TEST_WITH_TSAN", +) +TEST_WITH_UBSAN: bool = TestEnvironment.def_flag( + "TEST_WITH_UBSAN", + env_var="PYTORCH_TEST_WITH_UBSAN", +) +TEST_WITH_ROCM: bool = TestEnvironment.def_flag( + "TEST_WITH_ROCM", + env_var="PYTORCH_TEST_WITH_ROCM", +) + +# TODO: Remove PYTORCH_MIOPEN_SUGGEST_NHWC once ROCm officially supports NHWC in MIOpen +# See #64427 +TEST_WITH_MIOPEN_SUGGEST_NHWC = os.getenv('PYTORCH_MIOPEN_SUGGEST_NHWC', '0') == '1' +# Enables tests that are slow to run (disabled by default) +TEST_WITH_SLOW: bool = TestEnvironment.def_flag( + "TEST_WITH_SLOW", + env_var="PYTORCH_TEST_WITH_SLOW", +) + +# Disables non-slow tests (these tests enabled by default) +# This is usually used in conjunction with TEST_WITH_SLOW to +# run *only* slow tests. (I could have done an enum, but +# it felt a little awkward. +TEST_SKIP_FAST: bool = TestEnvironment.def_flag( + "TEST_SKIP_FAST", + env_var="PYTORCH_TEST_SKIP_FAST", +) + +# Enables crossref tests, in addition to standard tests which +# are being run. crossref tests work by installing a torch +# function mode that runs extra compute alongside the regular +# computation that happens with the test. After both computations +# are done, we cross-reference them (thus the name) to check for +# correction, before throwing out the extra compute and proceeding +# as we had before. By default, we don't run these tests. +TEST_WITH_CROSSREF: bool = TestEnvironment.def_flag( + "TEST_WITH_CROSSREF", + env_var="PYTORCH_TEST_WITH_CROSSREF", +) + +TEST_SKIP_CUDAGRAPH: bool = TestEnvironment.def_flag( + "TEST_SKIP_CUDAGRAPH", + env_var="PYTORCH_TEST_SKIP_CUDAGRAPH", +) +TEST_CUDA_GRAPH = TEST_CUDA and (not TEST_SKIP_CUDAGRAPH) and ( + core.version.cuda or + (core.version.hip and float(".".join(core.version.hip.split(".")[0:2])) >= 5.3) +) + +TEST_CUDA_CUDSS = TEST_CUDA and (core.version.cuda and int(core.version.cuda.split(".")[0]) >= 12) + +TEST_CUDA_PYTHON_BINDINGS = _check_module_exists("cuda.bindings") and ( + core.version.cuda and int(core.version.cuda.split(".")[0]) >= 12 +) + +if TEST_CUDA_PYTHON_BINDINGS: + def cuda_python_error_check(function_call_output): + """Makes calls to cuda-python's cuda runtime functions more + pythonic by throwing an exception if they return a status + which is not cudaSuccess + """ + import cuda.bindings # type: ignore[import] + + error, *others = function_call_output + if error != cuda.bindings.runtime.cudaError_t.cudaSuccess: + raise ValueError(f"CUDA failure! {error}") + else: + return tuple(others) +else: + cuda_python_error_check = None # type: ignore[assignment] + +def allocator_option_enabled_fn(allocator_config, _, option): + if allocator_config is None: + return False + allocator_config = allocator_config.split(',') if ',' in allocator_config else [allocator_config] + mapping = dict([var.split(':') for var in allocator_config]) + + if option in mapping and mapping[option] == 'True': + return True + else: + return False + +EXPANDABLE_SEGMENTS: bool = TestEnvironment.def_flag( + "EXPANDABLE_SEGMENTS", + env_var="PYTORCH_CUDA_ALLOC_CONF", + enabled_fn=functools.partial(allocator_option_enabled_fn, option='expandable_segments'), +) + +if TEST_CUDA and 'NUM_PARALLEL_PROCS' in os.environ: + num_procs = int(os.getenv("NUM_PARALLEL_PROCS", "2")) + gb_available = core.cuda.mem_get_info()[1] / 2 ** 30 + # other libraries take up about a little under 1 GB of space per process + core.cuda.set_per_process_memory_fraction(round((gb_available - num_procs * .85) / gb_available / num_procs, 2)) + +requires_cuda = unittest.skipUnless(core.cuda.is_available(), "Requires CUDA") + +def skipIfCrossRef(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if TEST_WITH_CROSSREF: + raise unittest.SkipTest("test doesn't currently with crossref") + else: + fn(*args, **kwargs) + return wrapper + +class CrossRefMode(core.overrides.TorchFunctionMode): + def __torch_function__(self, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + r = func(*args, **kwargs) + return r + +# Run PyTorch tests with TorchDynamo +TEST_WITH_TORCHINDUCTOR: bool = TestEnvironment.def_flag( + "TEST_WITH_TORCHINDUCTOR", + env_var="PYTORCH_TEST_WITH_INDUCTOR", +) +# AOT_EAGER not tested in ci, useful for debugging +TEST_WITH_AOT_EAGER: bool = TestEnvironment.def_flag( + "TEST_WITH_AOT_EAGER", + env_var="PYTORCH_TEST_WITH_AOT_EAGER", +) +TEST_WITH_TORCHDYNAMO: bool = TestEnvironment.def_flag( + "TEST_WITH_TORCHDYNAMO", + env_var="PYTORCH_TEST_WITH_DYNAMO", + implied_by_fn=lambda: TEST_WITH_TORCHINDUCTOR or TEST_WITH_AOT_EAGER, +) +TEST_WITHOUT_COMPILED_AUTOGRAD: bool = TestEnvironment.def_flag( + "TEST_WITHOUT_COMPILED_AUTOGRAD", + env_var="PYTORCH_TEST_WITHOUT_COMPILED_AUTOGRAD", +) + +if TEST_WITH_TORCHDYNAMO: + import core._dynamo + # Do not spend time on helper functions that are called with different inputs + core._dynamo.config.accumulated_recompile_limit = 64 + # Do not log compilation metrics from unit tests + core._dynamo.config.log_compilation_metrics = False + # Silence 3.13.0 guard performance warnings + core._dynamo.config.issue_3_13_0_warning = False + if TEST_WITH_TORCHINDUCTOR: + import core._inductor.config + core._inductor.config.fallback_random = True + else: + # only dynamo for now + core._dynamo.config.compiled_autograd = not TEST_WITHOUT_COMPILED_AUTOGRAD + + +# seems like this is only used in test/torch_np +def xpassIfTorchDynamo_np(func): + # numpy 2.0+ is causing issues + if TEST_WITH_TORCHDYNAMO and np.__version__[0] == '2': + return unittest.skip("skipping numpy 2.0+ dynamo-wrapped test")(func) + return func if TEST_WITH_TORCHDYNAMO else unittest.expectedFailure(func) + + +def xfailIfACL(func): + return unittest.expectedFailure(func) if TEST_ACL else func + + +def xfailIfTorchDynamo(func): + return unittest.expectedFailure(func) if TEST_WITH_TORCHDYNAMO else func + + +def xfailIfPy312Plus(func): + return unittest.expectedFailure(func) if sys.version_info >= (3, 12) else func + + +def xfailIfLinux(func): + return unittest.expectedFailure(func) if IS_LINUX and not TEST_WITH_ROCM and not IS_FBCODE else func + + +def skipIfTorchDynamo(msg="test doesn't currently work with dynamo"): + """ + Usage: + @skipIfTorchDynamo(msg) + def test_blah(self): + ... + """ + assert isinstance(msg, str), "Are you using skipIfTorchDynamo correctly?" + + def decorator(fn): + if not isinstance(fn, type): + @wraps(fn) + def wrapper(*args, **kwargs): + if TEST_WITH_TORCHDYNAMO: + raise unittest.SkipTest(msg) + else: + fn(*args, **kwargs) + return wrapper + + assert isinstance(fn, type) + if TEST_WITH_TORCHDYNAMO: + fn.__unittest_skip__ = True # type: ignore[attr-defined] + fn.__unittest_skip_why__ = msg # type: ignore[attr-defined] + + return fn + + return decorator + +def skipIfTorchInductor(msg="test doesn't currently work with torchinductor", + condition=TEST_WITH_TORCHINDUCTOR): + def decorator(fn): + if not isinstance(fn, type): + @wraps(fn) + def wrapper(*args, **kwargs): + if condition: + raise unittest.SkipTest(msg) + else: + fn(*args, **kwargs) + return wrapper + + assert isinstance(fn, type) + if condition: + fn.__unittest_skip__ = True # type: ignore[attr-defined] + fn.__unittest_skip_why__ = msg # type: ignore[attr-defined] + + return fn + + return decorator + +def runWithoutCompiledAutograd(msg="test doesn't currently work with compiled autograd"): + """ + Usage: + @runWithoutCompiledAutograd(msg) + def test_blah(self): + ... + """ + assert isinstance(msg, str) + + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + with core._dynamo.compiled_autograd._disable(): + func(*args, **kwargs) + return wrapper + + return decorator + +def serialTest(condition=True): + """ + Decorator for running tests serially. Requires pytest + """ + # If one apply decorator directly condition will be callable + # And test will essentially be essentially skipped, which is undesirable + assert type(condition) is bool + + def decorator(fn): + if has_pytest and condition: + return pytest.mark.serial(fn) + return fn + return decorator + +def unMarkDynamoStrictTest(cls=None): + def decorator(cls): + cls.dynamo_strict = False + return cls + + if cls is None: + return decorator + else: + return decorator(cls) + + +def markDynamoStrictTest(cls_or_func=None, nopython=False): + """ + Marks the test as 'strict'. In strict mode, we reset before and after the + test, and run without suppress errors. + + Args: + - nopython: if we should run core._dynamo.optimize with nopython={True/False}. + """ + def decorator(cls_or_func): + if inspect.isclass(cls_or_func): + cls_or_func.dynamo_strict = True + cls_or_func.dynamo_strict_nopython = nopython + return cls_or_func + + fn = cls_or_func + + @wraps(fn) + def wrapper(*args, **kwargs): + core._dynamo.reset() + with unittest.mock.patch("core._dynamo.config.suppress_errors", False): + fn(*args, **kwargs) + core._dynamo.reset() + return wrapper + + if cls_or_func is None: + return decorator + else: + return decorator(cls_or_func) + + +def skipRocmIfTorchInductor(msg="test doesn't currently work with torchinductor on the ROCm stack"): + return skipIfTorchInductor(msg=msg, condition=TEST_WITH_ROCM and TEST_WITH_TORCHINDUCTOR) + +def skipIfLegacyJitExecutor(msg="test doesn't currently work with legacy JIT executor"): + def decorator(fn): + if not isinstance(fn, type): + @wraps(fn) + def wrapper(*args, **kwargs): + if GRAPH_EXECUTOR == ProfilingMode.LEGACY: + raise unittest.SkipTest(msg) + else: + fn(*args, **kwargs) + return wrapper + + assert isinstance(fn, type) + if GRAPH_EXECUTOR == ProfilingMode.LEGACY: + fn.__unittest_skip__ = True # type: ignore[attr-defined] + fn.__unittest_skip_why__ = msg # type: ignore[attr-defined] + + return fn + + + return decorator + + +def make_dynamo_test( + fn: Optional[Callable[..., Any]] = None +) -> Callable[..., Any]: + """ + Decorator function to create a dynamo test case. A function annotate with + this decorator takes as input a unittest object. + """ + from core._dynamo.testing import CompileCounter, reset, optimize_assert + if fn is None: + return lambda fn: make_dynamo_test(fn) + + def standard_test( + self: Any, + fn: Callable[..., Any], + kwargs, + ) -> None: + def dummy() -> None: + fn(self, **kwargs) + + actual = CompileCounter() + + dummy() + reset() + opt_fn = optimize_assert(actual)(dummy) + opt_fn() + reset() + + @functools.wraps(fn) + def test_fn(self: Any, **kwargs) -> None: + return standard_test( + self, + fn=fn, + kwargs=kwargs, + ) + + return test_fn + + +# Run PyTorch tests with translation validation on. +TEST_WITH_TV = os.getenv('PYTORCH_TEST_WITH_TV') == '1' + +if TEST_WITH_TV: + core.fx.experimental._config.translation_validation = True + +# Determine whether to enable cuda memory leak check. +# CUDA mem leak check is expensive and thus we don't want to execute it on every +# test case / configuration. +# If this is True then CUDA memory leak checks are skipped. If this is false +# then CUDA memory leak checks are performed. +# See: https://github.com/pytorch/pytorch/pull/59402#issuecomment-858811135 +TEST_CUDA_MEM_LEAK_CHECK: bool = TestEnvironment.def_flag( + "TEST_CUDA_MEM_LEAK_CHECK", + env_var="PYTORCH_TEST_CUDA_MEM_LEAK_CHECK", +) + + +# Dict of NumPy dtype -> torch dtype (when the correspondence exists) +numpy_to_torch_dtype_dict = { + np.bool_ : core.bool, + np.uint8 : core.uint8, + np.uint16 : core.uint16, + np.uint32 : core.uint32, + np.uint64 : core.uint64, + np.int8 : core.int8, + np.int16 : core.int16, + np.int32 : core.int32, + np.int64 : core.int64, + np.float16 : core.float16, + np.float32 : core.float32, + np.float64 : core.float64, + np.complex64 : core.complex64, + np.complex128 : core.complex128 +} + + +# numpy dtypes like np.float64 are not instances, but rather classes. This leads to rather absurd cases like +# np.float64 != np.dtype("float64") but np.float64 == np.dtype("float64").type. +# Especially when checking against a reference we can't be sure which variant we get, so we simply try both. +def numpy_to_torch_dtype(np_dtype): + try: + return numpy_to_torch_dtype_dict[np_dtype] + except KeyError: + return numpy_to_torch_dtype_dict[np_dtype.type] + + +def has_corresponding_torch_dtype(np_dtype): + try: + numpy_to_torch_dtype(np_dtype) + return True + except KeyError: + return False + + +if IS_WINDOWS: + # Size of `np.intc` is platform defined. + # It is returned by functions like `bitwise_not`. + # On Windows `int` is 32-bit + # https://docs.microsoft.com/en-us/cpp/cpp/data-type-ranges?view=msvc-160 + numpy_to_torch_dtype_dict[np.intc] = core.int + +# Dict of torch dtype -> NumPy dtype +torch_to_numpy_dtype_dict = {value : key for (key, value) in numpy_to_torch_dtype_dict.items()} +torch_to_numpy_dtype_dict.update({ + core.bfloat16: np.float32, + core.complex32: np.complex64 +}) + +# def skipIfNNModuleInlined( +# msg="test doesn't currently work with nn module inlining", +# condition=core._dynamo.config.inline_inbuilt_nn_modules, +# ): +# def decorator(fn): +# if not isinstance(fn, type): + +# @wraps(fn) +# def wrapper(*args, **kwargs): +# if condition: +# raise unittest.SkipTest(msg) +# else: +# fn(*args, **kwargs) + +# return wrapper + +# assert isinstance(fn, type) +# if condition: +# fn.__unittest_skip__ = True # type: ignore[attr-defined] +# fn.__unittest_skip_why__ = msg # type: ignore[attr-defined] + +# return fn + +# return decorator + +def skipIfRocm(func=None, *, msg="test doesn't currently work on the ROCm stack"): + def dec_fn(fn): + reason = f"skipIfRocm: {msg}" + + @wraps(fn) + def wrapper(*args, **kwargs): + if TEST_WITH_ROCM: + raise unittest.SkipTest(reason) + else: + return fn(*args, **kwargs) + return wrapper + if func: + return dec_fn(func) + return dec_fn + +def skipIfRocmArch(arch: tuple[str, ...]): + def dec_fn(fn): + @wraps(fn) + def wrap_fn(self, *args, **kwargs): + if TEST_WITH_ROCM: + prop = core.cuda.get_device_properties(0) + if prop.gcnArchName.split(":")[0] in arch: + reason = f"skipIfRocm: test skipped on {arch}" + raise unittest.SkipTest(reason) + return fn(self, *args, **kwargs) + return wrap_fn + return dec_fn + +def runOnRocm(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if TEST_WITH_ROCM: + fn(*args, **kwargs) + else: + raise unittest.SkipTest("test currently only works on the ROCm stack") + return wrapper + +def runOnRocmArch(arch: tuple[str, ...]): + def dec_fn(fn): + @wraps(fn) + def wrap_fn(self, *args, **kwargs): + if TEST_WITH_ROCM: + prop = core.cuda.get_device_properties(0) + if prop.gcnArchName.split(":")[0] not in arch: + reason = f"skipIfRocm: test only runs on {arch}" + raise unittest.SkipTest(reason) + return fn(self, *args, **kwargs) + return wrap_fn + return dec_fn + +def xfailIfS390X(func): + return unittest.expectedFailure(func) if IS_S390X else func + +def xfailIf(condition): + def wrapper(func): + if condition: + return unittest.expectedFailure(func) + else: + return func + return wrapper + +def skipIfXpu(func=None, *, msg="test doesn't currently work on the XPU stack"): + def dec_fn(fn): + reason = f"skipIfXpu: {msg}" + + @wraps(fn) + def wrapper(*args, **kwargs): + if TEST_XPU: + raise unittest.SkipTest(reason) + else: + return fn(*args, **kwargs) + return wrapper + if func: + return dec_fn(func) + return dec_fn + +def skipIfMPS(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if TEST_MPS: + raise unittest.SkipTest("test doesn't currently work with MPS") + else: + fn(*args, **kwargs) + return wrapper + + +def skipIfMPSOnMacOS13(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if TEST_MPS and int(MACOS_VERSION) == 13: + raise unittest.SkipTest("Test crashes MPSGraph on MacOS13") + else: + fn(*args, **kwargs) + return wrapper + + +def skipIfHpu(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if TEST_HPU: + raise unittest.SkipTest("test doesn't currently work with HPU") + else: + fn(*args, **kwargs) + return wrapper + +# Skips a test on CUDA if ROCm is available and its version is lower than requested. +def skipIfRocmVersionLessThan(version=None): + def dec_fn(fn): + @wraps(fn) + def wrap_fn(self, *args, **kwargs): + if TEST_WITH_ROCM: + rocm_version = str(core.version.hip) + rocm_version = rocm_version.split("-", maxsplit=1)[0] # ignore git sha + rocm_version_tuple = tuple(int(x) for x in rocm_version.split(".")) + if rocm_version_tuple is None or version is None or rocm_version_tuple < tuple(version): + reason = f"ROCm {rocm_version_tuple} is available but {version} required" + raise unittest.SkipTest(reason) + return fn(self, *args, **kwargs) + return wrap_fn + return dec_fn + +def skipIfNotMiopenSuggestNHWC(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if not TEST_WITH_MIOPEN_SUGGEST_NHWC: + raise unittest.SkipTest("test doesn't currently work without MIOpen NHWC activation") + else: + fn(*args, **kwargs) + return wrapper + +def skipIfWindows(func=None, *, msg="test doesn't currently work on the Windows stack"): + def dec_fn(fn): + reason = f"skipIfWindows: {msg}" + + @wraps(fn) + def wrapper(*args, **kwargs): + if IS_WINDOWS: # noqa: F821 + raise unittest.SkipTest(reason) + else: + return fn(*args, **kwargs) + return wrapper + if func: + return dec_fn(func) + return dec_fn + +def requires_cuda_p2p_access(): + cuda_p2p_access_available = ( + core.cuda.is_available() + and core.cuda.get_device_capability() >= (8, 0) + and core.cuda.device_count() >= 2 + ) + num_devices = core.cuda.device_count() + for i in range(num_devices - 1): + for j in range(i + 1, num_devices): + if not core.cuda.can_device_access_peer(i, j): + cuda_p2p_access_available = False + break + if not cuda_p2p_access_available: + break + + return skip_but_pass_in_sandcastle_if( + not cuda_p2p_access_available, + "cuda p2p access is not available", + ) + +# Reverts the linalg backend back to default to make sure potential failures in one +# test do not affect other tests +def setLinalgBackendsToDefaultFinally(fn): + @wraps(fn) + def _fn(*args, **kwargs): + _preferred_backend = core.backends.cuda.preferred_linalg_library() + try: + fn(*args, **kwargs) + finally: + core.backends.cuda.preferred_linalg_library(_preferred_backend) + return _fn + + +# Reverts the blas backend back to default to make sure potential failures in one +# test do not affect other tests +def setBlasBackendsToDefaultFinally(fn): + @wraps(fn) + def _fn(*args, **kwargs): + _preferred_backend = core.backends.cuda.preferred_blas_library() + try: + fn(*args, **kwargs) + finally: + core.backends.cuda.preferred_blas_library(_preferred_backend) + return _fn + + +# Context manager for setting deterministic flag and automatically +# resetting it to its original value +class DeterministicGuard: + def __init__(self, deterministic, *, warn_only=False, fill_uninitialized_memory=True): + self.deterministic = deterministic + self.warn_only = warn_only + self.fill_uninitialized_memory = fill_uninitialized_memory + + @classmethod + def _current_state(cls): + return cls( + core.are_deterministic_algorithms_enabled(), + warn_only=core.is_deterministic_algorithms_warn_only_enabled(), + fill_uninitialized_memory=core.utils.deterministic.fill_uninitialized_memory, # type: ignore[attr-defined] + ) + + def _update(self): + core.use_deterministic_algorithms(self.deterministic, warn_only=self.warn_only) + core.utils.deterministic.fill_uninitialized_memory = self.fill_uninitialized_memory # type: ignore[attr-defined] + + def __enter__(self): + self._restore = self._current_state() + self._update() + + def __exit__(self, exception_type, exception_value, traceback): + self._restore._update() + +class AlwaysWarnTypedStorageRemoval: + def __init__(self, always_warn): + assert isinstance(always_warn, bool) + self.always_warn = always_warn + + def __enter__(self): + self.always_warn_restore = core.storage._get_always_warn_typed_storage_removal() + core.storage._set_always_warn_typed_storage_removal(self.always_warn) + + def __exit__(self, exception_type, exception_value, traceback): + core.storage._set_always_warn_typed_storage_removal(self.always_warn_restore) + +# Context manager for setting cuda sync debug mode and reset it +# to original value +# we are not exposing it to the core because sync debug mode is +# global and thus not thread safe +class CudaSyncGuard: + def __init__(self, sync_debug_mode): + self.mode = sync_debug_mode + + def __enter__(self): + self.debug_mode_restore = core.cuda.get_sync_debug_mode() + core.cuda.set_sync_debug_mode(self.mode) + + def __exit__(self, exception_type, exception_value, traceback): + core.cuda.set_sync_debug_mode(self.debug_mode_restore) + +# Context manager for setting core.__future__.set_swap_module_params_on_conversion +# and automatically resetting it to its original value +class SwapTensorsGuard: + def __init__(self, use_swap_tensors): + self.use_swap_tensors = use_swap_tensors + + def __enter__(self): + self.swap_tensors_restore = core.__future__.get_swap_module_params_on_conversion() + if self.use_swap_tensors is not None: + core.__future__.set_swap_module_params_on_conversion(self.use_swap_tensors) + + def __exit__(self, exception_type, exception_value, traceback): + core.__future__.set_swap_module_params_on_conversion(self.swap_tensors_restore) + +# This decorator can be used for API tests that call +# core.use_deterministic_algorithms(). When the test is finished, it will +# restore the previous deterministic flag setting. +# +# If CUDA >= 10.2, this will set the environment variable +# CUBLAS_WORKSPACE_CONFIG=:4096:8 so that the error associated with that +# setting is not thrown during the test unless the test changes that variable +# on purpose. The previous CUBLAS_WORKSPACE_CONFIG setting will also be +# restored once the test is finished. +# +# Note that if a test requires CUDA to actually register the changed +# CUBLAS_WORKSPACE_CONFIG variable, a new subprocess must be created, because +# CUDA only checks the variable when the runtime initializes. Tests can be +# run inside a subprocess like so: +# +# import subprocess, sys, os +# script = ''' +# # Test code should go here +# ''' +# try: +# subprocess.check_output( +# [sys.executable, '-c', script], +# stderr=subprocess.STDOUT, +# cwd=os.path.dirname(os.path.realpath(__file__)), +# env=os.environ.copy()) +# except subprocess.CalledProcessError as e: +# error_message = e.output.decode('utf-8') +# # Handle exceptions raised by the subprocess here +# +def wrapDeterministicFlagAPITest(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + with DeterministicGuard( + core.are_deterministic_algorithms_enabled(), + warn_only=core.is_deterministic_algorithms_warn_only_enabled()): + class CuBLASConfigGuard: + cublas_var_name = 'CUBLAS_WORKSPACE_CONFIG' + + def __enter__(self): + self.cublas_config_restore = os.environ.get(self.cublas_var_name) + os.environ[self.cublas_var_name] = ':4096:8' + + def __exit__(self, exception_type, exception_value, traceback): + cur_cublas_config = os.environ.get(self.cublas_var_name) + if self.cublas_config_restore is None: + if cur_cublas_config is not None: + del os.environ[self.cublas_var_name] + else: + os.environ[self.cublas_var_name] = self.cublas_config_restore + with CuBLASConfigGuard(): + fn(*args, **kwargs) + return wrapper + +# This decorator can be used for API tests that want to safely call +# core.__future__.set_swap_module_params_on_conversion. `swap` can be set to +# True, False or None where None indicates that the context manager does not +# set the flag. When the test is finished, it will restore the previous swap +# flag setting. +def wrapSwapTensorsTest(swap=None): + def dec_fn(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + with SwapTensorsGuard(swap): + fn(*args, **kwargs) + return wrapper + return dec_fn + +# test parametrizer for swapping +class swap(_TestParametrizer): + def __init__(self, swap_values): + super().__init__() + self.swap_values = swap_values + + def _parametrize_test(self, test, generic_cls, device_cls): + for swap in self.swap_values: + yield wrapSwapTensorsTest(swap)(test), f'swap_{swap}', {}, lambda _: [] + +def skipIfCompiledWithoutNumpy(fn): + # Even if the numpy module is present, if `USE_NUMPY=0` is used during the + # build, numpy tests will fail + numpy_support = TEST_NUMPY + if numpy_support: + try: + # The numpy module is present, verify that PyTorch is compiled with + # numpy support + core.from_numpy(np.array([2, 2])) + except RuntimeError: + numpy_support = False + + @wraps(fn) + def wrapper(*args, **kwargs): + if not numpy_support: + raise unittest.SkipTest("PyTorch was compiled without numpy support") + else: + fn(*args, **kwargs) + return wrapper + +def _test_function(fn, device): + def run_test_function(self): + return fn(self, device) + return run_test_function + +def skipIfNoXNNPACK(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if not core.backends.xnnpack.enabled: # type: ignore[attr-defined] + raise unittest.SkipTest('XNNPACK must be enabled for these tests. Please build with USE_XNNPACK=1.') + else: + fn(*args, **kwargs) + return wrapper + +def skipIfNoLapack(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if not core._C.has_lapack: + raise unittest.SkipTest('PyTorch compiled without Lapack') + else: + fn(*args, **kwargs) + return wrapper + +def skipIfNotRegistered(op_name, message): + """Wraps the decorator to hide the import of the `core`. + + Args: + op_name: Check if this op is registered in `core._REGISTERED_OPERATORS`. + message: message to fail with. + + Usage: + @skipIfNotRegistered('MyOp', 'MyOp is not linked!') + This will check if 'MyOp' is in the caffe2.python.core + """ + return unittest.skip("Pytorch is compiled without Caffe2") + +def skipIfNoSciPy(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if not TEST_SCIPY: + raise unittest.SkipTest("test require SciPy, but SciPy not found") + else: + fn(*args, **kwargs) + return wrapper + +def skip_if_pytest(fn): + @wraps(fn) + def wrapped(*args, **kwargs): + if "PYTEST_CURRENT_TEST" in os.environ: + raise unittest.SkipTest("does not work under pytest") + return fn(*args, **kwargs) + + return wrapped + +def skipIfNoXPU(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if not TEST_XPU: + raise unittest.SkipTest("test required PyTorched compiled with XPU") + else: + fn(*args, **kwargs) + return wrapper + +def slowTest(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if not TEST_WITH_SLOW: + raise unittest.SkipTest("test is slow; run with PYTORCH_TEST_WITH_SLOW to enable test") + else: + fn(*args, **kwargs) + wrapper.__dict__['slow_test'] = True + return wrapper + + +def slowTestIf(condition): + return slowTest if condition else lambda fn: fn + + +def skipCUDAMemoryLeakCheckIf(condition): + def dec(fn): + if getattr(fn, '_do_cuda_memory_leak_check', True): # if current True + fn._do_cuda_memory_leak_check = not condition + return fn + return dec + +def skipCUDANonDefaultStreamIf(condition): + def dec(fn): + if getattr(fn, '_do_cuda_non_default_stream', True): # if current True + fn._do_cuda_non_default_stream = not condition + return fn + return dec + +def suppress_warnings(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fn(*args, **kwargs) + return wrapper + + +def to_gpu(obj, type_map=None): + if type_map is None: + type_map = {} + if isinstance(obj, core.Tensor): + assert obj.is_leaf + t = type_map.get(obj.dtype, obj.dtype) + with core.no_grad(): + res = obj.to(dtype=t, device="cuda", copy=True) + res.requires_grad = obj.requires_grad + return res + elif core.is_storage(obj): + return obj.new().resize_(obj.size()).copy_(obj) # type: ignore[attr-defined, union-attr] + elif isinstance(obj, list): + return [to_gpu(o, type_map) for o in obj] + elif isinstance(obj, tuple): + return tuple(to_gpu(o, type_map) for o in obj) + else: + return deepcopy(obj) + + +def get_function_arglist(func): + return inspect.getfullargspec(func).args + + +def set_rng_seed(seed): + core.manual_seed(seed) + random.seed(seed) + if TEST_NUMPY: + np.random.seed(seed) + + +@contextlib.contextmanager +def set_default_dtype(dtype): + saved_dtype = core.get_default_dtype() + core.set_default_dtype(dtype) + try: + yield + finally: + core.set_default_dtype(saved_dtype) + +@contextlib.contextmanager +def set_default_tensor_type(tensor_type): + saved_tensor_type = core.tensor([]).type() + core.set_default_tensor_type(tensor_type) + try: + yield + finally: + core.set_default_tensor_type(saved_tensor_type) + +def iter_indices(tensor): + if tensor.dim() == 0: + return range(0) + if tensor.dim() == 1: + return range(tensor.size(0)) + return product(*(range(s) for s in tensor.size())) + + +def is_iterable(obj): + try: + iter(obj) + return True + except TypeError: + return False + + +def is_iterable_of_tensors(iterable, include_empty=False): + """ Returns True if iterable is an iterable of tensors and False o.w. + + If the iterable is empty, the return value is :attr:`include_empty` + """ + # Tensor itself is iterable so we check this first + if isinstance(iterable, core.Tensor): + return False + + try: + if len(iterable) == 0: + return include_empty + + for t in iter(iterable): + if not isinstance(t, core.Tensor): + return False + + except TypeError: + return False + + return True + + +class CudaNonDefaultStream: + def __enter__(self): + # Before starting CUDA test save currently active streams on all + # CUDA devices and set new non default streams to all CUDA devices + # to ensure CUDA tests do not use default stream by mistake. + beforeDevice = core.cuda.current_device() + self.beforeStreams = [] + for d in range(core.cuda.device_count()): + self.beforeStreams.append(core.cuda.current_stream(d)) + deviceStream = core.cuda.Stream(device=d) + self.beforeStreams[-1].synchronize() + core._C._cuda_setStream(stream_id=deviceStream.stream_id, + device_index=deviceStream.device_index, + device_type=deviceStream.device_type) + core._C._cuda_setDevice(beforeDevice) + + def __exit__(self, exc_type, exc_value, traceback): + # After completing CUDA test load previously active streams on all + # CUDA devices. + beforeDevice = core.cuda.current_device() + for d in range(core.cuda.device_count()): + core._C._cuda_setStream(stream_id=self.beforeStreams[d].stream_id, + device_index=self.beforeStreams[d].device_index, + device_type=self.beforeStreams[d].device_type) + core._C._cuda_setDevice(beforeDevice) + +class CudaMemoryLeakCheck: + def __init__(self, testcase, name=None): + self.name = testcase.id() if name is None else name + self.testcase = testcase + + # initialize context & RNG to prevent false positive detections + # when the test is the first to initialize those + from core.testing._internal.common_cuda import initialize_cuda_context_rng + initialize_cuda_context_rng() + + # Stores CUDA memory data provided by PyTorch's caching allocator and + # the CUDA driver. + # + # NOTE: The undocumented core.cuda.mem_get_info() returns + # (#free bytes, #total bytes available) on the GPU + def __enter__(self): + self.caching_allocator_befores = [] + self.driver_befores = [] + + # Performs a gc if required (required if any CUDA memory is held) + num_devices = core.cuda.device_count() + for i in range(num_devices): + caching_allocator_mem_allocated = core.cuda.memory_allocated(i) + # NOTE: gc is based exclusively on caching allocator memory + # because the driver will always have some bytes in use (context size?) + if caching_allocator_mem_allocated > 0: + gc.collect() + core._C._cuda_clearCublasWorkspaces() + core.cuda.empty_cache() + break + + # Acquires caching allocator and driver statistics before the test is run + for i in range(num_devices): + self.caching_allocator_befores.append(core.cuda.memory_allocated(i)) + bytes_free, bytes_total = core.cuda.mem_get_info(i) + driver_mem_allocated = bytes_total - bytes_free + self.driver_befores.append(driver_mem_allocated) + + def __exit__(self, exc_type, exc_value, traceback): + # Don't check for leaks if an exception was thrown + if exc_type is not None: + return + + # Compares caching allocator before/after statistics + # An increase in allocated memory is a discrepancy indicating a possible + # memory leak + discrepancy_detected = False + num_devices = core.cuda.device_count() + for i in range(num_devices): + # avoid counting cublasWorkspace allocations + core._C._cuda_clearCublasWorkspaces() + caching_allocator_mem_allocated = core.cuda.memory_allocated(i) + + if caching_allocator_mem_allocated > self.caching_allocator_befores[i]: + discrepancy_detected = True + break + + # Short-circuits if no discrepancy detected + if not discrepancy_detected: + return + + # Validates the discrepancy persists after garbage collection and + # is confirmed by the driver API + + # NOTE: driver API iscrepancies alone are ignored because with the jiterator + # some tests may permanently increase the CUDA context size and + # that will appear as a driver memory leak but is the expected behavior. + + # GCs and clears the cache + gc.collect() + core.cuda.empty_cache() + + for i in range(num_devices): + + discrepancy_detected = True + + # Query memory multiple items to ensure leak was not transient + for _ in range(3): + caching_allocator_mem_allocated = core.cuda.memory_allocated(i) + bytes_free, bytes_total = core.cuda.mem_get_info(i) + driver_mem_allocated = bytes_total - bytes_free + + caching_allocator_discrepancy = False + driver_discrepancy = False + + if caching_allocator_mem_allocated > self.caching_allocator_befores[i]: + caching_allocator_discrepancy = True + + if driver_mem_allocated > self.driver_befores[i]: + driver_discrepancy = True + + if not (caching_allocator_discrepancy or driver_discrepancy): + # Leak was false positive, exit loop + discrepancy_detected = False + break + + if not discrepancy_detected: + continue + + if caching_allocator_discrepancy and not driver_discrepancy: # type: ignore[possibly-undefined] + # Just raises a warning if the leak is not validated by the + # driver API + # NOTE: this may be a problem with how the caching allocator collects its + # statistics or a leak too small to trigger the allocation of an + # additional block of memory by the CUDA driver + msg = ("CUDA caching allocator reports a memory leak not " # type: ignore[possibly-undefined] + f"verified by the driver API in {self.name}! " + f"Caching allocator allocated memory was {self.caching_allocator_befores[i]} " + f"and is now reported as {caching_allocator_mem_allocated} " # type: ignore[possibly-undefined] + f"on device {i}. " + f"CUDA driver allocated memory was {self.driver_befores[i]} and is now {driver_mem_allocated}.") # type: ignore[possibly-undefined] + warnings.warn(msg) + elif caching_allocator_discrepancy and driver_discrepancy: # type: ignore[possibly-undefined] + # A caching allocator discrepancy validated by the driver API is a + # failure (except on ROCm, see below) + msg = (f"CUDA driver API confirmed a leak in {self.name}! " # type: ignore[possibly-undefined] + f"Caching allocator allocated memory was {self.caching_allocator_befores[i]} " + f"and is now reported as {caching_allocator_mem_allocated} " # type: ignore[possibly-undefined] + f"on device {i}. " + f"CUDA driver allocated memory was {self.driver_befores[i]} and is now {driver_mem_allocated}.") # type: ignore[possibly-undefined] + + raise RuntimeError(msg) + +@contextmanager +def skip_exception_type(exc_type): + try: + yield + except exc_type as e: + raise unittest.SkipTest(f"not implemented: {e}") from e + +@contextmanager +def print_repro_on_failure(repro_parts): + try: + yield + except unittest.SkipTest: + raise + except Exception as e: + # Get the index of the sample input that failed the test if possible. + sample_isolation_prefix = "" + tracked_input = getattr(e, "_tracked_input", None) + if tracked_input is not None: + sample_isolation_prefix = f"PYTORCH_OPINFO_SAMPLE_INPUT_INDEX={tracked_input.index}" + + repro_str = " ".join(filter(None, (sample_isolation_prefix, *repro_parts))) + + open_source_signpost( + subsystem="test_repros", + name="test_failure", + parameters=json.dumps( + { + "repro": " ".join(filter(None, (sample_isolation_prefix, *repro_parts))), + } + ), + ) + + repro_msg = f""" +To execute this test, run the following from the base repo dir: + {repro_str} + +This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0""" + + # NB: Hacking the exception args is the cleanest way I've found to append + # failure reproduction info without poisoning the stack trace. + if len(e.args) >= 1: + e.args = (f"{e.args[0]}\n{repro_msg}", *e.args[1:]) + raise + +# "min_satisfying_examples" setting has been deprecated in hypothesis +# 3.56.0 and removed in hypothesis 4.x +try: + import hypothesis + + def settings(*args, **kwargs): + if 'min_satisfying_examples' in kwargs and hypothesis.version.__version_info__ >= (3, 56, 0): + kwargs.pop('min_satisfying_examples') + return hypothesis.settings(*args, **kwargs) + + + hypothesis.settings.register_profile( + "pytorch_ci", + settings( + derandomize=True, + suppress_health_check=[hypothesis.HealthCheck.too_slow], + database=None, + max_examples=50, + verbosity=hypothesis.Verbosity.normal)) + hypothesis.settings.register_profile( + "dev", + settings( + suppress_health_check=[hypothesis.HealthCheck.too_slow], + database=None, + max_examples=10, + verbosity=hypothesis.Verbosity.normal)) + hypothesis.settings.register_profile( + "debug", + settings( + suppress_health_check=[hypothesis.HealthCheck.too_slow], + database=None, + max_examples=1000, + verbosity=hypothesis.Verbosity.verbose)) + + hypothesis.settings.load_profile( + "pytorch_ci" if IS_CI else os.getenv('PYTORCH_HYPOTHESIS_PROFILE', 'dev') + ) +except ImportError: + warnings.warn('Fail to import hypothesis in common_utils, tests are not derandomized', ImportWarning) + +# Used in check_if_enable to see if a test method should be disabled by an issue, +# sanitizes a test method name from appended suffixes by @dtypes parametrization. +# e.g., an issue with title "DISABLED test_bitwise_ops (__main__.TestBinaryUfuncs)" should +# disabled ALL parametrized test_bitwise_ops tests, such test_bitwise_ops_cuda_int32 +def remove_device_and_dtype_suffixes(test_name: str) -> str: + # import statement is localized to avoid circular dependency issues with common_device_type.py + from mindnlp.core.testing._internal.common_device_type import get_device_type_test_bases + device_suffixes = [x.device_type for x in get_device_type_test_bases()] + dtype_suffixes = [str(dt)[len("core."):] for dt in get_all_dtypes()] + + test_name_chunks = test_name.split("_") + if len(test_name_chunks) > 0 and test_name_chunks[-1] in dtype_suffixes: + if len(test_name_chunks) > 1 and test_name_chunks[-2] in device_suffixes: + return "_".join(test_name_chunks[0:-2]) + return "_".join(test_name_chunks[0:-1]) + return test_name + + +def check_if_enable(test: unittest.TestCase): + classname = str(test.__class__).split("'")[1].split(".")[-1] + sanitized_testname = remove_device_and_dtype_suffixes(test._testMethodName) + + def matches_test(target: str): + target_test_parts = target.split() + if len(target_test_parts) < 2: + # poorly formed target test name + return False + target_testname = target_test_parts[0] + target_classname = target_test_parts[1][1:-1].split(".")[-1] + # if test method name or its sanitized version exactly matches the disabled + # test method name AND allow non-parametrized suite names to disable + # parametrized ones (TestSuite disables TestSuiteCPU) + return classname.startswith(target_classname) and (target_testname in (test._testMethodName, sanitized_testname)) + + if any(matches_test(x) for x in slow_tests_dict.keys()): + getattr(test, test._testMethodName).__dict__['slow_test'] = True + if not TEST_WITH_SLOW: + raise unittest.SkipTest("test is slow; run with PYTORCH_TEST_WITH_SLOW to enable test") + + if not IS_SANDCASTLE: + should_skip = False + skip_msg = "" + + for disabled_test, (issue_url, platforms) in disabled_tests_dict.items(): + if matches_test(disabled_test): + platform_to_conditional: dict = { + "mac": IS_MACOS, + "macos": IS_MACOS, + "win": IS_WINDOWS, + "windows": IS_WINDOWS, + "linux": IS_LINUX, + "rocm": TEST_WITH_ROCM, + "xpu": TEST_XPU, + "asan": TEST_WITH_ASAN, + "dynamo": TEST_WITH_TORCHDYNAMO, + "dynamo_wrapped": TEST_WITH_TORCHDYNAMO, + "inductor": TEST_WITH_TORCHINDUCTOR, + "slow": TEST_WITH_SLOW, + } + + invalid_platforms = list(filter(lambda p: p not in platform_to_conditional, platforms)) + if len(invalid_platforms) > 0: + invalid_plats_str = ", ".join(invalid_platforms) + valid_plats = ", ".join(platform_to_conditional.keys()) + + print(f"Test {disabled_test} is disabled for some unrecognized ", + f"platforms: [{invalid_plats_str}]. Please edit issue {issue_url} to fix the platforms ", + 'assigned to this flaky test, changing "Platforms: ..." to a comma separated ', + f"subset of the following (or leave it blank to match all platforms): {valid_plats}") + + # Sanitize the platforms list so that we continue to disable the test for any valid platforms given + platforms = list(filter(lambda p: p in platform_to_conditional, platforms)) + + if platforms == [] or any(platform_to_conditional[platform] for platform in platforms): + should_skip = True + skip_msg = f"Test is disabled because an issue exists disabling it: {issue_url}" \ + f" for {'all' if platforms == [] else ''}platform(s) {', '.join(platforms)}. " \ + "If you're seeing this on your local machine and would like to enable this test, " \ + "please make sure CI is not set and you are not using the flag --import-disabled-tests." + break + + if should_skip and not RERUN_DISABLED_TESTS: + # Skip the disabled test when not running under --rerun-disabled-tests verification mode + raise unittest.SkipTest(skip_msg) + + if not should_skip and RERUN_DISABLED_TESTS: + # Probably test has disable issue but not for this platform + skip_msg = "Test is enabled but --rerun-disabled-tests verification mode is set, so only" \ + " disabled tests are run" + raise unittest.SkipTest(skip_msg) + + if TEST_SKIP_FAST: + if hasattr(test, test._testMethodName) and not getattr(test, test._testMethodName).__dict__.get('slow_test', False): + raise unittest.SkipTest("test is fast; we disabled it with PYTORCH_TEST_SKIP_FAST") + + +# `TestCase.assertEqual` is very permissive and coerced the inputs into a format that could be compared. This is very +# convenient when writing tests, but not so much while reviewing them. By default, the comparison `Pair` framework of +# `core.testing._comparison.are_equal`, used for example by the public testing function +# `core.testing.assert_close`, is more strict. In order to use the same framework and thus reduce the divergence +# between internal and external comparison logic as much as possible, we define some "relaxed" pairs here. They only +# change the supported inputs, but the comparison logic is the same. +# TODO: Revisit the relaxed pairs and check how much work it is to fix the tests that would fail without the relaxation. + +class RelaxedBooleanPair(BooleanPair): + """Pair for boolean-like inputs. + + In contrast to the builtin :class:`BooleanPair`, this class also supports one input being a number or a single + element tensor-like. + """ + _supported_number_types = NumberPair(0, 0)._supported_types + + def _process_inputs(self, actual, expected, *, id): + # We require only one of the inputs of the inputs to be a boolean and the other can also be a boolean, a + # number, or a single element tensor or array, whereas in default BooleanPair both inputs have to be booleans. + tensor_or_array_types: tuple[type, ...] = (core.Tensor, np.ndarray) + other_supported_types = (*self._supported_types, *self._supported_number_types, *tensor_or_array_types) + if not ( + (isinstance(actual, self._supported_types) and isinstance(expected, other_supported_types)) + or (isinstance(expected, self._supported_types) and isinstance(actual, other_supported_types)) + ): + self._inputs_not_supported() + + return [self._to_bool(input, id=id) for input in (actual, expected)] + + def _to_bool(self, bool_like, *, id): + if isinstance(bool_like, np.number): + return bool(bool_like.item()) + elif type(bool_like) in self._supported_number_types: + return bool(bool_like) + elif isinstance(bool_like, (core.Tensor, np.ndarray)): + numel = bool_like.numel() if isinstance(bool_like, core.Tensor) else bool_like.size + if numel > 1: + self._fail( + ValueError, + f"Only single element tensor-likes can be compared against a boolean. " + f"Got {numel} elements instead.", + id=id + ) + + return bool(bool_like.item()) + else: + return super()._to_bool(bool_like, id=id) + + +class RelaxedNumberPair(NumberPair): + """Pair for number-like inputs. + + In contrast to the builtin :class:`NumberPair`, this class also supports one input being a single element + tensor-like or a :class:`enum.Enum`. (D)Type checks are disabled, meaning comparing 1 to 1.0 succeeds even when + ``check_dtype=True`` is passed. + + In addition, this class uses looser default tolerances for :class:`float` and :class:`complex` inputs. Also + supports overriding the absolute and relative tolerance through the ``@precisionOverride`` and + ``@toleranceOverride`` decorators. + """ + _TYPE_TO_DTYPE = { + int: core.int64, + float: core.float32, + complex: core.complex64, + } + + def __init__( + self, actual, expected, *, rtol_override=0.0, atol_override=0.0, check_dtype=None, **other_parameters + ) -> None: + super().__init__(actual, expected, check_dtype=False, **other_parameters) + self.rtol = max(self.rtol, rtol_override) + self.atol = max(self.atol, atol_override) + + def _process_inputs(self, actual, expected, *, id): + # We require only one of the inputs of the inputs to be a number and the other can also be a number or a single + # element tensor or array, whereas in default NumberPair both inputs have to be numbers. + tensor_or_array_types: tuple[type, ...] = (core.Tensor, np.ndarray) + other_supported_types = (*self._supported_types, *tensor_or_array_types) + if not ( + (isinstance(actual, self._supported_types) and isinstance(expected, other_supported_types)) + or (isinstance(expected, self._supported_types) and isinstance(actual, other_supported_types)) + ): + self._inputs_not_supported() + + return [self._to_number(input, id=id) for input in (actual, expected)] + + def _to_number(self, number_like, *, id): + if isinstance(number_like, (core.Tensor, np.ndarray)): + numel = number_like.numel() if isinstance(number_like, core.Tensor) else number_like.size + if numel > 1: + self._fail( + ValueError, + f"Only single element tensor-likes can be compared against a number. " + f"Got {numel} elements instead.", + id=id + ) + number = number_like.item() + if isinstance(number, bool): + number = int(number) + + return number + elif isinstance(number_like, Enum): + return int(number_like) # type: ignore[call-overload] + else: + number = super()._to_number(number_like, id=id) + if type(number) not in self._TYPE_TO_DTYPE.keys(): + self._inputs_not_supported() + return number + + +class TensorOrArrayPair(TensorLikePair): + """Pair for tensor-like inputs. + + On the one hand this class is stricter than the builtin :class:`TensorLikePair` since it only allows instances of + :class:`core.Tensor` and :class:`numpy.ndarray` rather than allowing any tensor-like than can be converted into a + tensor. On the other hand this class is looser since it converts all inputs into tensors with no regard of their + relationship, e.g. comparing a :class:`core.Tensor` to :class:`numpy.ndarray` is fine. + + In addition, this class supports overriding the absolute and relative tolerance through the ``@precisionOverride`` + and ``@toleranceOverride`` decorators. + """ + def __init__(self, actual, expected, *, rtol_override=0.0, atol_override=0.0, **other_parameters): + super().__init__(actual, expected, **other_parameters) + self.rtol = max(self.rtol, rtol_override) + self.atol = max(self.atol, atol_override) + + def _process_inputs(self, actual, expected, *, id, allow_subclasses): + self._check_inputs_isinstance(actual, expected, cls=(core.Tensor, np.ndarray)) + + actual, expected = (self._to_tensor(input) for input in (actual, expected)) + for tensor in (actual, expected): + self._check_supported(tensor, id=id) + return actual, expected + + +class TypedStoragePair(TensorLikePair): + """Pair for :class:`core.storage.TypedStorage` inputs.""" + def __init__(self, actual, expected, *, rtol_override=0.0, atol_override=0.0, **other_parameters): + self._check_inputs_isinstance(actual, expected, cls=core.storage.TypedStorage) + super().__init__(actual, expected, **other_parameters) + self.rtol = max(self.rtol, rtol_override) + self.atol = max(self.atol, atol_override) + + def _to_tensor(self, typed_storage): + return core.tensor( + typed_storage._untyped_storage, + dtype={ + core.quint8: core.uint8, + core.quint4x2: core.uint8, + core.quint2x4: core.uint8, + core.qint32: core.int32, + core.qint8: core.int8 + }.get(typed_storage.dtype, typed_storage.dtype), + device=typed_storage.device, + ) + + +class UnittestPair(Pair): + """Fallback ABC pair that handles non-numeric inputs. + + To avoid recreating the mismatch messages of :meth:`unittest.TestCase.assertEqual`, this pair simply wraps it in + order to use it with the :class:`Pair` "framework" from :func:`are_equal`. + + Define the :attr:`UnittestPair.CLS` in a subclass to indicate which class(es) of the inputs the pair should support. + """ + CLS: Union[type, tuple[type, ...]] + TYPE_NAME: Optional[str] = None + + def __init__(self, actual, expected, **other_parameters): + self._check_inputs_isinstance(actual, expected, cls=self.CLS) + super().__init__(actual, expected, **other_parameters) + + def compare(self): + test_case = unittest.TestCase() + + try: + return test_case.assertEqual(self.actual, self.expected) + except test_case.failureException as error: + msg = str(error) + + type_name = self.TYPE_NAME or (self.CLS if isinstance(self.CLS, type) else self.CLS[0]).__name__ + self._fail(AssertionError, f"{type_name.title()} comparison failed: {msg}") + + +class StringPair(UnittestPair): + CLS = (str, bytes) + TYPE_NAME = "string" + + +class SetPair(UnittestPair): + CLS = set + + +class TypePair(UnittestPair): + CLS = type + + +class ObjectPair(UnittestPair): + CLS = object + + +# This implements a variant of assertRaises/assertRaisesRegex where we first test +# if the exception is NotImplementedError, and if so just skip the test instead +# of failing it. +# +# This is implemented by inheriting from the (private) implementation of +# assertRaises from unittest.case, and slightly tweaking it for this new +# behavior. The year is 2021: this private class hierarchy hasn't changed since +# 2010, seems low risk to inherit from. +class AssertRaisesContextIgnoreNotImplementedError(unittest.case._AssertRaisesContext): + def __exit__(self, exc_type, exc_value, tb): + if exc_type is not None and issubclass(exc_type, NotImplementedError): + self.test_case.skipTest(f"not_implemented: {exc_value}") # type: ignore[attr-defined] + return super().__exit__(exc_type, exc_value, tb) + + +@contextmanager +def set_warn_always_context(new_val: bool): + old_val = core.is_warn_always_enabled() + core.set_warn_always(new_val) + try: + yield + finally: + core.set_warn_always(old_val) + + +class NoTest: + # causes pytest to not recognize this class as a test + __test__ = False + + +class TestCase(expecttest.TestCase): + # NOTE: "precision" lets classes and generated tests set minimum + # atol values when comparing tensors. Used by @precisionOverride and @toleranceOverride, for + # example. + # NOTE: "rel_tol" lets classes and generated tests set minimum + # rtol values when comparing tensors. Used by @toleranceOverride, for example. + _precision: float = 0 + _rel_tol: float = 0 + + # Toggles whether to assert that `core.get_default_dtype()` returns + # `core.float` when `setUp` and `tearDown` are called. + _default_dtype_check_enabled: bool = False + + # Always use difflib to print diffs on multi line equality. + # Undocumented feature in unittest + _diffThreshold = sys.maxsize + maxDiff = None + + # checker to early terminate test suite if unrecoverable failure occurs. + def _should_stop_test_suite(self): + if core.cuda.is_initialized(): + # CUDA device side error will cause subsequence test cases to fail. + # stop entire test suite if catches RuntimeError during core.cuda.synchronize(). + try: + core.cuda.synchronize() + except RuntimeError as rte: + print("TEST SUITE EARLY TERMINATION due to core.cuda.synchronize() failure", file=sys.stderr) + print(str(rte), file=sys.stderr) + return True + return False + else: + return False + + @property + def precision(self) -> float: + return self._precision + + @precision.setter + def precision(self, prec: float) -> None: + self._precision = prec + + @property + def rel_tol(self) -> float: + return self._rel_tol + + @rel_tol.setter + def rel_tol(self, prec: float) -> None: + self._rel_tol = prec + + _do_cuda_memory_leak_check = False + _do_cuda_non_default_stream = False + + # When True, if a test case raises a NotImplementedError, instead of failing + # the test, skip it instead. + _ignore_not_implemented_error = False + + def __init__(self, method_name='runTest', methodName='runTest'): + # methodName is the correct naming in unittest and testslide uses keyword arguments. + # So we need to use both to 1) not break BC and, 2) support testslide. + if methodName != "runTest": + method_name = methodName + super().__init__(method_name) + + test_method = getattr(self, method_name, None) + if test_method is not None: + # Wraps the tested method if we should do CUDA memory check. + if TEST_CUDA_MEM_LEAK_CHECK: + self._do_cuda_memory_leak_check &= getattr(test_method, '_do_cuda_memory_leak_check', True) + # FIXME: figure out the flaky -1024 anti-leaks on windows. See #8044 + if self._do_cuda_memory_leak_check and not IS_WINDOWS: + self.wrap_with_cuda_policy(method_name, self.assertLeaksNoCudaTensors) + + # Wraps the tested method if we should enforce non default CUDA stream. + self._do_cuda_non_default_stream &= getattr(test_method, '_do_cuda_non_default_stream', True) + if self._do_cuda_non_default_stream and not IS_WINDOWS: + self.wrap_with_cuda_policy(method_name, self.enforceNonDefaultStream) + + if self._ignore_not_implemented_error: + self.wrap_with_policy(method_name, lambda: skip_exception_type(NotImplementedError)) + + if PRINT_REPRO_ON_FAILURE: + try: + def _get_rel_test_path(abs_test_path): + # Attempt to get relative path based on the "test" dir. + # In CI, the working dir is not guaranteed to be the base repo dir so + # we can't just compute relative path from that. + parts = Path(abs_test_path).parts + for i, part in enumerate(parts): + if part == "test": + base_dir = os.path.join(*parts[:i]) if i > 0 else '' + return os.path.relpath(abs_test_path, start=base_dir) + + # Can't determine containing dir; just return the test filename. + # The path isn't strictly correct but it's arguably better than nothing. + return os.path.split(abs_test_path)[1] + + abs_test_path = inspect.getfile(type(self)) + test_filename = _get_rel_test_path(abs_test_path) + class_name = type(self).__name__ + test_run_cmd = f"python {test_filename} {class_name}.{method_name}" + env_var_prefix = TestEnvironment.repro_env_var_prefix() + repro_parts = [env_var_prefix, test_run_cmd] + self.wrap_with_policy( + method_name, + lambda repro_parts=repro_parts: print_repro_on_failure(repro_parts)) + except Exception as e: + # Don't fail entirely if we can't get the test filename + log.info("could not print repro string", extra=str(e)) # type: ignore[arg-type] + + def assertLeaksNoCudaTensors(self, name=None): + name = self.id() if name is None else name + return CudaMemoryLeakCheck(self, name) + + def enforceNonDefaultStream(self): + return CudaNonDefaultStream() + + def _remove_ansi_escape(self, input): + # 7-bit C1 ANSI sequences + ansi_escape = re.compile(r''' + \x1B # ESC + (?: # 7-bit C1 Fe (except CSI) + [@-Z\\-_] + | # or [ for CSI, followed by a control sequence + \[ + [0-?]* # Parameter bytes + [ -/]* # Intermediate bytes + [@-~] # Final byte + ) + ''', re.VERBOSE) + return ansi_escape.sub('', input) + + def remove_comment_lines(self, input_string): + lines = input_string.split('\n') + filtered_lines = [line for line in lines if not line.strip().startswith('#')] + return '\n'.join(filtered_lines) + + def remove_empty_lines(self, input_string): + lines = input_string.split('\n') + filtered_lines = [line for line in lines if not line.strip() == ''] + return '\n'.join(filtered_lines) + + # ignore comments will ignore lines that starts with # after being stripped + def assertExpectedInline(self, actual, expect, skip=0, ignore_comments=False, ignore_empty_lines=False): + actual = actual if isinstance(actual, str) else str(actual) + actual = self._remove_ansi_escape(actual) + expect = self._remove_ansi_escape(expect) + if ignore_comments: + actual = self.remove_comment_lines(actual) + expect = self.remove_comment_lines(expect) + + if ignore_empty_lines: + actual = self.remove_empty_lines(actual) + expect = self.remove_empty_lines(expect) + + return super().assertExpectedInline(actual if isinstance(actual, str) else str(actual), expect, skip + 1) + + # Munges exceptions that internally contain stack traces, using munge_exc + def assertExpectedInlineMunged( + self, exc_type, callable, expect, *, skip=0, suppress_suffix=True, post_munge=None, + ): + try: + callable() + except exc_type as e: + munged = munge_exc(e, suppress_suffix=suppress_suffix, skip=skip + 1) + if post_munge: + munged = post_munge(munged) + self.assertExpectedInline( + munged, expect, skip=skip + 1 + ) + return + self.fail(msg="Did not raise when expected to") + + def assertLogs(self, logger=None, level=None): + if logger is None: + logger = logging.getLogger("torch") + return super().assertLogs(logger, level) + + def assertNoLogs(self, logger=None, level=None): + if logger is None: + logger = logging.getLogger("torch") + return super().assertNoLogs(logger, level) + + def wrap_with_cuda_policy(self, method_name, policy): + test_method = getattr(self, method_name) + # the import below may initialize CUDA context, so we do it only if + # self._do_cuda_memory_leak_check or self._do_cuda_non_default_stream + # is True. + # TODO: sure looks like we unconditionally initialize the context here + # -- ezyang + from mindnlp.core.testing._internal.common_cuda import TEST_CUDA + fullname = self.id().lower() # class_name.method_name + if TEST_CUDA and ('gpu' in fullname or 'cuda' in fullname): + setattr(self, method_name, self.wrap_method_with_policy(test_method, policy)) + + def wrap_with_policy(self, method_name, policy): + test_method = getattr(self, method_name) + setattr(self, method_name, self.wrap_method_with_policy(test_method, policy)) + + # A policy is a zero-argument function that returns a context manager. + # We don't take the context manager directly as it may be necessary to + # construct it once per test method + def wrap_method_with_policy(self, method, policy): + # Assumes that `method` is the tested function in `self`. + # NOTE: Python Exceptions (e.g., unittest.Skip) keeps objects in scope + # alive, so this cannot be done in setUp and tearDown because + # tearDown is run unconditionally no matter whether the test + # passes or not. For the same reason, we can't wrap the `method` + # call in try-finally and always do the check. + @wraps(method) + def wrapper(self, *args, **kwargs): + with policy(): + method(*args, **kwargs) + return types.MethodType(wrapper, self) + + def wrap_with_cuda_memory_check(self, method): + return self.wrap_method_with_policy(method, self.assertLeaksNoCudaTensors) + + def _dynamo_test_key(self): + return f"{self.__class__.__name__}.{self._testMethodName}" + + def compile_fn(self, fn, backend, nopython): + # Allows subclasses to control compilation + return core._dynamo.optimize(backend, nopython=nopython)(fn) + + def _run_custom(self, result=None): + using_unittest = isinstance(result, unittest.TestResult) + + super_run = super().run + test_cls = super_run.__self__ # type: ignore[attr-defined] + + super_run(result=result) + + # Early terminate test if necessary. If using pytest, use the -x flag instead + if using_unittest and self._should_stop_test_suite(): + if result.wasSuccessful(): + case = TestCase() + if TEST_SAVE_XML is not None: + # This is a big hacky, XMLRunner modifies expected type from TestCase to TestInfo + # Create dummy TestInfo to record results correctly + from xmlrunner.result import _TestInfo # type: ignore[import] + case = _TestInfo(result, case) + case.output = _TestInfo.ERROR # type: ignore[attr-defined] + case.elapsed_time = 0.0 # type: ignore[attr-defined] + case.test_description = "TestSuiteEarlyFailure" # type: ignore[attr-defined] + # This shouldn't really happen, but if does add fake failure + # For more details see https://github.com/pytorch/pytorch/issues/71973 + result.failures.append((case, "TestSuite execution was aborted early")) + assert result.wasSuccessful() is False + result.stop() + + + def run(self, result=None): + with contextlib.ExitStack() as stack: + if TEST_WITH_CROSSREF: + stack.enter_context(CrossRefMode()) + self._run_custom( + result=result, + ) + + def setUp(self): + check_if_enable(self) + set_rng_seed(SEED) + + # # Save global check sparse tensor invariants state that can be + # # restored from tearDown: + # self._check_invariants = core.sparse.check_sparse_tensor_invariants.is_enabled() + + # # Enable invariant checks for all sparse tensors constructions + # # including the unsafe ones. If this is not desired for some + # # test case, use check_invariants=False optional argument to + # # sparse tensor constructors or + # # @core.sparse.check_sparse_tensor_invariants(False) + # # decorator to disable the invariant checks. + # core.sparse.check_sparse_tensor_invariants.enable() + + if self._default_dtype_check_enabled: + assert core.get_default_dtype() == core.float + + # attempt to reset some global state at the end of the test + self._prev_grad_state = core.is_grad_enabled() + + def tearDown(self): + # There exists test cases that override TestCase.setUp + # definition, so we cannot assume that _check_invariants + # attribute is defined in general. + if hasattr(self, '_check_invariants'): + # Restore the global check sparse tensor invariants state + if self._check_invariants: + core.sparse.check_sparse_tensor_invariants.enable() + else: + core.sparse.check_sparse_tensor_invariants.disable() + + if self._default_dtype_check_enabled: + assert core.get_default_dtype() == core.float + + # attribute may not be defined, per above + if hasattr(self, '_prev_grad_state'): + core.set_grad_enabled(self._prev_grad_state) + + @staticmethod + def _make_crow_indices(n_rows, n_cols, nnz, + *, device, dtype, random=True): + """Return crow_indices of a CSR tensor with size (n_rows, n_cols) and + the number of specified elements nnz. + + If random is True, the column counts of rows are in random + order. Otherwise, the column counts of rows are defined by the + used sampling method. + + Sampling method + --------------- + + The used sampling method was introduced in + https://pearu.github.io/csr_sampling.html, and here we give + only an overall description of the method. + + Notice that crow_indices can be defined as cumsum(counts) + where counts is a sequence of non-negative integers satisfying + the following conditions: + + len(counts) == n_rows + 1 + counts.max() <= n_cols + + while counts[i + 1] is interpreted as the number of specified + elements in the i-th row. + + The used sampling method aims at increasing the diversity of + CSR samples, that is, a CSR sample should contain (i) rows + that are all filled, (ii) rows with no elements at all, and + (iii) rows that are partially filled. At the same time and for + the given total number of specified elements (nnz), there + should be minimal preference to rows with a given number of + elements. To achieve this, the sampling method is built-up on + using a sawteeth model for counts. In the simplest case, we + would have + + counts = arange(n_rows + 1) % (n_cols + 1) + + that has equal number of all possible column counts per row. + This formula can be used only for specific input values of + n_rows, n_cols, and nnz. To generalize this model to any + combinations of inputs, the counts model above is extended + with an incomplete sawtooth, and the right and lower + rectangular parts that will guarantee that + + counts.sum() == nnz + + for any combination of n_rows, n_cols, and nnz. Basically, + we'll find a maximal window in (n_rows + 1, n_cols + 1)-grid + that is able to hold a sequence of sawteeth and so-called + final correction, while the external part of the window is + filled with counts to meet the nnz constraint exactly. + """ + assert 0 <= nnz <= n_rows * n_cols, (nnz, n_rows, n_cols) + + def sawteeth(n, m): + # return the total number of counts in the sequence of + # sawteeth where n and m define a window in (n_rows+1, + # n_cols+1) rectangle where the sequence of sawteeth + # perfectly fit. + M = (n_cols - m) * (n_cols - m + 1) // 2 + K = (n_rows - n) % (n_cols - m + 1) + return M * ((n_rows - n) // (n_cols - m + 1)) + K * (K - 1) // 2 + + # Different from the original method description, here counts + # has leading 0 required by crow_indices: + counts = core.zeros(n_rows + 1, dtype=dtype, device=core.device('cpu')) + + n = m = 0 + N = sawteeth(n, m) + if N and nnz >= max(N, n_cols): + # determine the width of the sawteeth window. We use bisection to solve + # N(n, 0) == 0 or nnz - n * n_cols < max(N(n, 0), n_cols) + # for n + n_left = n + n_right = n_rows - 1 + N_right = sawteeth(n_right, m) + while n_right - n_left > 1: + n_middle = (n_left + n_right) // 2 + N_middle = sawteeth(n_middle, m) + if N_middle == 0 or nnz - n_middle * n_cols < max(N_middle, n_cols): + n_right, N_right = n_middle, N_middle + else: + n_left = n_middle + n, N = n_right, N_right + # fill the right rectangle with counts: + assert n + counts[-n:].fill_(n_cols) + + if N and nnz - n * n_cols >= max(N, n_rows - n): + # determine the height of the sawteeth window. We use bisection to solve + # N(n, m) == 0 or nnz - n * n_cols - m * (n_rows - n) < max(N(n, m), n_rows - n) + # for m. + m_left = m + m_right = n_cols - 1 + N_right = sawteeth(n, m_right) + while m_right - m_left > 1: + m_middle = (m_left + m_right) // 2 + N_middle = sawteeth(n, m_middle) + if N_middle == 0 or nnz - n * n_cols - m_middle * (n_rows - n) < max(N_middle, n_rows - n): + m_right, N_right = m_middle, N_middle + else: + m_left = m_middle + m, N = m_right, N_right + # fill the bottom rectangle with counts: + assert m + counts[1:n_rows - n + 1].fill_(m) + + if N: + # fill the sawteeth window with counts + q, r = divmod(nnz - n * n_cols - m * (n_rows - n), + (n_cols - m) * (n_cols - m + 1) // 2) + p = 1 + q * (n_cols - m + 1) + k = math.isqrt(2 * r) + if k * (k + 1) > 2 * r: + k -= 1 + corr = r - k * (k + 1) // 2 + assert not ((p > 1) and (m > 0)) # full sawteeth are never on top of a bottom rectangle + # sequence of full sawteeth: + counts[1:p] = core.arange(p - 1, dtype=dtype, device=counts.device) % (n_cols - m + 1) + # incomplete sawtooth: + counts[p:p + k + 1] += core.arange(k + 1, dtype=dtype, device=counts.device) + else: + # given input does not support sawteeth + p = 1 + corr = nnz - n * n_cols - m * (n_rows - n) + + # correction that will guarantee counts.sum() == nnz: + counts[p] += corr + + if random: + # randomize crow_indices by shuffling the sawteeth + # sequence: + perm = core.randperm(n_rows, device=counts.device) + counts[1:] = counts[1:][perm] + + # compute crow_indices: + crow_indices = counts + crow_indices.cumsum_(dim=0) + return crow_indices.to(device=device) + + def genSparseCompressedTensor(self, size, nnz, *, layout, device, dtype, index_dtype, blocksize=(), dense_dims=0): + from operator import mul + from functools import reduce + sparse_dim = 2 + assert all(size[d] > 0 for d in range(len(size))) or nnz == 0, 'invalid arguments' + assert len(size) >= sparse_dim + if blocksize: + assert len(blocksize) == 2, (size, blocksize) + assert size[-2 - dense_dims] % blocksize[0] == 0, (size, blocksize) + assert size[-1 - dense_dims] % blocksize[1] == 0, (size, blocksize) + blocksize0, blocksize1 = blocksize + else: + blocksize0 = blocksize1 = 1 + + size = tuple(size) + dense_size = size[(len(size) - dense_dims):] + + def random_sparse_compressed(n_compressed_dims, n_plain_dims, nnz): + compressed_indices = self._make_crow_indices(n_compressed_dims, n_plain_dims, nnz, device=device, dtype=index_dtype) + plain_indices = core.zeros(nnz, dtype=index_dtype, device=device) + for i in range(n_compressed_dims): + count = compressed_indices[i + 1] - compressed_indices[i] + plain_indices[compressed_indices[i]:compressed_indices[i + 1]], _ = core.sort( + core.randperm(n_plain_dims, dtype=index_dtype, device=device)[:count]) + low = -1 if dtype != core.uint8 else 0 + high = 1 if dtype != core.uint8 else 2 + values = make_tensor((nnz,) + blocksize + dense_size, device=device, dtype=dtype, low=low, high=high) + return values, compressed_indices, plain_indices + + batch_shape = size[:-2 - dense_dims] + n_batch = reduce(mul, batch_shape, 1) + + if layout in {core.sparse_csr, core.sparse_bsr}: + n_compressed_dims, n_plain_dims = size[-2 - dense_dims] // blocksize0, size[-1 - dense_dims] // blocksize1 + else: + n_compressed_dims, n_plain_dims = size[-1 - dense_dims] // blocksize1, size[-2 - dense_dims] // blocksize0 + blocknnz = nnz // (blocksize0 * blocksize1) + sparse_tensors = [random_sparse_compressed(n_compressed_dims, n_plain_dims, blocknnz) for _ in range(n_batch)] + sparse_tensors_it = map(list, zip(*sparse_tensors)) + + values = core.stack(next(sparse_tensors_it)).reshape(*batch_shape, blocknnz, *blocksize, *dense_size) + compressed_indices = core.stack(next(sparse_tensors_it)).reshape(*batch_shape, -1) + plain_indices = core.stack(next(sparse_tensors_it)).reshape(*batch_shape, -1) + return core.sparse_compressed_tensor(compressed_indices, plain_indices, + values, size=size, dtype=dtype, layout=layout, device=device) + + def genSparseCSRTensor(self, size, nnz, *, device, dtype, index_dtype, dense_dims=0): + return self.genSparseCompressedTensor(size, nnz, layout=core.sparse_csr, device=device, + dtype=dtype, index_dtype=index_dtype, blocksize=(), dense_dims=dense_dims) + + def genSparseCSCTensor(self, size, nnz, *, device, dtype, index_dtype, dense_dims=0): + return self.genSparseCompressedTensor(size, nnz, layout=core.sparse_csc, device=device, + dtype=dtype, index_dtype=index_dtype, blocksize=(), dense_dims=0) + + def genSparseBSRTensor(self, size, blocksize, nnz, *, device, dtype, index_dtype, dense_dims=0): + assert len(blocksize) == 2 + return self.genSparseCompressedTensor(size, nnz, layout=core.sparse_bsr, device=device, + dtype=dtype, index_dtype=index_dtype, blocksize=blocksize, dense_dims=dense_dims) + + def genSparseBSCTensor(self, size, blocksize, nnz, *, device, dtype, index_dtype, dense_dims=0): + assert len(blocksize) == 2 + return self.genSparseCompressedTensor(size, nnz, layout=core.sparse_bsc, device=device, + dtype=dtype, index_dtype=index_dtype, blocksize=blocksize, dense_dims=dense_dims) + + def genSparseTensor(self, size, sparse_dim, nnz, is_uncoalesced, device, dtype): + # Assert not given impossible combination, where the sparse dims have + # empty numel, but nnz > 0 makes the indices containing values. + assert all(size[d] > 0 for d in range(sparse_dim)) or nnz == 0, 'invalid arguments' + + v_size = [nnz] + list(size[sparse_dim:]) + v = make_tensor(v_size, device=device, dtype=dtype, low=-1, high=1) + i = core.rand(sparse_dim, nnz, device=device) + i.mul_(core.tensor(size[:sparse_dim]).unsqueeze(1).to(i)) + i = i.to(core.long) + if is_uncoalesced: + i1 = i[:, :(nnz // 2), ...] + i2 = i[:, :((nnz + 1) // 2), ...] + i = core.cat([i1, i2], 1) + x = core.sparse_coo_tensor(i, v, core.Size(size), dtype=dtype, device=device) + + if not is_uncoalesced: + x = x.coalesce() + else: + # FIXME: `x` is a sparse view of `v`. Currently rebase_history for + # sparse views is not implemented, so this workaround is + # needed for inplace operations done on `x`, e.g., copy_(). + # Remove after implementing something equivalent to CopySlice + # for sparse views. + # NOTE: We do clone() after detach() here because we need to be able to change size/storage of x afterwards + x = x.detach().clone()._coalesced_(False) + return x, x._indices().clone(), x._values().clone() + + def generate_simple_inputs(self, layout, + device=None, + dtype=None, + index_dtype=None, + pin_memory=None, + members_pin_memory=None, + enable_batch=True, + enable_hybrid=True, + enable_zero_sized=True, + enable_non_contiguous_indices=True, + enable_non_contiguous_values=True, + enable_batch_variable_nse=False, + output_tensor=True, + patterns=None): + """Generator of simple inputs for tensor constructors of the given layout. + + The generated tensor inputs have the following properties: + + - tensor shapes are minimal but not trivial + - tensor values are sorted sequences for COO and CSR formats, e.g. [1, 2, 3, 4] + - the generated tensors represent the same mathematical tensor for all layouts + - the generated tensors include regular, zero-sized, and optionally, batched or/and hybrid tensors. + - the generated tensors include contiguous or non-contiguous tensors both in indices and values + + If output_tensor is True, yield tensors with the given + layout. Otherwise, yield inputs to the corresponding tensor + constructors: + + - sparse compressed input is defined as + (compressed_indices, plain_indices, values), dict(size=expected_size_from_shape_inference, device=device, dtype=dtype, + pin_memory=pin_memory) + + - sparse COO input is defined as + (indices, values), dict(size=expected_size_from_shape_inference, device=device, dtype=dtype, pin_memory=pin_memory) + + - strided input is defined as + (values,), dict(device=device, dtype=dtype) + """ + if index_dtype is None: + index_dtype = core.int64 + + is_compressed_sparse_layout = layout in {core.sparse_csr, core.sparse_csc, core.sparse_bsr, core.sparse_bsc} + + if output_tensor: + for args, kwargs in self.generate_simple_inputs(layout, device=device, dtype=dtype, index_dtype=index_dtype, + pin_memory=pin_memory, + enable_batch=enable_batch, enable_hybrid=enable_hybrid, + enable_zero_sized=enable_zero_sized, + enable_non_contiguous_indices=enable_non_contiguous_indices, + enable_non_contiguous_values=enable_non_contiguous_values, + enable_batch_variable_nse=enable_batch_variable_nse, + output_tensor=False): + if members_pin_memory: + args = tuple(a.pin_memory() for a in args) + if layout is core.strided: + assert len(args) == 1 + size = kwargs.pop('size', None) # to ensure that a zero-sized tensor has the desired shape + assert size is not None + if pin_memory: + yield args[0].reshape(size).pin_memory() + else: + yield args[0].reshape(size) + elif layout is core.sparse_coo: + yield core.sparse_coo_tensor(*args, **kwargs) + elif is_compressed_sparse_layout: + kwargs.update(layout=layout) + yield core.sparse_compressed_tensor(*args, **kwargs) + else: + assert 0 # unreachable + return + + def get_blockpattern(pattern, blocksize): + basesize = pattern.shape + assert basesize[0] % blocksize[0] == 0, (basesize, blocksize) + assert basesize[1] % blocksize[1] == 0, (basesize, blocksize) + blockpattern = pattern.reshape(-1, + blocksize[0], + basesize[1] // blocksize[1], + blocksize[1]).transpose(-3, -2).any(-1).any(-1) + block_ids = core.arange(1, blockpattern.numel() + 1).reshape(blockpattern.shape) + return (blockpattern != 0) * block_ids + + def get_sparse_data(pattern): + basesize = pattern.shape + assert len(basesize) == 2, basesize # pattern is expected to be a matrix + + # We cannot use `core.sparse_xyz_tensor(pattern)` to + # compute the sparse layout indices and values because + # generate_simple_inputs is used to generate the inputs to + # test `core.sparse_xyz_tensor` factory functions, so + # we'll compute the indices and values independently of + # the factory functions. + + indices = core.where(pattern != 0) + coo_indices = core.stack(indices) + crow_indices = core.zeros(basesize[0] + 1, dtype=core.int64) + crow_indices[1:] = core.cumsum(coo_indices[0].bincount(minlength=basesize[0]), 0) + col_indices = coo_indices[1] + strided_values = core.zeros(basesize, dtype=core.int64) + + # the property of `values == range(1, 1+nnz)` is used in + # get_sparse_data_with_block to relate BSR and BSC values, + # so, don't change the following line: + values = core.arange(1, 1 + len(indices[0]), dtype=core.int64) + strided_values[indices] = values + + indices_T = core.where(pattern.transpose(0, 1) != 0) + coo_indices_T = core.stack(indices_T) + ccol_indices = core.zeros(basesize[1] + 1, dtype=core.int64) + ccol_indices[1:] = core.cumsum(coo_indices_T[0].bincount(minlength=basesize[1]), 0) + row_indices = coo_indices_T[1] + csc_values = strided_values.transpose(0, 1)[indices_T] + + return {core.sparse_coo: (coo_indices, values), + core.sparse_csr: (crow_indices, col_indices, values), + core.sparse_csc: (ccol_indices, row_indices, csc_values), + core.strided: (strided_values,)} + + def get_sparse_data_with_block(pattern, blocksize): + nonblock_data = get_sparse_data(pattern) + blockpattern = get_blockpattern(pattern, blocksize) + block_data = get_sparse_data(blockpattern) + + strided_values = nonblock_data[core.strided][0] + block_indices = block_data[core.sparse_coo][0] + bsr_values = core.stack([strided_values[bi * blocksize[0]:(bi + 1) * blocksize[0], + bj * blocksize[1]:(bj + 1) * blocksize[1]] + for bi, bj in block_indices.transpose(0, 1)]) + + # here we use the property `values == range(1, 1+nnz)` and + # `values` relation to `csc_values` (see get_sparse_data) + # to get BSC blocks via reordering the BSR blocks: + bsc_values = bsr_values[block_data[core.sparse_csc][2] - 1] + + return {core.sparse_bsr: (*block_data[core.sparse_csr][:2], bsr_values), + core.sparse_bsc: (*block_data[core.sparse_csc][:2], bsc_values), + **nonblock_data} + + def get_batch_sparse_data(pattern, blocksize): + size = pattern.shape + if len(size) <= 2: # non-batch + return get_sparse_data_with_block(pattern, blocksize) + + # batch data is created recursively: + batch_data = {} # type: ignore[var-annotated] + for i, item in enumerate(pattern): + for layout, d in get_batch_sparse_data(item, blocksize).items(): + target = batch_data.get(layout) + if layout is core.sparse_coo: + # a "batch COO" means a COO with the leading + # sparse dimensions interpreted as batch + # dimensions + ext_coo_indices1 = core.cat((core.full((1, len(d[1])), i, dtype=core.int64), d[0])) + if target is None: + target = batch_data[layout] = (ext_coo_indices1, d[1]) + else: + target[0].set_(core.cat((target[0], ext_coo_indices1), 1)) # type: ignore[call-overload] + target[1].set_(core.cat((target[1], d[1]))) + else: + if target is None: + target = batch_data[layout] = tuple(d[j].unsqueeze(0) for j in range(len(d))) + else: + for j in range(len(d)): + target[j].set_(core.cat((target[j], d[j].unsqueeze(0)))) # type: ignore[call-overload] + return batch_data + + def generate_values(base, densesize): + """Generates a tensor of shape densesize with values equal to + + base + i_1 * 10^0 + ... + i_d * 10^{d - 1} + + at indices i_1, ..., i_d (with 0 <= i_j < densesize[j] for any 1 <= j <= + len(densesize)) + + This mapping produces unique values as long as + densesize[i] < 10 for all i in range(len(densesize)). + """ + + if not densesize: + return base + if not isinstance(base, int) and base.ndim > 0: + return core.stack([generate_values(b, densesize) for b in base]) + if base == 0: + return core.zeros(densesize, dtype=core.int64) + r = core.arange(densesize[0], dtype=core.int64) + for i, d in enumerate(densesize[1:]): + y = core.arange(d, dtype=core.int64) * (10 ** (i + 1)) + r = r[..., None] + y[None, ...] + r.add_(base) + return r + + if patterns is None: + # A pattern is a 3-tuple with the following items: + # + # - a list of integers with the depth of two or more. The + # integers define the sparsity patterns of the generated + # inputs: zero values correspond to unspecified + # elements/blocks, and non-zero values to the specified + # elements. + # + # For debugging convenience, the elements with the same + # value typically belong to the same block. However, it + # is not a hard requirement: as long as the shape of a + # pattern divides with block sizes, the pattern will be + # a valid one. + # + # If the depth of the list is larger than two, inputs + # with batch dimensions will be generated. + # + # - a list of 2-tuples of block sizes, used to generate + # BSR/BSC tensors with various block size parameters + # + # - a list of tuples of dense dimensions, used to generate + # hybrid tensors with various dense dimensions + # + patterns = [ + # a simple 3 x 2 tensor: non-hybrid, hybrid with 1 and 2 dense dimensions + ([[1, 2, 0], + [1, 0, 3]], [(2, 1), (1, 3)], [(), (2,), (4, 5)]), + # 2 x 3 batch of 3 x 2 tensors: non-hybrid and hybrid with 2 dense dimensions + ([[[[1, 2, 0], + [1, 0, 3]], + [[1, 2, 3], + [1, 0, 0]], + [[1, 0, 0], + [1, 2, 3]]], + [[[0, 2, 0], + [1, 2, 3]], + [[1, 0, 3], + [1, 2, 0]], + [[1, 2, 3], + [0, 2, 0]]]], [(2, 1), (2, 3)], [(), (2,)]), + # tensor with non-trivial blocksize + ([[0, 1, 0, 2, 0, 2], + [0, 1, 0, 0, 2, 0], + [3, 3, 3, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 5, 0, 6, 6, 6], + [5, 0, 5, 6, 6, 6], + [0, 0, 0, 0, 8, 8], + [7, 7, 7, 0, 8, 8]], [(2, 3)], [(), (4, 5)]), + # batch tensor with variable NSE + # Requires https://github.com/pytorch/pytorch/pull/84843 or similar. + ([[[1, 2], + [3, 4]], + [[1, 0], + [0, 0]]], [(1, 1)], ([()] if enable_batch_variable_nse else []))] + + def non_contiguous_copy(t, dim=-1, offset=0): + # return a copy of t that is non-contiguous along the + # given dimension and with the given storage offset + self.assertTrue(t.is_contiguous()) + if dim < 0: + dim = dim + t.ndim + assert dim >= 0 and dim < t.ndim + step = max(2, offset + 1) + tmp = core.zeros((*t.shape[:dim], t.shape[dim] * step, *t.shape[dim + 1:]), dtype=t.dtype, device=t.device) + dim_slices = (*((slice(None),) * dim), slice(offset, None, step)) + r = tmp[dim_slices].copy_(t) + self.assertFalse(r.is_contiguous()) + self.assertEqual(t, r) + return r + + # the main loop of the method: + for pattern, blocksizes, densesizes in patterns: + if not enable_hybrid: + densesizes = [s for s in densesizes if not s] + if not (densesizes and blocksizes): + continue + pattern = core.tensor(pattern, dtype=core.int64) + if not enable_batch and pattern.ndim > 2: + continue + for blocksize in blocksizes: + data = get_batch_sparse_data(pattern, blocksize)[layout] + for densesize in densesizes: + indices = [a.to(device=device, dtype=index_dtype) for a in data[:-1]] + values = generate_values(data[-1], densesize).to(device=device, dtype=dtype) + kwargs = dict(device=device, dtype=dtype, size=pattern.shape + densesize) + if pin_memory is not None: + kwargs.update(pin_memory=pin_memory) + + yield (*indices, values), kwargs.copy() + if enable_non_contiguous_indices and pattern.ndim > 2: + # sparse compressed indices can be sliced only along batch dimensions + for (dim, offset) in {(0, 1), (-2, 0)}: + indices_copy = [non_contiguous_copy(a, dim=dim, offset=offset) for a in indices] + yield (*indices_copy, values), kwargs.copy() + + if enable_non_contiguous_values: + values_copy = non_contiguous_copy(values, dim=-1, offset=1) + yield (*indices_copy, values_copy), kwargs.copy() + + if enable_non_contiguous_values: + values_copy = non_contiguous_copy(values, dim=-1, offset=1) + yield (*indices, values_copy), kwargs.copy() + + # zero-sized tensor inputs, non-batch, non-hybrid/hybrid + if enable_zero_sized: + for basesize, blocksizes, densesizes in [ + ((2, 0), [(1, 2)], [(), (2,), (2, 3)] if enable_hybrid else [()]), + ((0, 2), [(1, 2), (2, 1), (3, 2)], [()]), + ((0, 0), [(1, 2)], [()]), + ]: + for blocksize in blocksizes: + for densesize in densesizes: # type: ignore[attr-defined] + if layout == core.strided: + indices = () # type: ignore[assignment] + values = core.empty((basesize + densesize), device=device, dtype=dtype) + elif layout == core.sparse_coo: + indices = (core.empty(len(basesize), 0, device=device, dtype=index_dtype),) # type: ignore[assignment] + values = core.empty((0, *densesize), device=device, dtype=dtype) + elif layout == core.sparse_csr: + crow_indices = core.tensor([0] * (basesize[0] + 1), device=device, dtype=index_dtype) + col_indices = core.empty(0, device=device, dtype=index_dtype) + indices = (crow_indices, col_indices) # type: ignore[assignment] + values = core.empty((0, *densesize), device=device, dtype=dtype) + elif layout == core.sparse_csc: + ccol_indices = core.tensor([0] * (basesize[1] + 1), device=device, dtype=index_dtype) + row_indices = core.empty(0, device=device, dtype=index_dtype) + indices = (ccol_indices, row_indices) # type: ignore[assignment] + values = core.empty((0, *densesize), device=device, dtype=dtype) + elif layout == core.sparse_bsr: + crow_indices = core.tensor([0] * (basesize[0] // blocksize[0] + 1), device=device, dtype=index_dtype) + col_indices = core.empty(0, device=device, dtype=index_dtype) + indices = (crow_indices, col_indices) # type: ignore[assignment] + values = core.empty((0, *blocksize, *densesize), device=device, dtype=dtype) + elif layout == core.sparse_bsc: + ccol_indices = core.tensor([0] * (basesize[1] // blocksize[1] + 1), device=device, dtype=index_dtype) + row_indices = core.empty(0, device=device, dtype=index_dtype) + indices = (ccol_indices, row_indices) # type: ignore[assignment] + values = core.empty((0, *blocksize, *densesize), device=device, dtype=dtype) + else: + assert 0 # unreachable + kwargs = dict(device=device, dtype=dtype, size=basesize + densesize) + if pin_memory is not None: + kwargs.update(pin_memory=pin_memory) + yield (*indices, values), kwargs + + def safeToDense(self, t): + # coalesce is only implemented for COO + if t.layout == core.sparse_coo: + t = t.coalesce() + return t.to_dense() + + # Compares a torch function with a reference function for a given sample input (object of SampleInput) + # Note: only values are compared, type comparison is not done here + def compare_with_reference(self, torch_fn, ref_fn, sample_input, **kwargs): + numpy_sample = sample_input.numpy() + n_inp, n_args, n_kwargs = numpy_sample.input, numpy_sample.args, numpy_sample.kwargs + t_inp, t_args, t_kwargs = sample_input.input, sample_input.args, sample_input.kwargs + + actual = torch_fn(t_inp, *t_args, **t_kwargs) + expected = ref_fn(n_inp, *n_args, **n_kwargs) + + self.assertEqual(actual, expected, exact_device=False, **kwargs) + + # Compares the given Torch and NumPy functions on the given tensor-like object. + # NOTE: both torch_fn and np_fn should be functions that take a single + # tensor (array). If the torch and/or NumPy function require additional + # arguments then wrap the function in a lambda or pass a partial function. + # TODO: add args/kwargs for passing to assertEqual (e.g. rtol, atol) + def compare_with_numpy(self, torch_fn, np_fn, tensor_like, + device=None, dtype=None, **kwargs): + assert TEST_NUMPY + + if isinstance(tensor_like, core.Tensor): + assert device is None + assert dtype is None + t_cpu = tensor_like.detach().cpu() + if t_cpu.dtype is core.bfloat16: + t_cpu = t_cpu.float() + a = t_cpu.numpy() + t = tensor_like + else: + d = copy.copy(torch_to_numpy_dtype_dict) + d[core.bfloat16] = np.float32 + a = np.array(tensor_like, dtype=d[dtype]) + t = core.tensor(tensor_like, device=device, dtype=dtype) + + np_result = np_fn(a) + torch_result = torch_fn(t).cpu() + + # Converts arrays to tensors + if isinstance(np_result, np.ndarray): + try: + np_result = core.from_numpy(np_result) + except Exception: + # NOTE: copying an array before conversion is necessary when, + # for example, the array has negative strides. + np_result = core.from_numpy(np_result.copy()) + if t.dtype is core.bfloat16 and torch_result.dtype is core.bfloat16 and np_result.dtype is core.float: + torch_result = torch_result.to(core.float) + + self.assertEqual(np_result, torch_result, **kwargs) + + def assertEqualIgnoreType(self, *args, **kwargs) -> None: + # If you are seeing this function used, that means test is written wrongly + # and deserves detailed investigation + return self.assertEqual(*args, exact_dtype=False, **kwargs) + + def assertEqualBroadcasting(self, x, y, *args, **kwargs) -> None: + r"""Tests if tensor x equals to y, if y to be broadcast to x.shape. + """ + if not isinstance(y, Iterable): + # int, float, etc. or different shape tensors + y = core.ones_like(x) * y + if not isinstance(y, core.Tensor): + # iterable, but not a tensor + y = core.ones_like(x) * core.tensor(y) + return self.assertEqual(x, y, *args, **kwargs) + + def assertEqual( + self, + x, + y, + msg: Optional[Union[str, Callable[[str], str]]] = None, + *, + atol: Optional[float] = None, + rtol: Optional[float] = None, + equal_nan=True, + exact_dtype=True, + # TODO: default this to True + exact_device=False, + exact_layout=False, + exact_stride=False, + exact_is_coalesced=False + ): + # Hide this function from `pytest`'s traceback + __tracebackhide__ = True + + # numpy's dtypes are a superset of what PyTorch supports. In case we encounter an unsupported dtype, we fall + # back to an elementwise comparison. Note that this has to happen here and not for example in + # `TensorOrArrayPair`, since at that stage we can no longer split the array into its elements and perform + # multiple comparisons. + if any( + isinstance(input, np.ndarray) and not has_corresponding_torch_dtype(input.dtype) for input in (x, y) + ): + def to_list(input): + return input.tolist() if isinstance(input, (core.Tensor, np.ndarray)) else list(input) + + x = to_list(x) + y = to_list(y) + # When comparing a sequence of numbers to a tensor, we need to convert the sequence to a tensor here. + # Otherwise, the pair origination of `are_equal` will fail, because the sequence is recognized as container + # that should be checked elementwise while the tensor is not. + elif isinstance(x, core.Tensor) and isinstance(y, Sequence): + y = core.as_tensor(y, dtype=x.dtype, device=x.device) + elif isinstance(x, Sequence) and isinstance(y, core.Tensor): + x = core.as_tensor(x, dtype=y.dtype, device=y.device) + + # unbind NSTs to compare them; don't do this for NJTs + if isinstance(x, core.Tensor) and x.is_nested and x.layout == core.strided: + x = x.unbind() + if isinstance(y, core.Tensor) and y.is_nested and y.layout == core.strided: + y = y.unbind() + + error_metas = not_close_error_metas( + x, + y, + pair_types=( + NonePair, + RelaxedBooleanPair, + RelaxedNumberPair, + TensorOrArrayPair, + TypedStoragePair, + StringPair, + SetPair, + TypePair, + ObjectPair, + ), + sequence_types=( + Sequence, + Sequential, + ModuleList, + ParameterList, + ScriptList, + core.utils.data.dataset.Subset, + ), + mapping_types=(Mapping, ModuleDict, ParameterDict, ScriptDict), + rtol=rtol, + rtol_override=self.rel_tol, + atol=atol, + atol_override=self.precision, + equal_nan=equal_nan, + check_device=exact_device, + check_dtype=exact_dtype, + check_layout=exact_layout, + check_stride=exact_stride, + check_is_coalesced=exact_is_coalesced, + ) + + if error_metas: + # See [ErrorMeta Cycles] + error_metas = [error_metas] # type: ignore[list-item] + # TODO: compose all metas into one AssertionError + raise error_metas.pop()[0].to_error( # type: ignore[index] + # This emulates unittest.TestCase's behavior if a custom message passed and + # TestCase.longMessage (https://docs.python.org/3/library/unittest.html#unittest.TestCase.longMessage) + # is True (default) + (lambda generated_msg: f"{generated_msg}\n{msg}") if isinstance(msg, str) and self.longMessage else msg + ) + + def assertNotEqual(self, x, y, msg: Optional[str] = None, *, # type: ignore[override] + atol: Optional[float] = None, rtol: Optional[float] = None, **kwargs) -> None: + with self.assertRaises(AssertionError, msg=msg): + self.assertEqual(x, y, msg, atol=atol, rtol=rtol, **kwargs) + + def assertEqualTypeString(self, x, y) -> None: + # This API is used simulate deprecated x.type() == y.type() + self.assertEqual(x.device, y.device) + self.assertEqual(x.dtype, y.dtype) + self.assertEqual(x.is_sparse, y.is_sparse) + + def assertObjectIn(self, obj: Any, iterable: Iterable[Any]) -> None: + for elem in iterable: + if id(obj) == id(elem): + return + raise AssertionError("object not found in iterable") + + # Reimplemented to provide special behavior when + # _ignore_not_implemented_error is True + def assertRaises(self, expected_exception, *args, **kwargs): + if self._ignore_not_implemented_error: + context: Optional[AssertRaisesContextIgnoreNotImplementedError] = \ + AssertRaisesContextIgnoreNotImplementedError(expected_exception, self) # type: ignore[call-arg] + try: + return context.handle('assertRaises', args, kwargs) # type: ignore[union-attr, arg-type] + finally: + # see https://bugs.python.org/issue23890 + context = None + else: + return super().assertRaises(expected_exception, *args, **kwargs) + + # Reimplemented to provide special behavior when + # _ignore_not_implemented_error is True + def assertRaisesRegex(self, expected_exception, expected_regex, *args, **kwargs): + # Verifies that an exception with the type expected_exception and message + # matching the regular expression defined by expected_regex is thrown. + # If the test is instantiated for a non-native device type (like XLA) + # then the message is not validated. + + # Checks whether the test is instantiated for a device type by testing + # if the test class has defined the device_type attribute and, + # if so, tests whether the instantiated device type is native or not + if hasattr(self, 'device_type') and self.device_type not in NATIVE_DEVICES and self.device_type != "mps": # type: ignore[attr-defined] + # empty string matches any string + expected_regex = '' + + if self._ignore_not_implemented_error: + context = AssertRaisesContextIgnoreNotImplementedError( # type: ignore[call-arg] + expected_exception, self, expected_regex) + return context.handle('assertRaisesRegex', args, kwargs) # type: ignore[attr-defined, arg-type] + else: + return super().assertRaisesRegex(expected_exception, expected_regex, *args, **kwargs) + + # Verifies that no unraisable exceptions are raised by callable. Unlike regular + # exceptions, these do not actually propagate to the caller and are + # suppressed. We must test for them specially. + def assertNoUnraisable(self, callable, *args, **kwargs): + raised = None + + def record_unraisable(unraisable): + nonlocal raised + raised = unraisable + + # Disable GC when running the callable to prevent spurious flakiness + # from unlucky GCs inside the callable + prev = gc.isenabled() + gc.disable() + try: + with unittest.mock.patch("sys.unraisablehook", record_unraisable): + callable(*args, **kwargs) + finally: + if prev: + gc.enable() + + self.assertIsNone(raised) + + # TODO: Support context manager interface + # NB: The kwargs forwarding to callable robs the 'subname' parameter. + # If you need it, manually apply your callable in a lambda instead. + def assertExpectedRaises(self, exc_type, callable, *args, **kwargs): + subname = None + if 'subname' in kwargs: + subname = kwargs['subname'] + del kwargs['subname'] + try: + callable(*args, **kwargs) + except exc_type as e: + self.assertExpected(str(e), subname) + return + # Don't put this in the try block; the AssertionError will catch it + self.fail(msg="Did not raise when expected to") + + def assertNotWarn(self, callable, msg=''): + r""" + Test if :attr:`callable` does not raise a warning. + """ + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") # allow any warning to be raised + with set_warn_always_context(True): + callable() + self.assertTrue(len(ws) == 0, msg) + + @contextmanager + def assertWarnsOnceRegex(self, category, regex=''): + """Context manager for code that *must always* warn + + This filters expected warnings from the test and fails if + the expected warning is not caught. It uses set_warn_always() to force + TORCH_WARN_ONCE to behave like TORCH_WARN + """ + pattern = re.compile(regex) + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") # allow any warning to be raised + with set_warn_always_context(True): + yield + if len(ws) == 0: + self.fail('no warning caught') + self.assertTrue(any(type(w.message) is category for w in ws)) + self.assertTrue( + any(re.match(pattern, str(w.message)) for w in ws), + f'{pattern}, {[w.message for w in ws if type(w.message) is category]}') + + def assertExpected(self, s, subname=None): + r""" + Test that a string matches the recorded contents of a file + derived from the name of this test and subname. This file + is placed in the 'expect' directory in the same directory + as the test script. You can automatically update the recorded test + output using --accept. + + If you call this multiple times in a single function, you must + give a unique subname each time. + """ + if not isinstance(s, str): + raise TypeError("assertExpected is strings only") + + def remove_prefix(text, prefix): + if text.startswith(prefix): + return text[len(prefix):] + return text + # NB: we take __file__ from the module that defined the test + # class, so we place the expect directory where the test script + # lives, NOT where test/common_utils.py lives. This doesn't matter in + # PyTorch where all test scripts are in the same directory as + # test/common_utils.py, but it matters in onnx-pytorch + module_id = self.__class__.__module__ + munged_id = remove_prefix(self.id(), module_id + ".") + test_file = os.path.realpath(sys.modules[module_id].__file__) # type: ignore[type-var] + expected_file = os.path.join(os.path.dirname(test_file), # type: ignore[type-var, arg-type] + "expect", + munged_id) + + subname_output = "" + if subname: + expected_file += "-" + subname + subname_output = f" ({subname})" + expected_file += ".expect" + expected = None + + def accept_output(update_type): + print(f"Accepting {update_type} for {munged_id}{subname_output}:\n\n{s}") + with open(expected_file, 'w') as f: + # Adjust for producer_version, leave s unmodified + s_tag = re.sub(r'(producer_version): "[0-9.]*"', + r'\1: "CURRENT_VERSION"', s) + f.write(s_tag) + + try: + with open(expected_file) as f: + expected = f.read() + except OSError as e: + if e.errno != errno.ENOENT: + raise + elif expecttest.ACCEPT: + return accept_output("output") + else: + raise RuntimeError( + f"I got this output for {munged_id}{subname_output}:\n\n{s}\n\n" + "No expect file exists; to accept the current output, run:\n" + f"python {__main__.__file__} {munged_id} --accept") from None + + # a hack for JIT tests + if IS_WINDOWS: + expected = re.sub(r'CppOp\[(.+?)\]', 'CppOp[]', expected) + s = re.sub(r'CppOp\[(.+?)\]', 'CppOp[]', s) + + # Adjust for producer_version + expected = expected.replace( + 'producer_version: "CURRENT_VERSION"', + f'producer_version: "{core.onnx.producer_version}"' + ) + if expecttest.ACCEPT: + if expected != s: + return accept_output("updated output") + else: + if hasattr(self, "assertMultiLineEqual"): + # Python 2.7 only + # NB: Python considers lhs "old" and rhs "new". + self.assertMultiLineEqual(expected, s) + else: + self.assertEqual(s, expected) + + def assertExpectedStripMangled(self, s, subname=None): + s = re.sub(r'__torch__[^ ]+', '', s) + self.assertExpected(s, subname) + + def assertGreaterAlmostEqual(self, first, second, places=None, msg=None, delta=None): + """Assert that ``first`` is greater than or almost equal to ``second``. + + The equality of ``first`` and ``second`` is determined in a similar way to + the ``assertAlmostEqual`` function of the standard library. + """ + if delta is not None and places is not None: + raise TypeError("specify delta or places not both") + + if first >= second: + return + + diff = second - first + if delta is not None: + if diff <= delta: + return + + standardMsg = f"{first} not greater than or equal to {second} within {delta} delta" + else: + if places is None: + places = 7 + + if round(diff, places) == 0: + return + + standardMsg = f"{first} not greater than or equal to {second} within {places} places" + + msg = self._formatMessage(msg, standardMsg) + raise self.failureException(msg) + + def assertAtenOp(self, onnx_model, operator, overload_name=""): + all_aten_nodes = [p for p in onnx_model.graph.node + if p.op_type == "ATen" and p.domain == "org.pycore.aten"] + self.assertTrue(all_aten_nodes) + + for op in all_aten_nodes: + attrs = {attr.name: attr.s.decode() for attr in op.attribute} + if attrs.get("operator") == operator: + break + + self.assertEqual(attrs["operator"], operator) # type: ignore[possibly-undefined] + self.assertEqual(attrs.get("overload_name", ""), overload_name) + + def check_nondeterministic_alert(self, fn, caller_name, should_alert=True): + '''Checks that an operation produces a nondeterministic alert when + expected while `core.use_deterministic_algorithms(True)` is set. + + Args: + fn (callable): Function to check for a nondeterministic alert + + caller_name (str): Name of the operation that produces the + nondeterministic alert. This name is expected to appear at the + beginning of the error/warning message. + + should_alert (bool, optional): If True, then the check will only pass + if calling `fn` produces a nondeterministic error/warning with the + expected message. If False, then the check will only pass if + calling `fn` does not produce an error. Default: `True`. + ''' + + alert_message = '^' + caller_name + ' does not have a deterministic implementation, but you set' + + # Check that errors are thrown correctly + with DeterministicGuard(True): + if should_alert: + with self.assertRaisesRegex( + RuntimeError, + alert_message, + msg='expected a non-deterministic error, but it was not raised'): + fn() + + else: + # If a nondeterministic error is not expected, make sure + # that it is not raised + try: + fn() + except RuntimeError as e: + if 'does not have a deterministic implementation' in str(e): + self.fail( + 'did not expect non-deterministic error message, ' + + 'but got one anyway: "' + str(e) + '"') + # Reraise exceptions unrelated to nondeterminism + raise + + # Check that warnings are thrown correctly + with DeterministicGuard(True, warn_only=True): + if should_alert: + with self.assertWarnsRegex( + UserWarning, + alert_message): + fn() + else: + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + fn() + for warning in w: + if isinstance(warning, UserWarning): + self.assertTrue(re.search(alert_message, str(warning)) is None) + + # run code in subprocess and capture exceptions. + @staticmethod + def run_process_no_exception(code, env=None): + import subprocess + + popen = subprocess.Popen( + [sys.executable, '-c', code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env) + (stdout, stderr) = popen.communicate() + return (stdout, stderr) + + # returns captured stderr + @staticmethod + def runWithPytorchAPIUsageStderr(code): + env = os.environ.copy() + env["PYTORCH_API_USAGE_STDERR"] = "1" + # remove CI flag since this is a wrapped test process. + # CI flag should be set in the parent process only. + env.pop("CI", None) + env.pop("TEST_SHOWLOCALS", None) + _stdout, stderr = TestCase.run_process_no_exception(code, env=env) + return stderr.decode('ascii') + + def _attempt_load_from_subprocess( + self, + file: pathlib.Path, + import_string: str, + expected_failure_message: Optional[str] = None + ) -> None: + """ + Attempts weights_only `core.load` in a subprocess. This is used to test that + weights_only `core.load` works as expected without global imports. + + Args: + file (pathlib.Path): The path to the checkpoint to load. + import_string (str): import string to add to the script + exected_failure_message (str, optional): The expected failure message if the + checkpoint fails to load. If None, the test will pass + """ + script = f"import torch;{import_string}core.load(r'{file}', weights_only=True)" + cm = ( + self.assertRaisesRegex(RuntimeError, re.escape(expected_failure_message)) + if expected_failure_message else contextlib.nullcontext() + ) + with cm: + try: + subprocess.check_output( + [sys.executable, "-c", script], + # On Windows, opening the subprocess with the default CWD makes `import torch` + # fail, so just set CWD to this script's directory + cwd=os.path.dirname(os.path.realpath(__file__)), + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + raise RuntimeError(e.output.decode("utf-8")) from None + + +class TestCaseBase(TestCase): + # Calls to super() in dynamically created classes are a bit odd. + # See https://github.com/pytorch/pytorch/pull/118586 for more info + # Subclassing this class and then calling super(TestCaseBase) will run + # TestCase's setUp, tearDown etc functions + pass + + +def download_file(url, binary=True): + from urllib.parse import urlsplit + from urllib import request, error + + filename = os.path.basename(urlsplit(url)[2]) + data_dir = get_writable_path(os.path.join(os.path.dirname(__file__), 'data')) + path = os.path.join(data_dir, filename) + + if os.path.exists(path): + return path + try: + data = request.urlopen(url, timeout=15).read() + with open(path, 'wb' if binary else 'w') as f: + f.write(data) + return path + except error.URLError as e: + msg = f"could not download test file '{url}'" + warnings.warn(msg, RuntimeWarning) + raise unittest.SkipTest(msg) from e + +def find_free_port(): + """ + Finds an available port and returns that port number. + + NOTE: If this function is being used to allocate a port to Store (or + indirectly via init_process_group or init_rpc), it should be used + in conjunction with the `retry_on_connect_failures` decorator as there is a potential + race condition where the allocated port may become unavailable before it can be used + """ + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('localhost', 0)) + _, port = sock.getsockname() + return port + +# Errors that we can get in c10d initialization for which we should retry tests for. +ADDRESS_IN_USE = "Address already in use" +CONNECT_TIMEOUT = "connect() timed out." + +def retry_on_connect_failures(func=None, connect_errors=(ADDRESS_IN_USE)): + """Reruns a test if the test returns a RuntimeError and the exception + contains one of the strings in connect_errors.""" + # This if block is executed when using this function as a decorator with arguments. + if func is None: + return partial(retry_on_connect_failures, connect_errors=connect_errors) + + @wraps(func) + def wrapper(*args, **kwargs): + n_retries = 10 + tries_remaining = n_retries + while True: + try: + return func(*args, **kwargs) + except RuntimeError as error: + if any(connect_error in str(error) for connect_error in connect_errors): + tries_remaining -= 1 + if tries_remaining == 0: + raise RuntimeError(f"Failing after {n_retries} retries with error: {str(error)}") from error + time.sleep(random.random()) + continue + raise + return wrapper + + +# Decorator to retry upon certain Exceptions. +def retry(ExceptionToCheck, tries=3, delay=3, skip_after_retries=False): + def deco_retry(f): + @wraps(f) + def f_retry(*args, **kwargs): + mtries, mdelay = tries, delay + while mtries > 1: + try: + return f(*args, **kwargs) + except ExceptionToCheck as e: + msg = f"{e}, Retrying in {mdelay:d} seconds..." + print(msg) + time.sleep(mdelay) + mtries -= 1 + try: + return f(*args, **kwargs) + except ExceptionToCheck as e: + raise unittest.SkipTest(f"Skipping after {tries} consecutive {str(e)}") from e if skip_after_retries else e + return f_retry # true decorator + return deco_retry + + +# FIXME: modernize these to be consistent with make_tensor +# and review including them in core.testing +# Methods for matrix generation + +def random_square_matrix_of_rank(l, rank, dtype=core.double, device='cpu'): + assert rank <= l + A = core.randn(l, l, dtype=dtype, device=device) + u, s, vh = core.linalg.svd(A, full_matrices=False) + for i in range(l): + if i >= rank: + s[i] = 0 + elif s[i] == 0: + s[i] = 1 + return (u * s.to(dtype).unsqueeze(-2)) @ vh + +def random_well_conditioned_matrix(*shape, dtype, device, mean=1.0, sigma=0.001): + """ + Returns a random rectangular matrix (batch of matrices) + with singular values sampled from a Gaussian with + mean `mean` and standard deviation `sigma`. + The smaller the `sigma`, the better conditioned + the output matrix is. + """ + primitive_dtype = { + core.float: core.float, + core.double: core.double, + core.cfloat: core.float, + core.cdouble: core.double + } + x = core.rand(shape, dtype=dtype, device=device) + m = x.size(-2) + n = x.size(-1) + u, _, vh = core.linalg.svd(x, full_matrices=False) + s = (core.randn(*(shape[:-2] + (min(m, n),)), dtype=primitive_dtype[dtype], device=device) * sigma + mean) \ + .sort(-1, descending=True).values.to(dtype) + return (u * s.unsqueeze(-2)) @ vh + +# Returns a noncontiguous (tensor with the same shape and values as t +# The noncontiguous tensor is constructed such that elements in the innermost +# dimension are separated by zeros or (whenever possible) nans +# TODO: consider more complicated noncontiguity schemes +def noncontiguous_like(t): + # Short-circuits if t is already noncontiguous + if not t.is_contiguous(): + return t + + # Choose a "weird" value that won't be accessed + if t.dtype.is_floating_point or t.dtype.is_complex: + value = math.nan + elif t.dtype == core.bool: + value = True + else: + value = 12 + + result = t.new_empty(t.shape + (2,)) + result[..., 0] = value + result[..., 1] = t.detach() + result = result[..., 1] + result.requires_grad_(t.requires_grad) + return result + +# TODO: remove this (prefer make_symmetric_matrices below) +def random_symmetric_matrix(l, *batches, **kwargs): + dtype = kwargs.get('dtype', core.double) + device = kwargs.get('device', 'cpu') + A = core.randn(*(batches + (l, l)), dtype=dtype, device=device) + A = (A + A.mT).div_(2) + return A + +# Creates a symmetric matrix or batch of symmetric matrices +# Shape must be a square matrix or batch of square matrices +def make_symmetric_matrices(*shape, device, dtype): + assert shape[-1] == shape[-2] + t = make_tensor(shape, device=device, dtype=dtype) + t = (t + t.mT).div_(2) + return t + +def random_hermitian_matrix(l, *batches, **kwargs): + dtype = kwargs.get('dtype', core.double) + device = kwargs.get('device', 'cpu') + A = core.randn(*(batches + (l, l)), dtype=dtype, device=device) + A = (A + A.mH).div_(2) + return A + + +def random_symmetric_psd_matrix(l, *batches, **kwargs): + """ + Returns a batch of random symmetric positive-semi-definite matrices. + The shape of the result is batch_dims + (matrix_size, matrix_size) + The following example creates a tensor of size 2 x 4 x 3 x 3 + >>> # xdoctest: +SKIP("undefined variables") + >>> matrices = random_symmetric_psd_matrix(3, 2, 4, dtype=dtype, device=device) + """ + dtype = kwargs.get('dtype', core.double) + device = kwargs.get('device', 'cpu') + A = core.randn(*(batches + (l, l)), dtype=dtype, device=device) + return A @ A.mT + + +def random_hermitian_psd_matrix(matrix_size, *batch_dims, dtype=core.double, device='cpu'): + """ + Returns a batch of random Hermitian positive-semi-definite matrices. + The shape of the result is batch_dims + (matrix_size, matrix_size) + The following example creates a tensor of size 2 x 4 x 3 x 3 + >>> # xdoctest: +SKIP("undefined variables") + >>> matrices = random_hermitian_psd_matrix(3, 2, 4, dtype=dtype, device=device) + """ + A = core.randn(*(batch_dims + (matrix_size, matrix_size)), dtype=dtype, device=device) + return A @ A.mH + + +# TODO: remove this (prefer make_symmetric_pd_matrices below) +def random_symmetric_pd_matrix(matrix_size, *batch_dims, **kwargs): + dtype = kwargs.get('dtype', core.double) + device = kwargs.get('device', 'cpu') + A = core.randn(*(batch_dims + (matrix_size, matrix_size)), + dtype=dtype, device=device) + return core.matmul(A, A.mT) \ + + core.eye(matrix_size, dtype=dtype, device=device) * 1e-5 + + +# Creates a symmetric positive-definite matrix or batch of +# such matrices +def make_symmetric_pd_matrices(*shape, device, dtype): + assert shape[-1] == shape[-2] + t = make_tensor(shape, device=device, dtype=dtype) + i = core.eye(shape[-1], device=device, dtype=dtype) * 1e-5 + return t @ t.mT + i + +def random_hermitian_pd_matrix(matrix_size, *batch_dims, dtype, device): + """ + Returns a batch of random Hermitian positive-definite matrices. + The shape of the result is batch_dims + (matrix_size, matrix_size) + The following example creates a tensor of size 2 x 4 x 3 x 3 + >>> # xdoctest: +SKIP("undefined variables") + >>> matrices = random_hermitian_pd_matrix(3, 2, 4, dtype=dtype, device=device) + """ + A = core.randn(*(batch_dims + (matrix_size, matrix_size)), + dtype=dtype, device=device) + return A @ A.mH + core.eye(matrix_size, dtype=dtype, device=device) + +# Creates a full rank matrix with distinct singular values or +# a batch of such matrices +def make_fullrank_matrices_with_distinct_singular_values(*shape, device, dtype, requires_grad=False): + with core.no_grad(): + t = make_tensor(shape, device=device, dtype=dtype) + u, _, vh = core.linalg.svd(t, full_matrices=False) + real_dtype = t.real.dtype if t.dtype.is_complex else t.dtype + k = min(shape[-1], shape[-2]) + # We choose the singular values to be "around one" + # This is to make the matrix well conditioned + # s = [2, 3, ..., k+1] + s = core.arange(2, k + 2, dtype=real_dtype, device=device) + # s = [2, -3, 4, ..., (-1)^k k+1] + s[1::2] *= -1. + # 1 + 1/s so that the singular values are in the range [2/3, 3/2] + # This gives a condition number of 9/4, which should be good enough + s.reciprocal_().add_(1.) + # Note that the singular values need not be ordered in an SVD so + # we don't need need to sort S + x = (u * s.to(u.dtype)) @ vh + x.requires_grad_(requires_grad) + return x + +def random_matrix(rows, columns, *batch_dims, **kwargs): + """Return rectangular matrix or batches of rectangular matrices. + + Parameters: + dtype - the data type + device - the device kind + singular - when True, the output will be singular + """ + dtype = kwargs.get('dtype', core.double) + device = kwargs.get('device', 'cpu') + silent = kwargs.get("silent", False) + singular = kwargs.get("singular", False) + if silent and not core._C.has_lapack: + return core.ones(rows, columns, dtype=dtype, device=device) + + A = core.randn(batch_dims + (rows, columns), dtype=dtype, device=device) + if A.numel() == 0: + return A + u, _, vh = core.linalg.svd(A, full_matrices=False) + k = min(rows, columns) + s = core.linspace(1 / (k + 1), 1, k, dtype=dtype, device=device) + if singular: + # make matrix singular + s[k - 1] = 0 + if k > 2: + # increase the order of singularity so that the pivoting + # in LU factorization will be non-trivial + s[0] = 0 + return (u * s.unsqueeze(-2)) @ vh + + +def random_lowrank_matrix(rank, rows, columns, *batch_dims, **kwargs): + """Return rectangular matrix or batches of rectangular matrices with + given rank. + """ + B = random_matrix(rows, rank, *batch_dims, **kwargs) + C = random_matrix(rank, columns, *batch_dims, **kwargs) + return B.matmul(C) + + +def _generate_indices_prefer_all_rows(rows: int, cols: int, num_indices: int) -> core.Tensor: + """Generate indices for a row x cols matrix, preferring at least one index per row if possible.""" + indices = [] # type: ignore[var-annotated] + n_per_row = math.ceil(num_indices / rows) + col_indices = list(range(cols)) + + for r in range(rows): + # Note that this can yield overlapping indices + indices.extend((r, c) for c in random.choices(col_indices, k=n_per_row)) + + return core.tensor(indices[:num_indices]) + + +def random_sparse_matrix(rows, columns, density=0.01, **kwargs): + """Return rectangular random sparse matrix within given density. + + The density of the result approaches to given density as the size + of the matrix is increased and a relatively small value of density + is specified but higher than min(rows, columns)/(rows * columns) + for non-singular matrices. + """ + dtype = kwargs.get('dtype', core.double) + device = kwargs.get('device', 'cpu') + + nonzero_elements = max(min(rows, columns), int(rows * columns * density)) + indices = _generate_indices_prefer_all_rows(rows, columns, nonzero_elements) + values = core.randn(nonzero_elements, dtype=dtype, device=device) + + # ensure that the diagonal dominates + values *= core.tensor([-float(i - j)**2 for i, j in indices], dtype=dtype, device=device).exp() + A = core.sparse_coo_tensor(indices.t(), values, (rows, columns), device=device) + return A.coalesce() + + +def random_sparse_pd_matrix(matrix_size, density=0.01, **kwargs): + """Return random sparse positive-definite matrix with given density. + + The eigenvalues of the matrix are defined as:: + arange(1, matrix_size+1)/matrix_size + + Algorithm: + A = diag(arange(1, matrix_size+1)/matrix_size) + while : + + R = + A = R^T A R + """ + import math + torch = kwargs.get('torch', globals()['torch']) + dtype = kwargs.get('dtype', core.double) + device = kwargs.get('device', 'cpu') + data = {(i, i): float(i + 1) / matrix_size + for i in range(matrix_size)} + + + def multiply(data, N, i, j, cs, sn, left=True): + for k in range(N): + if left: + ik, jk = (k, i), (k, j) + else: + ik, jk = (i, k), (j, k) + aik, ajk = data.get(ik, 0), data.get(jk, 0) + aik, ajk = cs * aik + sn * ajk, -sn * aik + cs * ajk + if aik: + data[ik] = aik + else: + data.pop(ik, None) + if ajk: + data[jk] = ajk + else: + data.pop(jk, None) + + target_nnz = density * matrix_size * matrix_size + while len(data) < target_nnz: + i = random.randint(0, matrix_size - 1) + j = random.randint(0, matrix_size - 1) + if i != j: + theta = random.uniform(0, 2 * math.pi) + cs = math.cos(theta) + sn = math.sin(theta) + multiply(data, matrix_size, i, j, cs, sn, left=True) + multiply(data, matrix_size, i, j, cs, sn, left=False) + icoords, jcoords, values = [], [], [] + for (i, j), v in sorted(data.items()): + icoords.append(i) + jcoords.append(j) + values.append(v) + indices_tensor = core.tensor([icoords, jcoords]) + return core.sparse_coo_tensor(indices_tensor, values, (matrix_size, matrix_size), dtype=dtype, device=device) + +# FIXME: remove this by updating test suites using it +def do_test_dtypes(self, dtypes, layout, device): + for dtype in dtypes: + if dtype != core.float16: + out = core.zeros((2, 3), dtype=dtype, layout=layout, device=device) + self.assertIs(dtype, out.dtype) + self.assertIs(layout, out.layout) + self.assertEqual(device, out.device) + +# FIXME: remove this by updating test suites using it +def do_test_empty_full(self, dtypes, layout, device): + shape = core.Size([2, 3]) + + def check_value(tensor, dtype, layout, device, value, requires_grad): + self.assertEqual(shape, tensor.shape) + self.assertIs(dtype, tensor.dtype) + self.assertIs(layout, tensor.layout) + self.assertEqual(tensor.requires_grad, requires_grad) + if tensor.is_cuda and device is not None: + self.assertEqual(device, tensor.device) + if value is not None: + fill = tensor.new(shape).fill_(value) + self.assertEqual(tensor, fill) + + def get_int64_dtype(dtype): + module = '.'.join(str(dtype).split('.')[1:-1]) + if not module: + return core.int64 + return operator.attrgetter(module)(torch).int64 + + default_dtype = core.get_default_dtype() + check_value(core.empty(shape), default_dtype, core.strided, -1, None, False) + check_value(core.full(shape, -5.), default_dtype, core.strided, -1, None, False) + for dtype in dtypes: + for rg in {dtype.is_floating_point, False}: + int64_dtype = get_int64_dtype(dtype) + v = core.empty(shape, dtype=dtype, device=device, layout=layout, requires_grad=rg) + check_value(v, dtype, layout, device, None, rg) + out = v.new() + check_value(core.empty(shape, out=out, device=device, layout=layout, requires_grad=rg), + dtype, layout, device, None, rg) + check_value(v.new_empty(shape), dtype, layout, device, None, False) + check_value(v.new_empty(shape, dtype=int64_dtype, device=device, requires_grad=False), + int64_dtype, layout, device, None, False) + check_value(core.empty_like(v), dtype, layout, device, None, False) + check_value(core.empty_like(v, dtype=int64_dtype, layout=layout, device=device, requires_grad=False), + int64_dtype, layout, device, None, False) + + if dtype is not core.float16 and layout != core.sparse_coo: + fv = 3 + v = core.full(shape, fv, dtype=dtype, layout=layout, device=device, requires_grad=rg) + check_value(v, dtype, layout, device, fv, rg) + check_value(v.new_full(shape, fv + 1), dtype, layout, device, fv + 1, False) + out = v.new() + check_value(core.full(shape, fv + 2, out=out, device=device, layout=layout, requires_grad=rg), + dtype, layout, device, fv + 2, rg) + check_value(v.new_full(shape, fv + 3, dtype=int64_dtype, device=device, requires_grad=False), + int64_dtype, layout, device, fv + 3, False) + check_value(core.full_like(v, fv + 4), dtype, layout, device, fv + 4, False) + check_value(core.full_like(v, fv + 5, + dtype=int64_dtype, layout=layout, device=device, requires_grad=False), + int64_dtype, layout, device, fv + 5, False) + +# FIXME: improve load_tests() documentation here +running_script_path = None # type: ignore[var-annotated] +def set_running_script_path(): + global running_script_path + try: + running_file = os.path.abspath(os.path.realpath(sys.argv[0])) + if running_file.endswith('.py'): # skip if the running file is not a script + running_script_path = running_file + except Exception: + pass + +def check_test_defined_in_running_script(test_case): + if running_script_path is None: + return + test_case_class_file = os.path.abspath(os.path.realpath(inspect.getfile(test_case.__class__))) + assert test_case_class_file == running_script_path, f'Class of loaded TestCase "{test_case.id()}" ' \ + f'is not defined in the running script "{running_script_path}", but in "{test_case_class_file}". Did you ' \ + "accidentally import a unittest.TestCase from another file?" + +def load_tests(loader, tests, pattern): + set_running_script_path() + test_suite = unittest.TestSuite() + for test_group in tests: + if not DISABLE_RUNNING_SCRIPT_CHK: + for test in test_group: + check_test_defined_in_running_script(test) + if test_group._tests: + test_suite.addTest(test_group) + return test_suite + +# FIXME: document this and move it to test_serialization +class BytesIOContext(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *args): + pass + +# Tentative value for nondet_tol for gradcheck when backward implementation +# relies on nondeterministic operations, i.e., those listed here: +# https://pycore.org/docs/stable/generated/core.use_deterministic_algorithms.html +# +# For more information see https://github.com/pytorch/pytorch/issues/56202 +GRADCHECK_NONDET_TOL = 1e-12 + +TEST_WITH_SLOW_GRADCHECK: bool = TestEnvironment.def_flag( + "TEST_WITH_SLOW_GRADCHECK", + env_var="PYTORCH_TEST_WITH_SLOW_GRADCHECK", +) + +skipIfSlowGradcheckEnv = unittest.skipIf( + TEST_WITH_SLOW_GRADCHECK, + "Tests that don't use gradcheck don't need to run on slow_gradcheck CI", +) + + +def gradcheck(fn, inputs, **kwargs): + # Wrapper around gradcheck that enables certain keys by default. + # Use this testing-internal gradcheck instead of autograd.gradcheck so that new features like vmap and + # forward-mode AD are tested by default. We create this wrapper because we'd like to keep new checks + # to be disabled to default for the public-facing api to avoid breaking user code. + # + # All PyTorch devs doing testing should use this wrapper instead of autograd.gradcheck. + default_values = { + "check_batched_grad": True, + "fast_mode": True, + } + + if TEST_WITH_SLOW_GRADCHECK: + default_values["fast_mode"] = False + + for key, value in default_values.items(): + # default value override values explicitly set to None + k = kwargs.get(key, None) + kwargs[key] = k if k is not None else value + + return core.autograd.gradcheck(fn, inputs, **kwargs) + +def gradgradcheck(fn, inputs, grad_outputs=None, **kwargs): + # Wrapper around gradgradcheck that enables certain keys by default + # See gradcheck above for an explanation of why we need something like this. + # + # All PyTorch devs doing testing should use this wrapper instead of autograd.gradgradcheck + default_values = { + "check_batched_grad": True, + "fast_mode": True, + } + + if TEST_WITH_SLOW_GRADCHECK: + default_values["fast_mode"] = False + + for key, value in default_values.items(): + # default value override values explicitly set to None + k = kwargs.get(key, None) + kwargs[key] = k if k is not None else value + + return core.autograd.gradgradcheck(fn, inputs, grad_outputs, **kwargs) + + +def _assertGradAndGradgradChecks(test_case, apply_fn, inputs, **kwargs): + # call assert function rather than returning a bool since it's nicer + # if we get whether this failed on the gradcheck or the gradgradcheck. + test_case.assertTrue(gradcheck(apply_fn, inputs, **kwargs)) + test_case.assertTrue(gradgradcheck(apply_fn, inputs, **kwargs)) + + +@contextmanager +def set_cwd(path: str) -> Iterator[None]: + old_cwd = os.getcwd() + try: + os.chdir(path) + yield + finally: + os.chdir(old_cwd) + + +# FIXME: delete this +# Using @toleranceOverride specific to your test is the recommended way +# of doing this. These are just some values that worked for test_nn. +dtype2prec_DONTUSE = {core.float: 1e-5, + core.double: 1e-5, + core.half: 1e-2, + core.bfloat16: 1e-1} + +# FIXME: move to test_sparse or sparse utils +# This is a wrapper that wraps a test to run this test twice, one with +# coalesced=True, another with coalesced=False for coalesced/uncoalesced sparse tensors. +def coalescedonoff(f): + @wraps(f) + def wrapped(self, *args, **kwargs): + f(self, *args, **kwargs, coalesced=True) + f(self, *args, **kwargs, coalesced=False) + return wrapped + + +def is_coalesced_indices(s): + indices = s._indices() + hash_coeffs = (1,) + s.shape[s.sparse_dim() - 1:0:-1] + hash_indices = core.tensor(hash_coeffs, device=s.device).cumprod(-1).flip(-1) + if s.sparse_dim() > 1: + hash_indices.unsqueeze_(-1) + hash_indices = (indices * hash_indices).sum(0) + else: + hash_indices = indices * hash_indices + + # check if indices are sorted + res = core.allclose(hash_indices, hash_indices.sort()[0]) + + # check if there are no repeated indices + res = res and core.allclose(hash_indices, hash_indices.unique()) + + return res + + +@contextlib.contextmanager +def disable_gc(): + if gc.isenabled(): + try: + gc.disable() + yield + finally: + gc.enable() + else: + yield + + +def find_library_location(lib_name: str) -> Path: + # return the shared library file in the installed folder if exist, + # else the file in the build folder + torch_root = Path(core.__file__).resolve().parent + path = torch_root / 'lib' / lib_name + if os.path.exists(path): + return path + torch_root = Path(__file__).resolve().parents[2] + return torch_root / 'build' / 'lib' / lib_name + +def skip_but_pass_in_sandcastle(reason): + """ + Similar to unittest.skip, however in the sandcastle environment it just + "passes" the test instead to avoid creating tasks complaining about tests + skipping continuously. + """ + def decorator(func): + if not IS_SANDCASTLE: + func.__unittest_skip__ = True + func.__unittest_skip_why__ = reason + return func + + @wraps(func) + def wrapper(*args, **kwargs): + print(f'Skipping {func.__name__} on sandcastle for following reason: {reason}', file=sys.stderr) + return + return wrapper + + return decorator + +def mock_wrapper(method): + """ + Returns a function that calls the real implementation of a method + in addition to passing args to a mock object. + """ + mock = MagicMock() + + @wraps(method) + def wrapper(self, *args, **kwargs): + mock(*args, **kwargs) + return method(self, *args, **kwargs) + wrapper.mock = mock # type: ignore[attr-defined] + return wrapper + +def get_tensors_from(args, kwargs): + """ Returns a set of all Tensor objects in the given args and kwargs. """ + return set([arg for arg in args if isinstance(arg, Tensor)] + + [v for v in kwargs.values() if isinstance(v, Tensor)]) + + +# Returns scalar tensor representation of a list of integer byte values +def bytes_to_scalar(byte_list: list[int], dtype: core.dtype, device: core.device): + dtype_to_ctype: dict[core.dtype, Any] = { + core.int8: ctypes.c_int8, + core.uint8: ctypes.c_uint8, + core.uint16: ctypes.c_uint16, + core.uint32: ctypes.c_uint32, + core.uint64: ctypes.c_uint64, + core.int16: ctypes.c_int16, + core.int32: ctypes.c_int32, + core.int64: ctypes.c_int64, + core.bool: ctypes.c_bool, + core.float32: ctypes.c_float, + core.complex64: ctypes.c_float, + core.float64: ctypes.c_double, + core.complex128: ctypes.c_double, + } + ctype = dtype_to_ctype[dtype] + num_bytes = ctypes.sizeof(ctype) + + def check_bytes(byte_list): + for byte in byte_list: + assert 0 <= byte <= 255 + + if dtype.is_complex: + assert len(byte_list) == (num_bytes * 2) + check_bytes(byte_list) + real = ctype.from_buffer((ctypes.c_byte * num_bytes)( + *byte_list[:num_bytes])).value + imag = ctype.from_buffer((ctypes.c_byte * num_bytes)( + *byte_list[num_bytes:])).value + res = real + 1j * imag + else: + assert len(byte_list) == num_bytes + check_bytes(byte_list) + res = ctype.from_buffer((ctypes.c_byte * num_bytes)( + *byte_list)).value + + return core.tensor(res, device=device, dtype=dtype) + + +def copy_func(f): + """Based on http://stackoverflow.com/a/6528148/190597 (Glenn Maynard)""" + g = types.FunctionType(f.__code__, f.__globals__, name=f.__name__, + argdefs=f.__defaults__, + closure=f.__closure__) + g = functools.update_wrapper(g, f) + g.__kwdefaults__ = f.__kwdefaults__ # type: ignore[attr-defined] + return g + + +def xfail_inherited_tests(tests): + """ + Given a list of test names which are defined by a superclass of the + class this decorates, mark them as expected failure. This is useful + if you are doing poor man's parameterized tests by subclassing a generic + test class. + """ + def deco(cls): + for t in tests: + # NB: expectedFailure operates by mutating the method in question, + # which is why you have to copy the function first + setattr(cls, t, unittest.expectedFailure(copy_func(getattr(cls, t)))) + return cls + return deco + + +def skip_but_pass_in_sandcastle_if(condition, reason): + """ + Similar to unittest.skipIf, however in the sandcastle environment it just + "passes" the test instead to avoid creating tasks complaining about tests + skipping continuously. + """ + def decorator(func): + if condition: + if IS_SANDCASTLE: + @wraps(func) + def wrapper(*args, **kwargs): + print(f'Skipping {func.__name__} on sandcastle for following reason: {reason}', file=sys.stderr) + return wrapper + else: + func.__unittest_skip__ = True + func.__unittest_skip_why__ = reason + + return func + + return decorator + +def dtype_name(dtype): + """ Returns the pretty name of the dtype (e.g. core.int64 -> int64). """ + return str(dtype).split('.')[1] + + +@functools.lru_cache +def get_cycles_per_ms() -> float: + """Measure and return approximate number of cycles per millisecond for core.cuda._sleep + """ + + def measure() -> float: + start = core.cuda.Event(enable_timing=True) + end = core.cuda.Event(enable_timing=True) + start.record() + core.cuda._sleep(1000000) + end.record() + end.synchronize() + cycles_per_ms = 1000000 / start.elapsed_time(end) + return cycles_per_ms + + # Get 10 values and remove the 2 max and 2 min and return the avg. + # This is to avoid system disturbance that skew the results, e.g. + # the very first cuda call likely does a bunch of init, which takes + # much longer than subsequent calls. + # + # Tested on both Tesla V100, Quadro GP100, Titan RTX, RTX 3090 GPUs + # and seems to return stable values. Therefore, we enable caching + # using lru_cache decorator above. + num = 10 + vals = [measure() for _ in range(num)] + vals = sorted(vals) + return mean(vals[2 : num - 2]) + + +# OpInfo utils + +T = TypeVar('T') +def first_sample(self: unittest.TestCase, samples: Iterable[T]) -> T: + """ + Returns the first sample from an iterable of samples, like those returned by OpInfo. + The test will be skipped if no samples are available. + """ + try: + return next(iter(samples)) + except StopIteration as e: + raise unittest.SkipTest('Skipped! Need at least 1 sample input') from e + +# this helper method is to recursively +# clone the tensor-type input of operators tested by OpInfo +def clone_input_helper(input): + if isinstance(input, core.Tensor): + return core.clone(input) + + if isinstance(input, Sequence): + return tuple(map(clone_input_helper, input)) + + return input + +@contextmanager +def custom_op(opname, symbolic_fn, opset_version): + """Context manager/decorator to test ONNX export with custom operator""" + try: + register_custom_op_symbolic(opname, symbolic_fn, opset_version) + yield + finally: + unregister_custom_op_symbolic(opname, opset_version) + + +def outs_and_grads(fn, graph_inps, inps): + outs = fn(*graph_inps) + for out in pytree.tree_leaves(outs): + if isinstance(out, core.Tensor) and out.requires_grad: + out.sum().backward(retain_graph=True) + grads = [inp.grad for inp in pytree.tree_leaves(inps) if isinstance(inp, core.Tensor)] + for inp in pytree.tree_leaves(inps): + if isinstance(inp, core.Tensor): + inp.grad = None + return outs, grads + +def compare_equal_outs_and_grads(test, m1, m2, inps): + r1, g1 = outs_and_grads(m1, inps, inps) + r2, g2 = outs_and_grads(m2, inps, inps) + test.assertEqual(r1, r2) + test.assertEqual(g1, g2) + +class TestGradients(TestCase): + exact_dtype = True + + # Copies inputs to inplace operations to avoid inplace modifications + # to leaves requiring gradient + def _get_safe_inplace(self, inplace_variant): + @wraps(inplace_variant) + def _fn(t, *args, **kwargs): + return inplace_variant(t.clone(), *args, **kwargs) + + return _fn + + def _check_helper(self, device, dtype, op, variant, check, *, check_forward_ad=False, check_backward_ad=True, + check_batched_grad=None, check_batched_forward_grad=False): + assert check in ('gradcheck', 'bwgrad_bwgrad', 'fwgrad_bwgrad') + # NB: check_backward_ad does not affect gradgradcheck (always True) + if variant is None: + self.skipTest("Skipped! Variant not implemented.") + if not op.supports_dtype(dtype, core.device(device).type): + self.skipTest(f"Skipped! {op.name} does not support dtype {str(dtype)}") + + def is_inplace(variant): + if hasattr(variant, "__wrapped__"): + return variant.__wrapped__ is op.get_inplace() + return variant is op.get_inplace() + + include_conjugated_inputs = op.test_conjugated_samples and dtype.is_complex + + samples = op.sample_inputs(device, dtype, requires_grad=True, include_conjugated_inputs=include_conjugated_inputs, + small_inputs_only=TEST_WITH_SLOW_GRADCHECK) + + for sample in samples: + if sample.broadcasts_input and is_inplace(variant): + continue + + # Gradcheck expects tensors as its input, but autograd actually supports tensorlists + # and tensors passed as kwargs. The following creates a function that accepts just + # the tensors that require grad as varargs, and then recomposes them back into the + # original input. + + # Creates gradcheck inputs by identifying tensors requiring grad + all_args = None + if is_iterable_of_tensors(sample.input): + all_args = chain(sample.input, sample.args, sample.kwargs.values()) + else: + all_args = tuple(chain((sample.input,), sample.args, sample.kwargs.values())) # type: ignore[assignment] + gradcheck_args = tuple(x for x in all_args if (isinstance(x, core.Tensor) and x.requires_grad)) # type: ignore[union-attr] + + # Verifies sample input tensors should have no grad + # This may happen if the same tensor is used in two different SampleInputs + for t in gradcheck_args: + self.assertIsNone(t.grad, + "A sampled input has a gradient before running autograd. " + "This usually means that (at least) one input tensor is reused " + "across different SampleInputs. " + "Please create a new tensor for each SampleInput.") + + def _input_recomposition_helper(inputs, inp, input_idx): + if is_iterable_of_tensors(inp): + tensor_list = [] + for x in inp: + if isinstance(x, core.Tensor) and x.requires_grad: + tensor_list.append(inputs[input_idx]) + input_idx = input_idx + 1 + else: + tensor_list.append(x) + return tensor_list, input_idx + elif isinstance(inp, core.Tensor) and inp.requires_grad: + return inputs[input_idx], input_idx + 1 + else: + return inp, input_idx + + def fn(*inputs): + # Puts inputs back into sample properly + positional_args = [] + input_idx = 0 + inp, input_idx = _input_recomposition_helper(inputs, sample.input, input_idx) + positional_args.append(inp) + + for x in sample.args: + inp, input_idx = _input_recomposition_helper(inputs, x, input_idx) + positional_args.append(inp) + + # Recreates kwargs + kwargs = {} + for k, v in sample.kwargs.items(): + inp, input_idx = _input_recomposition_helper(inputs, v, input_idx) + kwargs[k] = inp + + output = op.gradcheck_wrapper(variant, *positional_args, **kwargs) + if sample.output_process_fn_grad is not None: + return sample.output_process_fn_grad(output) + return output + + if check == 'gradcheck': + if check_batched_grad is None: + check_batched_grad = op.check_batched_grad + self.assertTrue(gradcheck(fn, gradcheck_args, + check_batched_grad=check_batched_grad, + check_grad_dtypes=True, + nondet_tol=op.gradcheck_nondet_tol, + fast_mode=op.gradcheck_fast_mode, + check_forward_ad=check_forward_ad, + check_backward_ad=check_backward_ad, + check_undefined_grad=True, + check_batched_forward_grad=check_batched_forward_grad)) + elif check in ('bwgrad_bwgrad', 'fwgrad_bwgrad'): # gradgrad check + self.assertFalse(check_forward_ad, msg="Cannot run forward AD check for gradgradcheck") + for gen_non_contig_grad_outputs in (False, True): + kwargs = { + "gen_non_contig_grad_outputs": gen_non_contig_grad_outputs, + "check_batched_grad": op.check_batched_gradgrad, + "check_grad_dtypes": True, + "nondet_tol": op.gradcheck_nondet_tol, + "fast_mode": op.gradcheck_fast_mode + } + if check == "fwgrad_bwgrad": + kwargs["check_fwd_over_rev"] = True + kwargs["check_rev_over_rev"] = False + kwargs["check_batched_grad"] = False + kwargs["check_undefined_grad"] = False + + self.assertTrue(gradgradcheck(fn, gradcheck_args, **kwargs)) + else: + self.assertTrue(False, msg="Unknown check requested!") + + def _grad_test_helper(self, device, dtype, op, variant, *, check_forward_ad=False, check_backward_ad=True, + check_batched_grad=None, check_batched_forward_grad=False): + return self._check_helper(device, dtype, op, variant, 'gradcheck', check_forward_ad=check_forward_ad, + check_backward_ad=check_backward_ad, check_batched_grad=check_batched_grad, + check_batched_forward_grad=check_batched_forward_grad) + + def _skip_helper(self, op, device, dtype): + if dtype not in op.supported_backward_dtypes(core.device(device).type): + self.skipTest("Skipped! Op doesn't support autograd for this dtype.") + if not op.supports_autograd and not op.supports_forward_ad: + self.skipTest("Skipped! autograd not supported.") + +def make_lazy_class(cls): + + def lazy_init(self, cb): + self._cb = cb + self._value = None + + cls.__init__ = lazy_init + + for basename in [ + "add", "sub", "mul", "truediv", "floordiv", "mod", "divmod", "pow", + "lshift", "rshift", "and", "or", "xor", "neg", "pos", "abs", "invert", + "eq", "ne", "lt", "le", "gt", "ge", "bool", "int", "index", + ]: + name = f"__{basename}__" + + def inner_wrapper(name): + use_operator = basename not in ("bool", "int") + + def wrapped(self, *args, **kwargs): + if self._cb is not None: + self._value = self._cb() + self._cb = None + if not use_operator: + return getattr(self._value, name)(*args, **kwargs) + else: + return getattr(operator, name)(self._value, *args, **kwargs) + return wrapped + + setattr(cls, name, inner_wrapper(name)) + + return cls + + +# Base TestCase for NT tests; used to define common helpers, etc. +class NestedTensorTestCase(TestCase): + def assertEqualIgnoringNestedInts(self, a, b): + # unbinding NJTs allows us to compare them as essentially equal without + # caring about exact nested int comparison + def _unbind_njts(x): + if isinstance(x, core.Tensor) and x.is_nested and x.layout == core.jagged: + return x.unbind() + else: + return x + + self.assertEqual(pytree.tree_map(_unbind_njts, a), pytree.tree_map(_unbind_njts, b)) + + def assertEqualNoncontigAware(self, a, b): + # assertEqual() doesn't take into account lengths, so hack around this + # by comparing unbound components and shapes + self.assertEqualIgnoringNestedInts(a, b) + + def _get_njt_shapes(x): + return ( + x.shape + if isinstance(x, core.Tensor) and x.is_nested + else None + ) + + a_shapes = pytree.tree_map(_get_njt_shapes, a) + b_shapes = pytree.tree_map(_get_njt_shapes, b) + self.assertEqual(a_shapes, b_shapes) + + @contextlib.contextmanager + def branch_nested_state(self): + """Context manager to branch and restore the nested tensor state.""" + nested_tensor_module = core.nested._internal.nested_tensor + original_tensor_symint_registry = nested_tensor_module._tensor_symint_registry.copy() + original_tensor_id_counter = nested_tensor_module._tensor_id_counter + try: + yield + finally: + nested_tensor_module._tensor_id_counter = original_tensor_id_counter + nested_tensor_module._tensor_symint_registry = original_tensor_symint_registry + + +@make_lazy_class +class LazyVal: + pass + + +def munge_exc(e, *, suppress_suffix=True, suppress_prefix=True, file=None, skip=0): + if file is None: + file = inspect.stack()[1 + skip].filename # skip one frame + + file = _as_posix_path(file) + s = _as_posix_path(str(e)) + + # Remove everything that looks like stack frames in NOT this file + def repl_frame(m): + if m.group(1) != file: + return "" + # Don't accept top-level, even for this script, these will wobble + # depending on how the testing script was invoked + if m.group(2) == "": + return "" + + return m.group(0) + + s = re.sub(r' File "([^"]+)", line \d+, in (.+)\n( .+\n( +[~^]+ *\n)?)+', repl_frame, s) + s = re.sub(r"line \d+", "line N", s) + s = re.sub(r".py:\d+", ".py:N", s) + s = re.sub(r'https:/([a-zA-Z0-9_.-]+)', r'https://\1', s) + s = re.sub(file, _as_posix_path(os.path.basename(file)), s) + s = re.sub(_as_posix_path(os.path.join(os.path.dirname(core.__file__), "")), "", s) + if suppress_suffix: + s = re.sub(r"\n*Set TORCH_LOGS.+", "", s, flags=re.DOTALL) + s = re.sub(r"\n*You can suppress this exception.+", "", s, flags=re.DOTALL) + s = re.sub(r"\n*Set TORCHDYNAMO_VERBOSE=1.+", "", s, flags=re.DOTALL) + if suppress_prefix: + s = re.sub(r"Cannot export model.+\n\n", "", s) + s = re.sub(r" +$", "", s, flags=re.MULTILINE) + return s + + +@contextmanager +def check_leaked_tensors(limit=1, matched_type=core.Tensor): + """Wrap around operations you want to ensure are not leaking tensor memory. + + This code intentionally ignores other reference cycles, which can be benign and which we have plenty + of in pytorch code. It focuses on any reference cycles that directly or indirectly result holding a Tensor alive, + since this is likely a more serious leak than typical python refcycles. + + limit specifies how many tensors to dump debug graphs for (default=1) + """ + def match_obj(obj): + return isinstance(obj, matched_type) + + try: + gc.collect() + gc.set_debug(gc.DEBUG_SAVEALL) + garbage_objs = [] # type: ignore[var-annotated] + + # run the user code, after cleaning any existing refcycles, and then check for new ones + # also allow usercode to check the garbage objs (e.g. for assertion) after exiting ctxmgr + yield garbage_objs + + gc.collect() + garbage_objs.extend(filter(match_obj, gc.garbage)) + num_garbage_objs = len(garbage_objs) + if num_garbage_objs > 0: + warnings.warn( + f"{num_garbage_objs} tensors were found in the garbage. Did you introduce a reference cycle?" + ) + try: + import objgraph # type: ignore[import-not-found,import-untyped] + warnings.warn( + f"Dumping first {limit} objgraphs of leaked {matched_type}s rendered to png" + ) + for g in garbage_objs[:limit]: + objgraph.show_backrefs([g], max_depth=10) + except ImportError: + warnings.warn("`pip install objgraph` to enable memory leak debugging") + + finally: + gc.set_debug(0) + + +def remove_cpp_extensions_build_root(): + """ + Removes the default root folder under which extensions are built. + """ + default_build_root = cpp_extension.get_default_build_root() + if os.path.exists(default_build_root): + if IS_WINDOWS: + # rmtree returns permission error: [WinError 5] Access is denied + # on Windows, this is a workaround + subprocess.run(["rm", "-rf", default_build_root], stdout=subprocess.PIPE) + else: + shutil.rmtree(default_build_root, ignore_errors=True) + + +def install_cpp_extension(extension_root): + # Wipe the build / install dirs if they exist + build_dir = os.path.join(extension_root, "build") + install_dir = os.path.join(extension_root, "install") + for d in (build_dir, install_dir): + if os.path.exists(d): + shutil.rmtree(d) + + # Build the extension + setup_py_path = os.path.join(extension_root, "setup.py") + cmd = [sys.executable, setup_py_path, "install", "--root", install_dir] + return_code = shell(cmd, cwd=extension_root, env=os.environ) + if return_code != 0: + raise RuntimeError(f"build failed for cpp extension at {extension_root}") + + mod_install_dir = None + # install directory is the one that is named site-packages + for root, directories, _ in os.walk(install_dir): + for directory in directories: + if "-packages" in directory: + mod_install_dir = os.path.join(root, directory) + + if mod_install_dir is None: + raise RuntimeError(f"installation failed for cpp extension at {extension_root}") + + if mod_install_dir not in sys.path: + sys.path.insert(0, mod_install_dir) + + +# Decorator to provide a helper to load inline extensions to a temp directory +def scoped_load_inline(func): + + @wraps(func) + def wrapper(*args, **kwargs): + def load_inline(*args, **kwargs): + if IS_WINDOWS: + # TODO(xmfan): even using TemporaryDirectoryName will result in permission error + return cpp_extension.load_inline(*args, **kwargs) + + assert "build_directory" not in kwargs + with TemporaryDirectoryName() as temp_dir_name: + if kwargs.get("verbose", False): + print(f'Using temporary extension directory {temp_dir_name}...', file=sys.stderr) + kwargs["build_directory"] = temp_dir_name + return cpp_extension.load_inline(*args, **kwargs) + + return func(*args, load_inline=load_inline, **kwargs) + return wrapper + +def recover_orig_fp32_precision(fn): + @contextlib.contextmanager + def recover(): + old_mkldnn_conv_p = core.backends.mkldnn.conv.fp32_precision # type: ignore[attr-defined] + old_mkldnn_rnn_p = core.backends.mkldnn.rnn.fp32_precision # type: ignore[attr-defined] + old_mkldnn_matmul_p = core.backends.mkldnn.matmul.fp32_precision # type: ignore[attr-defined] + old_cudnn_conv_p = core.backends.cudnn.conv.fp32_precision # type: ignore[attr-defined] + old_cudnn_rnn_p = core.backends.cudnn.rnn.fp32_precision # type: ignore[attr-defined] + old_cuda_matmul_p = core.backends.cuda.matmul.fp32_precision + try: + yield + finally: + core.backends.mkldnn.conv.fp32_precision = old_mkldnn_conv_p # type: ignore[attr-defined] + core.backends.mkldnn.rnn.fp32_precision = old_mkldnn_rnn_p # type: ignore[attr-defined] + core.backends.mkldnn.matmul.fp32_precision = old_mkldnn_matmul_p # type: ignore[attr-defined] + core.backends.cudnn.conv.fp32_precision = old_cudnn_conv_p # type: ignore[attr-defined] + core.backends.cudnn.rnn.fp32_precision = old_cudnn_rnn_p # type: ignore[attr-defined] + core.backends.cuda.matmul.fp32_precision = old_cuda_matmul_p + + return recover()(fn) + +def skipIfPythonVersionMismatch(predicate): + vi = sys.version_info + + def dec_fn(fn): + @wraps(fn) + def wrap_fn(self, *args, **kwargs): + if predicate(vi.major, vi.minor, vi.micro): + return fn(self, *args, **kwargs) + else: + raise unittest.SkipTest("Python version mismatch") + return wrap_fn + return dec_fn diff --git a/mindnlp/core/types.py b/mindnlp/core/types.py index 8b0a47672..3a659cd96 100644 --- a/mindnlp/core/types.py +++ b/mindnlp/core/types.py @@ -7,82 +7,15 @@ str as _str, ) import mindspore -from typing import Any, IO, TYPE_CHECKING, Union, Dict +from typing import Any, IO, TYPE_CHECKING, Union, Dict, Sequence from typing_extensions import Self, TypeAlias from mindnlp import core +from mindnlp.core import Tensor from ._dtype import dtype -from .configs import DEVICE_TARGET - -DEVICE_MAP = { - 'GPU': 'cuda', - 'Ascend': 'npu', - 'CPU': 'cpu' -} - -class device(): - def __init__(self, type=None, index=None): - if type is not None: - if isinstance(type, str): - if ':' in type: - if index is not None: - raise ValueError("`type` must not include an index because index was " - f"passed explicitly: {type}") - _target, _id = type.split(':') - _id = int(_id) - else: - _target = type - _id = None if _target == 'cpu' else 0 - elif isinstance(type, device): - if index is not None: - raise ValueError("core.device(): When input is core.device, `index` can not be set.") - _target = type.type - _id = type.index - elif isinstance(type, int): - _id = type - try: - device_target = mindspore.get_current_device().device_target - except: - device_target = mindspore.get_context('device_target') - _target = DEVICE_MAP[device_target] - else: - print(type) - raise TypeError("core.device(): `type` must be type of 'str' or 'core.device'.") - else: - raise ValueError("core.device(): `type` can not be None") - - self.type = _target - self.index = _id - if DEVICE_TARGET == 'Ascned' and self.type == 'cuda': - self.type = 'npu' - - def __repr__(self): - if self.index is None: - return f"device(type={self.type})" - return f"device(type={self.type}, index={self.index})" - - def __eq__(self, __value): - if not isinstance(__value, device): - return False - return hash(self) == hash(__value) - - def __hash__(self): - return hash(self.type) ^ hash(self.index) - - def __gt__(self, other): - if self.type == 'cpu': - return False - return True - - def __enter__(self): - # self.prev_idx = torch.cuda._exchange_device(self.idx) - core._bind.set_device_in_context(self) - - def __exit__(self, type: Any, value: Any, traceback: Any): - # self.idx = torch.cuda._maybe_exchange_device(self.prev_idx) - core._bind.set_device_in_context(None) - return False +from ._C import device as _device +_TensorOrTensors: TypeAlias = Union[Tensor, Sequence[Tensor]] # noqa: PYI047 # Meta-type for "numeric" things; matches our docs Number: TypeAlias = Union[int, float, bool] @@ -93,7 +26,7 @@ def __exit__(self, type: Any, value: Any, traceback: Any): # Storage protocol implemented by ${Type}StorageBase classes class Storage: _cdata: int - device: device + device: _device dtype: dtype _torch_load_uninitialized: bool @@ -145,6 +78,5 @@ def _new_with_file( ) -> "Storage": raise NotImplementedError -_device = device _dtype = dtype _size = tuple \ No newline at end of file diff --git a/mindnlp/core/utils/_backport_slots.py b/mindnlp/core/utils/_backport_slots.py new file mode 100644 index 000000000..7fdfd6af2 --- /dev/null +++ b/mindnlp/core/utils/_backport_slots.py @@ -0,0 +1,116 @@ +# This code is backported from python 3.10 dataclasses. Once 3.10 becomes the +# minimum supported we should use dataclass(slots=True) instead. + +from __future__ import annotations + +import dataclasses +import itertools +from typing import TYPE_CHECKING, TypeVar + + +if TYPE_CHECKING: + from collections.abc import Generator + + from _typeshed import DataclassInstance + + +__all__ = ["dataclass_slots"] + +_T = TypeVar("_T", bound="DataclassInstance") + + +def dataclass_slots(cls: type[_T]) -> type[DataclassInstance]: + assert dataclasses.is_dataclass(cls), "Can only be used on dataclasses." + + def _get_slots(cls: type[DataclassInstance]) -> Generator[str, None, None]: + slots = cls.__dict__.get("__slots__") + # `__dictoffset__` and `__weakrefoffset__` can tell us whether + # the base type has dict/weakref slots, in a way that works correctly + # for both Python classes and C extension types. Extension types + # don't use `__slots__` for slot creation + if slots is None: + slots = [] + if getattr(cls, "__weakrefoffset__", -1) != 0: + slots.append("__weakref__") + if getattr(cls, "__dictrefoffset__", -1) != 0: + slots.append("__dict__") + yield from slots + elif isinstance(slots, str): + yield slots + # Slots may be any iterable, but we cannot handle an iterator + # because it will already be (partially) consumed. + elif not hasattr(cls, "__next__"): + yield from slots + else: + raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") + + def _add_slots( + cls: type[DataclassInstance], is_frozen: bool, weakref_slot: bool + ) -> type[DataclassInstance]: + # Need to create a new class, since we can't set __slots__ + # after a class has been created. + + # Make sure __slots__ isn't already set. + if "__slots__" in cls.__dict__: + raise TypeError(f"{cls.__name__} already specifies __slots__") + + # Create a new dict for our new class. + cls_dict = dict(cls.__dict__) + field_names = tuple(f.name for f in dataclasses.fields(cls)) + # Make sure slots don't overlap with those in base classes. + inherited_slots = set( + itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) + ) + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + cls_dict["__slots__"] = tuple( + itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, + ("__weakref__",) if weakref_slot else (), + ), + ), + ) + + for field_name in field_names: + # Remove our attributes, if present. They'll still be + # available in _MARKER. + cls_dict.pop(field_name, None) + + # Remove __dict__ itself. + cls_dict.pop("__dict__", None) + + # Clear existing `__weakref__` descriptor, it belongs to a previous type: + cls_dict.pop("__weakref__", None) # gh-102069 + + # And finally create the class. + qualname = getattr(cls, "__qualname__", None) + cls = type(cls.__name__, cls.__bases__, cls_dict) + if qualname is not None: + cls.__qualname__ = qualname + + def _dataclass_getstate(self: _T) -> object: + fields = dataclasses.fields(self) + return [getattr(self, f.name) for f in fields] + + def _dataclass_setstate(self: _T, state: list[object]) -> None: + fields = dataclasses.fields(self) + for field, value in zip(fields, state): + # use setattr because dataclass may be frozen + object.__setattr__(self, field.name, value) + + if is_frozen: + # Need this for pickling frozen classes with slots. + if "__getstate__" not in cls_dict: + cls.__getstate__ = _dataclass_getstate # type: ignore[method-assign, assignment] + if "__setstate__" not in cls_dict: + cls.__setstate__ = _dataclass_setstate # type: ignore[attr-defined] + + return cls + + params = getattr(cls, dataclasses._PARAMS) # type: ignore[attr-defined] + weakref_slot = getattr(params, "weakref_slot", False) + return _add_slots(cls, params.frozen, weakref_slot) \ No newline at end of file diff --git a/mindnlp/core/utils/_config_module.py b/mindnlp/core/utils/_config_module.py new file mode 100644 index 000000000..2f31bf78f --- /dev/null +++ b/mindnlp/core/utils/_config_module.py @@ -0,0 +1,804 @@ +import contextlib +import copy +import hashlib +import importlib +import inspect +import io +import os +import pickle +import sys +import tokenize +import unittest +from dataclasses import dataclass +from types import FunctionType, ModuleType +from typing import ( + Any, + Callable, + Generic, + NoReturn, + Optional, + TYPE_CHECKING, + TypeVar, + Union, +) +from typing_extensions import deprecated +from unittest import mock + +# from torch._utils_internal import justknobs_check + + +# Types saved/loaded in configs +CONFIG_TYPES = (int, float, bool, type(None), str, list, set, tuple, dict) + + +# Duplicated, because mypy needs these types statically +T = TypeVar("T", bound=Union[int, float, bool, None, str, list, set, tuple, dict]) + + +_UNSET_SENTINEL = object() + + +@dataclass +class _Config(Generic[T]): + """Represents a config with richer behaviour than just a default value. + :: + i.e. + foo = Config(justknob="//foo:bar", default=False) + install_config_module(...) + + This configs must be installed with install_config_module to be used + + Precedence Order: + alias: If set, the directly use the value of the alias. + env_name_force: If set, this environment variable has precedence over + everything after this. + If multiple env variables are given, the precedence order is from + left to right. + user_override: If a user sets a value (i.e. foo.bar=True), that + has precedence over everything after this. + env_name_default: If set, this environment variable will override everything + after this. + If multiple env variables are given, the precedence order is from + left to right. + justknob: If this pytorch installation supports justknobs, that will + override defaults, but will not override the user_override precedence. + default: This value is the lowest precedence, and will be used if nothing is + set. + + Environment Variables: + These are interpreted to be either "0" or "1" to represent true and false. + + Arguments: + justknob: the name of the feature / JK. In OSS this is unused. + default: is the value to default this knob to in OSS. + alias: The alias config to read instead. + env_name_force: The environment variable, or list of, to read that is a FORCE + environment variable. I.e. it overrides everything except for alias. + env_name_default: The environment variable, or list of, to read that changes the + default behaviour. I.e. user overrides take preference. + """ + + default: Union[T, object] + justknob: Optional[str] = None + env_name_default: Optional[list[str]] = None + env_name_force: Optional[list[str]] = None + alias: Optional[str] = None + + def __init__( + self, + default: Union[T, object] = _UNSET_SENTINEL, + justknob: Optional[str] = None, + env_name_default: Optional[Union[str, list[str]]] = None, + env_name_force: Optional[Union[str, list[str]]] = None, + value_type: Optional[type] = None, + alias: Optional[str] = None, + ): + # python 3.9 does not support kw_only on the dataclass :(. + self.default = default + self.justknob = justknob + self.env_name_default = _Config.string_or_list_of_string_to_list( + env_name_default + ) + self.env_name_force = _Config.string_or_list_of_string_to_list(env_name_force) + self.value_type = value_type + self.alias = alias + if self.alias is not None: + assert ( + default is _UNSET_SENTINEL + and justknob is None + and env_name_default is None + and env_name_force is None + ), "if alias is set, none of {default, justknob and env var} can be set" + + @staticmethod + def string_or_list_of_string_to_list( + val: Optional[Union[str, list[str]]], + ) -> Optional[list[str]]: + if val is None: + return None + if isinstance(val, str): + return [val] + assert isinstance(val, list) + return val + + +# In runtime, we unbox the Config[T] to a T, but typechecker cannot see this, +# so in order to allow for this dynamic behavior to work correctly with +# typechecking we are going to lie to the typechecker that Config[T] returns +# a T. +if TYPE_CHECKING: + + def Config( + default: Union[T, object] = _UNSET_SENTINEL, + justknob: Optional[str] = None, + env_name_default: Optional[Union[str, list[str]]] = None, + env_name_force: Optional[Union[str, list[str]]] = None, + value_type: Optional[type] = None, + alias: Optional[str] = None, + ) -> T: ... + +else: + + def Config( + default: Union[T, object] = _UNSET_SENTINEL, + justknob: Optional[str] = None, + env_name_default: Optional[Union[str, list[str]]] = None, + env_name_force: Optional[Union[str, list[str]]] = None, + value_type: Optional[type] = None, + alias: Optional[str] = None, + ) -> _Config[T]: + return _Config( + default, justknob, env_name_default, env_name_force, value_type, alias + ) + + +def _read_env_variable(name: str) -> Optional[Union[bool, str]]: + value = os.environ.get(name) + if value == "1": + return True + if value == "0": + return False + return value + + +def install_config_module(module: ModuleType) -> None: + """ + Converts a module-level config into a `ConfigModule()`. + + See _config_typing.pyi for instructions on how to get the converted module to typecheck. + """ + + class ConfigModuleInstance(ConfigModule): + # __annotations__ is written to by Sphinx autodoc + _bypass_keys = set({"_is_dirty", "_hash_digest", "__annotations__"}) + + def visit( + source: Union[ModuleType, type], + dest: Union[ModuleType, SubConfigProxy], + prefix: str, + ) -> None: + """Walk the module structure and move everything to module._config""" + if sys.version_info[:2] < (3, 10): + type_hints = getattr(source, "__annotations__", {}) + else: + type_hints = inspect.get_annotations(source) + for key, value in list(source.__dict__.items()): + if ( + key.startswith("__") + or isinstance(value, (ModuleType, FunctionType)) + or (hasattr(value, "__module__") and value.__module__ == "typing") + # Handle from torch.utils._config_module import Config + or (isinstance(value, type) and issubclass(value, _Config)) + ): + continue + + name = f"{prefix}{key}" + annotated_type = type_hints.get(key, None) + if isinstance(value, CONFIG_TYPES): + config[name] = _ConfigEntry( + _Config(default=value, value_type=annotated_type) + ) + if dest is module: + delattr(module, key) + elif isinstance(value, _Config): + if annotated_type is not None and value.value_type is None: + value.value_type = annotated_type + + config[name] = _ConfigEntry(value) + + if dest is module: + delattr(module, key) + elif isinstance(value, type): + assert value.__module__ == module.__name__ + # a subconfig with `class Blah:` syntax + proxy = SubConfigProxy(module, f"{name}.") + visit(value, proxy, f"{name}.") + if dest is module: + setattr(dest, key, proxy) + else: + dest.__dict__[key] = proxy + else: + raise AssertionError(f"Unhandled config {key}={value} ({type(value)})") + + config: dict[str, _ConfigEntry] = {} + + compile_ignored_keys = get_assignments_with_compile_ignored_comments(module) + + visit(module, module, "") + module._config = config # type: ignore[attr-defined] + module._compile_ignored_keys = compile_ignored_keys # type: ignore[attr-defined] + module.__class__ = ConfigModuleInstance + module._is_dirty = True # type: ignore[attr-defined] + module._hash_digest = None # type: ignore[attr-defined] + + +COMPILE_IGNORED_MARKER = "@compile_ignored" + + +# Gets all the keys (i.e. assignments) with a @compile_ignored comment +def get_assignments_with_compile_ignored_comments(module: ModuleType) -> set[str]: + source_code = inspect.getsource(module) + assignments = set() + + # Tokenize the source code to retrieve comments + tokens = tokenize.tokenize(io.BytesIO(source_code.encode("utf-8")).readline) + current_comment = "", -1 + prev_name = "" + + for token in tokens: + if token.type == tokenize.COMMENT: + prev_name = "" + maybe_current = token.string.strip() + if COMPILE_IGNORED_MARKER in maybe_current: + assert current_comment == ( + "", + -1, + ), f"unconsumed {COMPILE_IGNORED_MARKER}" + current_comment = maybe_current, token.start[0] + elif token.type == tokenize.NAME: + # Only accept the first name token, to handle if you have + # something like foo: Bar = ... + if not prev_name: + prev_name = token.string + elif token.type == tokenize.OP and token.string == "=": + # Check if the current assignment follows a comment + # with COMPILE_IGNORED_MARKER + if ( + COMPILE_IGNORED_MARKER in current_comment[0] + and current_comment[1] == token.start[0] - 1 + ): + assignments.add(prev_name) + current_comment = "", -1 # reset + prev_name = "" + assert current_comment == ("", -1), f"unconsumed {COMPILE_IGNORED_MARKER}" + return assignments + + +@dataclass +class _ConfigEntry: + # The default value specified in the configuration + default: Any + # The type of the configuration value + value_type: type + # The value specified by the user when they overrode the configuration + # _UNSET_SENTINEL indicates the value is not set. + user_override: Any = _UNSET_SENTINEL + # The justknob to check for this config + justknob: Optional[str] = None + # environment variables are read at install time + env_value_force: Any = _UNSET_SENTINEL + env_value_default: Any = _UNSET_SENTINEL + # Used to work arounds bad assumptions in unittest.mock.patch + # The code to blame is + # https://github.com/python/cpython/blob/94a7a4e22fb8f567090514785c69e65298acca42/Lib/unittest/mock.py#L1637 + # Essentially, mock.patch requires, that if __dict__ isn't accessible + # (which it isn't), that after delattr is called on the object, the + # object must throw when hasattr is called. Otherwise, it doesn't call + # setattr again. + # Technically we'll have an intermediate state of hiding the config while + # mock.patch is unpatching itself, but it calls setattr after the delete + # call so the final state is correct. It's just very unintuitive. + # upstream bug - python/cpython#126886 + hide: bool = False + alias: Optional[str] = None + + def __init__(self, config: _Config): + self.default = config.default + self.value_type = ( + config.value_type if config.value_type is not None else type(self.default) + ) + self.justknob = config.justknob + self.alias = config.alias + if config.env_name_default is not None: + for val in config.env_name_default: + if (env_value := _read_env_variable(val)) is not None: + self.env_value_default = env_value + break + if config.env_name_force is not None: + for val in config.env_name_force: + if (env_value := _read_env_variable(val)) is not None: + self.env_value_force = env_value + break + + # Ensure justknobs and envvars are allowlisted types + if self.justknob is not None and self.default is not None: + assert isinstance(self.default, bool), ( + f"justknobs only support booleans, {self.default} is not a boolean" + ) + if self.value_type is not None and ( + config.env_name_default is not None or config.env_name_force is not None + ): + assert self.value_type in ( + bool, + str, + Optional[bool], + Optional[str], + ), ( + f"envvar configs only support (optional) booleans or strings, {self.value_type} is neither" + ) + + +class ConfigModule(ModuleType): + # NOTE: This should be kept in sync with _config_typing.pyi. + + # The actual configuration settings. E.g., torch._dynamo.config.debug + # would live as "debug" in the key, and torch._inductor.config.triton.cudagraphs + # maps as "triton.cudagraphs". See discussion on the class for meaning of various sub items + _config: dict[str, _ConfigEntry] + _bypass_keys: set[str] + _compile_ignored_keys: set[str] + _is_dirty: bool + _hash_digest: Optional[bytes] + + def __init__(self) -> None: + raise NotImplementedError( + f"use {__name__}.install_config_module(sys.modules[__name__])" + ) + + def __setattr__(self, name: str, value: object) -> None: + if name in self._bypass_keys: + super().__setattr__(name, value) + elif name not in self._config: + raise AttributeError(f"{self.__name__}.{name} does not exist") + elif self._config[name].alias is not None: + self._set_alias_val(self._config[name], value) + else: + self._config[name].user_override = value + self._is_dirty = True + self._config[name].hide = False + + def __getattr__(self, name: str) -> Any: + try: + config = self._config[name] + + if config.hide: + raise AttributeError(f"{self.__name__}.{name} does not exist") + + alias_val = self._get_alias_val(config) + if alias_val is not _UNSET_SENTINEL: + return alias_val + + if config.env_value_force is not _UNSET_SENTINEL: + return config.env_value_force + + if config.user_override is not _UNSET_SENTINEL: + return config.user_override + + if config.env_value_default is not _UNSET_SENTINEL: + return config.env_value_default + + if config.justknob is not None: + # JK only supports bools and ints + return justknobs_check(name=config.justknob, default=config.default) + + # Note that reference types can still be modified, so we + # copy them to user_overrides in case the user overrides + # them + if isinstance(config.default, (list, set, dict)): + config.user_override = copy.deepcopy(config.default) + return config.user_override + return config.default + + except KeyError as e: + # make hasattr() work properly + raise AttributeError(f"{self.__name__}.{name} does not exist") from e + + def __delattr__(self, name: str) -> None: + self._is_dirty = True + # must support delete because unittest.mock.patch deletes + # then recreate things + self._config[name].user_override = _UNSET_SENTINEL + self._config[name].hide = True + + def _get_alias_module_and_name( + self, entry: _ConfigEntry + ) -> Optional[tuple[ModuleType, str]]: + alias = entry.alias + if alias is None: + return None + module_name, constant_name = alias.rsplit(".", 1) + try: + module = importlib.import_module(module_name) + except ImportError as e: + raise AttributeError("config alias {alias} does not exist") from e + return module, constant_name + + def _get_alias_val(self, entry: _ConfigEntry) -> Any: + data = self._get_alias_module_and_name(entry) + if data is None: + return _UNSET_SENTINEL + module, constant_name = data + constant_value = getattr(module, constant_name) + return constant_value + + def _set_alias_val(self, entry: _ConfigEntry, val: Any) -> None: + data = self._get_alias_module_and_name(entry) + assert data is not None + module, constant_name = data + setattr(module, constant_name, val) + + def _is_default(self, name: str) -> bool: + """ + Returns true if the config is at its default value. + configs overridden by the env are not considered default. + """ + config_val = self._config[name] + # The config is not overridden by the user, and the env_value_default + # is different from the default value (meaning user has set the env to + # change the default value). + not_set_env_default = ( + config_val.env_value_default is _UNSET_SENTINEL + or config_val.env_value_default == config_val.default + ) + not_set_env_force = ( + config_val.env_value_force is _UNSET_SENTINEL + or config_val.env_value_force == config_val.default + ) + + unset = config_val.user_override is _UNSET_SENTINEL + # Handle reference types specially to avoid spammy warnings + if isinstance(config_val.default, (list, set, dict)): + unset = unset or config_val.user_override == config_val.default + return unset and not_set_env_default and not_set_env_force + + def _get_dict( + self, + ignored_keys: Optional[list[str]] = None, + ignored_prefixes: Optional[list[str]] = None, + skip_default: bool = False, + ) -> dict[str, Any]: + """Export a dictionary of current configuration keys and values. + + This function is design to provide a single point which handles + accessing config options and exporting them into a dictionary. + This is used by a number of different user facing export methods + which all have slightly different semantics re: how and what to + skip. + If a config is aliased, it skips this config. + + Arguments: + ignored_keys are keys that should not be exported. + ignored_prefixes are prefixes that if a key matches should + not be exported + skip_default does two things. One if a key has not been modified + it skips it. + """ + config: dict[str, Any] = {} + for key in self._config: + if ignored_keys and key in ignored_keys: + continue + if ignored_prefixes: + if any(key.startswith(prefix) for prefix in ignored_prefixes): + continue + if skip_default and self._is_default(key): + continue + if self._config[key].alias is not None: + continue + config[key] = copy.deepcopy(getattr(self, key)) + + return config + + def get_type(self, config_name: str) -> type: + return self._config[config_name].value_type + + def save_config(self) -> bytes: + """Convert config to a pickled blob""" + ignored_keys = getattr(self, "_save_config_ignore", []) + return pickle.dumps( + self._get_dict(ignored_keys=ignored_keys), + protocol=2, + ) + + def save_config_portable( + self, *, ignore_private_configs: bool = True + ) -> dict[str, Any]: + """Convert config to portable format""" + prefixes = [] + if ignore_private_configs: + prefixes.append("_") + prefixes.extend(getattr(self, "_cache_config_ignore_prefix", [])) + return self._get_dict(ignored_prefixes=prefixes) + + def codegen_config(self) -> str: + """Convert config to Python statements that replicate current config. + This does NOT include config settings that are at default values. + """ + + # additional imports required + imports = set() + + def get_module_name(func: Callable, add_dot: bool) -> str: + module_name = func.__module__ + if module_name == "builtins": + module_name = "" + if add_dot and module_name != "": + module_name += "." + return module_name + + def add_import(func: Callable) -> None: + module_name = get_module_name(func, False) + if module_name: + imports.add(module_name) + + def list_of_callables_to_string(v: Union[list, set]) -> list[str]: + return [f"{get_module_name(item, True)}{item.__name__}" for item in v] + + def importable_callable(v: Any) -> bool: + # functools.partial has no attributes below but is a callable + return callable(v) and hasattr(v, "__module__") and hasattr(v, "__name__") + + def get_config_line(mod, k, v) -> str: # type: ignore[no-untyped-def] + """ + Return a string version of the config line. + Handle v when v is a callable, or a list/dict of callables. Add import statements for callables if necessary. + We assume that the value of a single config won't be a mix of callables and non-callables. + + Example output: + import logging + import _warnings + torch._dynamo.config.reorderable_logging_functions = { _warnings.warn, logging.warn, print } + """ + if importable_callable(v): + add_import(v) + return f"{mod}.{k} = {get_module_name(v, True)}{v.__name__}" + elif isinstance(v, (list, set)) and all( + importable_callable(item) for item in v + ): + for item in v: + add_import(item) + v_list = list_of_callables_to_string(v) + if isinstance(v, list): + return f"{mod}.{k} = {v_list}" + else: + return f"{mod}.{k} = {{ {', '.join(v_list)} }}" + else: + return f"{mod}.{k} = {v!r}" + + lines = [] + mod = self.__name__ + for k, v in self._get_dict( + ignored_keys=getattr(self, "_save_config_ignore", []), skip_default=True + ).items(): + lines.append(get_config_line(mod, k, v)) + for import_name in imports: + lines.insert(0, f"import {import_name}") + return "\n".join(lines) + + def get_hash(self) -> bytes: + """Hashes the configs that are not compile_ignored""" + if self._is_dirty or self._hash_digest is None: + dict_to_hash = self._get_dict(ignored_keys=list(self._compile_ignored_keys)) + string_to_hash = repr(sorted(dict_to_hash.items())) + self._hash_digest = hashlib.md5( + string_to_hash.encode("utf-8"), usedforsecurity=False + ).digest() + self._is_dirty = False + return self._hash_digest + + @deprecated( + "`config.to_dict()` has been deprecated. It no longer changes the underlying config." + " use `config.get_config_copy()` instead if you just want a copy of the config, or " + "config.load_config if you need mutable access", + category=FutureWarning, + ) + def to_dict(self) -> dict[str, Any]: + return self.get_config_copy() + + @deprecated( + "`config.shallow_copy_dict()` has been deprecated. It no longer changes the underlying config." + " use `config.get_config_copy()` instead if you just want a copy of the config, or " + "config.load_config if you need mutable access", + category=FutureWarning, + ) + def shallow_copy_dict(self) -> dict[str, Any]: + return self.get_config_copy() + + def load_config(self, maybe_pickled_config: Union[bytes, dict[str, Any]]) -> None: + """Restore from a prior call to save_config() or shallow_copy_dict()""" + if not isinstance(maybe_pickled_config, dict): + config = pickle.loads(maybe_pickled_config) + else: + config = maybe_pickled_config + for k, v in config.items(): + if k in self._config: + setattr(self, k, v) + else: + from torch._dynamo.utils import warn_once + + warn_once(f"key {k} with value {v} is not understood by this config") + + def get_config_copy(self) -> dict[str, Any]: + return self._get_dict() + + def patch( + self, + arg1: Optional[Union[str, dict[str, Any]]] = None, + arg2: Any = None, + **kwargs: dict[str, Any], + ) -> "ContextDecorator": + """ + Decorator and/or context manager to make temporary changes to a config. + + As a decorator: + + @config.patch("name", val) + @config.patch(name1=val1, name2=val2) + @config.patch({"name1": val1, "name2", val2}) + def foo(...): + ... + + As a context manager: + + with config.patch("name", val): + ... + """ + changes: dict[str, Any] + if arg1 is not None: + if arg2 is not None: + assert isinstance(arg1, str) + # patch("key", True) syntax + changes = {arg1: arg2} + else: + assert isinstance(arg1, dict) + # patch({"key": True}) syntax + changes = arg1 + assert not kwargs + else: + # patch(key=True) syntax + changes = kwargs + assert arg2 is None + assert isinstance(changes, dict), f"expected `dict` got {type(changes)}" + prior: dict[str, Any] = {} + config = self + + class ConfigPatch(ContextDecorator): + def __init__(self) -> None: + self.changes = changes + + def __enter__(self) -> None: + assert not prior + for key in self.changes.keys(): + # KeyError on invalid entry + prior[key] = config.__getattr__(key) + for k, v in self.changes.items(): + config.__setattr__(k, v) + + def __exit__(self, exc_type, exc_val, exc_tb): # type: ignore[no-untyped-def] + for k, v in prior.items(): + config.__setattr__(k, v) + prior.clear() + + return ConfigPatch() + + def _make_closure_patcher(self, **changes: dict[str, Any]) -> Any: + """ + A lower-overhead version of patch() for things on the critical path. + + Usage: + + # do this off the critical path + change_fn = config.make_closure_patcher(foo=True) + + ... + + revert = change_fn() + try: + ... + finally: + revert() + + """ + config = self._config + + def change() -> Callable[[], None]: + prior = {k: config[k].user_override for k in changes} + for k, v in changes.items(): + self._config[k].user_override = v + + def revert() -> None: + for k, v in prior.items(): + self._config[k].user_override = v + + return revert + + return change + + +class ContextDecorator(contextlib.ContextDecorator): + """ + Same as contextlib.ContextDecorator, but with support for + `unittest.TestCase` + """ + + def __enter__(self) -> None: + raise NotImplementedError("NYI") + + def __exit__(self, exc_type, exc_val, exc_tb) -> NoReturn: # type: ignore[no-untyped-def] + raise NotImplementedError("NYI") + + def __call__(self, func: Callable[[Any], Any]) -> Any: + if isinstance(func, type) and issubclass(func, unittest.TestCase): + + class _TestCase(func): # type: ignore[valid-type, misc] + @classmethod + def setUpClass(cls) -> None: + self.__enter__() + try: + super().setUpClass() + except Exception: + self.__exit__(None, None, None) + raise + + @classmethod + def tearDownClass(cls) -> None: + try: + super().tearDownClass() + finally: + self.__exit__(None, None, None) + + _TestCase.__name__ = func.__name__ + _TestCase.__qualname__ = func.__qualname__ + _TestCase.__module__ = func.__module__ + + return _TestCase + + return super().__call__(func) + + +class SubConfigProxy: + """ + Shim to redirect to main config. + `config.triton.cudagraphs` maps to _config["triton.cudagraphs"] + """ + + def __init__(self, config: object, prefix: str): + # `super().__setattr__` to bypass custom `__setattr__` + super().__setattr__("_config", config) + super().__setattr__("_prefix", prefix) + + def __setattr__(self, name: str, value: object) -> None: + return self._config.__setattr__(self._prefix + name, value) + + def __getattr__(self, name: str) -> Any: + return self._config.__getattr__(self._prefix + name) + + def __delattr__(self, name: str) -> None: + return self._config.__delattr__(self._prefix + name) + + +def patch_object(obj: object, name: str, value: object) -> object: + """ + Workaround `mock.patch.object` issue with ConfigModule + """ + if isinstance(obj, ConfigModule): + return obj.patch(name, value) + return mock.patch.object(obj, name, value) + + +def get_tristate_env(name: str, default: Any = None) -> Optional[bool]: + value = os.environ.get(name) + if value == "1": + return True + if value == "0": + return False + return default \ No newline at end of file diff --git a/mindnlp/core/utils/_import_utils.py b/mindnlp/core/utils/_import_utils.py new file mode 100644 index 000000000..89f2a67df --- /dev/null +++ b/mindnlp/core/utils/_import_utils.py @@ -0,0 +1,38 @@ +import functools +import importlib.util +from types import ModuleType +from typing import Optional + + +def _check_module_exists(name: str) -> bool: + r"""Returns if a top-level module with :attr:`name` exists *without** + importing it. This is generally safer than try-catch block around a + `import X`. It avoids third party libraries breaking assumptions of some of + our tests, e.g., setting multiprocessing start method when imported + (see librosa/#747, torchvision/#544). + """ + try: + spec = importlib.util.find_spec(name) + return spec is not None + except ImportError: + return False + + +@functools.lru_cache +def dill_available() -> bool: + return _check_module_exists("dill") + + +@functools.lru_cache +def import_dill() -> Optional[ModuleType]: + if not dill_available(): + return None + + import dill + + # XXX: By default, dill writes the Pickler dispatch table to inject its + # own logic there. This globally affects the behavior of the standard library + # pickler for any user who transitively depends on this module! + # Undo this extension to avoid altering the behavior of the pickler globally. + dill.extend(use_dill=False) + return dill \ No newline at end of file diff --git a/mindnlp/core/utils/_traceback.py b/mindnlp/core/utils/_traceback.py new file mode 100644 index 000000000..f6ec505d5 --- /dev/null +++ b/mindnlp/core/utils/_traceback.py @@ -0,0 +1,260 @@ +# mypy: allow-untyped-defs +import contextlib +import inspect +import os.path +import tempfile +import traceback +from types import TracebackType +from typing import Optional + + +# This file contains utilities for ensuring dynamically compile()'d +# code fragments display their line numbers in backtraces. +# +# The constraints: +# +# - We don't have control over the user exception printer (in particular, +# we cannot assume the linecache trick will work, c.f. +# https://stackoverflow.com/q/50515651/23845 ) +# +# - We don't want to create temporary files every time we compile() +# some code; file creation should happen lazily only at exception +# time. Arguably, you *should* be willing to write out your +# generated Python code to file system, but in some situations +# (esp. library code) it would violate user expectation to write +# to the file system, so we try to avoid it. In particular, we'd +# like to keep the files around, so users can open up the files +# mentioned in the trace; if the file is invisible, we want to +# avoid clogging up the filesystem. +# +# If this is not a constraint for you, there is a substantially simpler +# way to implement the functionality in this PR: instead of using +# eval/exec directly, just always write a Python file to filesystem +# and compile that. +# +# - You have control over a context where the compiled code will get +# executed, so that we can interpose while the stack is unwinding +# (otherwise, we have no way to interpose on the exception printing +# process.) +# +# There are two things you have to do to make use of the utilities here: +# +# - When you compile your source code, you must save its string source +# in its f_globals under the magic name "__compile_source__" +# +# - Before running the compiled code, enter the +# report_compile_source_on_error() context manager. + + +@contextlib.contextmanager +def report_compile_source_on_error(): + try: + yield + except Exception as exc: + tb = exc.__traceback__ + + # Walk the traceback, looking for frames that have + # source attached + stack = [] + while tb is not None: + filename = tb.tb_frame.f_code.co_filename + source = tb.tb_frame.f_globals.get("__compile_source__") + + if filename == "" and source is not None: + # What black magic are we doing here? Intuitively, what + # we would like to do is overwrite the co_filename on any + # frames that were generated from exec/eval so that they + # point to a temporary file that has the actual line + # information, so Python's default error printer can print + # useful line information on it. + # + # Writing out the temporary file is easy. But overwriting + # co_filename is not! You can't modify the code object + # associated with a frame. You can, however, reconstruct + # a traceback with entirely new frames from scratch, so that's + # what we do. But there's another problem, which is how to + # make the frame? + # + # The black magic is we make a frankenstein frame and code + # object which resembles the original frame/code enough so + # that it will print properly under traceback and the default + # error printer, but IT IS NOT THE ORIGINAL FRAME (you + # couldn't, e.g., execute its code with different variables + # and expect it to work.) + + # Don't delete the temporary file so the user can inspect it + # TODO: This creates a temporary file for every frame, but we + # technically only need one per distinct __compile_source__ + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".py" + ) as f: + f.write(source) + # Create a frame. Python doesn't let you construct + # FrameType directly, so just make one with compile + frame = tb.tb_frame + code = compile("__inspect_currentframe()", f.name, "eval") + code = code.replace(co_name=frame.f_code.co_name) + # Python 3.11 only + if hasattr(frame.f_code, "co_linetable"): + # We can't copy ALL of the metadata over, because you + # can cause Python to segfault this way. What exactly + # do we need? We need enough information for + # traceback to be able to print the exception + # correctly. Code reading Lib/traceback.py reveals + # that traceback calls code.co_positions() in order to + # get the augmented line/col numbers. Objects/codeobject.c, + # specifically _PyCode_InitAddressRange, reveals that + # this iterator is initialized from co_linetable and + # co_firstfileno. So copy these we must! + code = code.replace( # type: ignore[call-arg] + co_linetable=frame.f_code.co_linetable, # type: ignore[attr-defined] + co_firstlineno=frame.f_code.co_firstlineno, # type: ignore[attr-defined] + ) + fake_frame = eval( + code, + frame.f_globals, + {**frame.f_locals, "__inspect_currentframe": inspect.currentframe}, + ) + fake_tb = TracebackType(None, fake_frame, tb.tb_lasti, tb.tb_lineno) + stack.append(fake_tb) + else: + stack.append(tb) + + tb = tb.tb_next + + # Reconstruct the linked list + tb_next = None + for tb in reversed(stack): + tb.tb_next = tb_next + tb_next = tb + + raise exc.with_traceback(tb_next) # noqa: B904 + + +def shorten_filename(fn, *, base=None): + """Shorten a source filepath, with the assumption that torch/ subdirectories don't need to be shown to user.""" + if base is None: + base = os.path.dirname(os.path.dirname(__file__)) + # Truncate torch/foo.py to foo.py + try: + prefix = os.path.commonpath([fn, base]) + except ValueError: + return fn + else: + return fn[len(prefix) + 1 :] + + +def format_frame(frame, *, base=None, line=False): + """ + Format a FrameSummary in a short way, without printing full absolute path or code. + + The idea is the result fits on a single line. + """ + extra_line = "" + if line: + extra_line = f"{frame.line} # " + return f"{extra_line}{shorten_filename(frame.filename, base=base)}:{frame.lineno} in {frame.name}" + + +def format_traceback_short(tb): + """Format a TracebackType in a short way, printing only the inner-most frame.""" + return format_frame(traceback.extract_tb(tb)[-1]) + + +class CapturedTraceback: + __slots__ = ["tb", "skip"] + + def __init__(self, tb, skip=0): + self.tb = tb + self.skip = skip + + def cleanup(self): + self.tb = None + + def summary(self): + import torch._C._profiler + + if self.tb is None: + # TODO: Maybe indicate that the traceback was elided? + return traceback.StackSummary() + + return _extract_symbolized_tb( + torch._C._profiler.symbolize_tracebacks([self.tb])[0], self.skip + ) + + def __getstate__(self): + return ( + None, + { + "tb": None, # TB is not pickleable + "skip": self.skip, + }, + ) + + @staticmethod + def extract(*, script=False, cpp=False, skip=0): + """ + Like traceback.extract_stack(), but faster (approximately 20x faster); it + is fast enough that you can unconditionally log stacks this way as part of + normal execution. It returns a torch._C._profiler.CapturedTraceback + object that must be formatted specially with format_captured_tb. + + By default, this only reports Python backtraces (like extract_stack). You + can set the script/cpp kwargs to also turn on TorchScript/C++ trace + reporting. + """ + import torch._C._profiler + + if script or cpp: + assert skip == 0, "skip with script/cpp NYI" + + return CapturedTraceback( + torch._C._profiler.gather_traceback(python=True, script=script, cpp=cpp), + # Elide extract() frame if we don't have script/cpp frames. If + # we do have those frames, it doesn't work so force zero. + 0 if script or cpp else skip + 1, + ) + + def format(self): + """ + Formats a single torch._C._profiler.CapturedTraceback into a list of + strings equivalent to the output of traceback.format_list. Note that if + pass it CapturedTraceback with C++ traces, it is better not to use this + function and use the batch formatting API format_captured_tbs to amortize + the cost of symbolization + """ + return traceback.format_list(self.summary()) + + @staticmethod + def format_all(tbs): + """ + Bulk version of CapturedTraceback.format. Returns a list of list of strings. + """ + import torch._C._profiler + + # Directly populate tracebacks that already have cached summaries + rs: list[Optional[list[str]]] = [] + delayed_idxs = [] + for i, tb in enumerate(tbs): + if tb.tb is None: + rs.append([]) + else: + rs.append(None) + delayed_idxs.append(i) + + torch._C._profiler.symbolize_tracebacks([tbs[i].tb for i in delayed_idxs]) + for i in delayed_idxs: + rs[i] = traceback.format_list(tbs[i].summary()) + + return rs + + +def _extract_symbolized_tb(tb, skip): + """ + Given a symbolized traceback from symbolize_tracebacks, return a StackSummary object of + pre-processed stack trace entries. + """ + stack = traceback.StackSummary() + for f in reversed(tb[skip:]): + stack.append(traceback.FrameSummary(f["filename"], f["line"], f["name"])) + return stack \ No newline at end of file diff --git a/mindnlp/core/utils/weak.py b/mindnlp/core/utils/weak.py new file mode 100644 index 000000000..03a9307d8 --- /dev/null +++ b/mindnlp/core/utils/weak.py @@ -0,0 +1,363 @@ +# mypy: allow-untyped-defs +from __future__ import annotations + +import collections.abc as _collections_abc +import weakref +from collections.abc import Mapping, MutableMapping +from weakref import ref + +from torch import Tensor + + +WeakRef = ref + + +__all__ = [ + "TensorWeakRef", + "WeakIdRef", + "WeakIdKeyDictionary", + "WeakTensorKeyDictionary", +] + + +# TODO: make weakref properly thread safe following +# https://github.com/python/cpython/pull/125325 +class _IterationGuard: + # This context manager registers itself in the current iterators of the + # weak container, such as to delay all removals until the context manager + # exits. + # This technique should be relatively thread-safe (since sets are). + + def __init__(self, weakcontainer): + # Don't create cycles + self.weakcontainer = ref(weakcontainer) + + def __enter__(self): + w = self.weakcontainer() + if w is not None: + w._iterating.add(self) + return self + + def __exit__(self, e, t, b): + w = self.weakcontainer() + if w is not None: + s = w._iterating + s.remove(self) + if not s: + w._commit_removals() + + +# This file defines a variant of WeakKeyDictionary that overrides the hashing +# behavior of the key to use object identity, rather than the builtin +# __eq__/__hash__ functions. This is useful for Tensor weak keys, as their +# __eq__ implementation return a Tensor (elementwise equality), which means +# you can't use them directly with the WeakKeyDictionary in standard library. +# +# Our implementation strategy is to create a wrapper weak key object, which we +# use as a key in a stock Python dictionary. This is similar to how weakref +# implements WeakKeyDictionary, but instead of using weakref.ref as the +# wrapper, we use a custom wrapper that has different __eq__ and __hash__ +# behavior. Note that we subsequently store this weak key directly in an +# ORDINARY dictionary, since the newly constructed WeakIdKey's only use would +# be a dictionary so it would have no strong references. Ensuring that +# only live WeakIdKeys are in the map is handled by putting finalizers on the +# original key object. + + +# It is simpler to implement this with composition, but if we want to +# directly reuse the callback mechanism on weakref, we need the weakref +# and the key to be exactly the same object. Reusing the callback mechanism +# minimizes the divergence between our implementation and Lib/weakref.py +# +# NB: Prefer using this when working with weakrefs of Tensors; e.g., do +# WeakIdRef(tensor) rather than weakref.ref(tensor); it handles a number of +# easy to get wrong cases transparently for you. +class WeakIdRef(weakref.ref): + __slots__ = ["_id"] + + def __init__(self, key, callback=None): + # Unlike stock weakref, which preserves hash semantics of the + # original object but lazily defers hash calls until the first + # time the user attempts to hash the weakref, we can eagerly + # cache the id of the key as we know this is definitely the hash + # method + self._id = id(key) + super().__init__(key, callback) # type: ignore[call-arg] + + def __call__(self): + r = super().__call__() + # Special logic for Tensor PyObject resurrection + if hasattr(r, "_fix_weakref"): + r._fix_weakref() # type: ignore[union-attr] + return r + + def __hash__(self): + return self._id + + def __eq__(self, other): + # An attractive but wrong alternate implementation is to only test if + # the stored _ids match. This can lead to an ABA problem if you have: + # + # a1 = A() + # w1 = WeakIdRef(a1) + # del a1 + # a2 = A() # suppose it gets the same ID as a1 + # w2 = WeakIdRef(a2) + # print(w1 == w2) + # + # This should be False, as a1 and a2 are unrelated (and a1 is + # dead anyway) + a = self() + b = other() + if a is not None and b is not None: + return a is b + return self is other + + +# This is the same as WeakIdRef but equality is checked using hash() rather than id. +# This will be equivalent to the one above except for classes where hash is not their id. +class _WeakHashRef(weakref.ref): + __slots__ = ["_id"] + + def __init__(self, key, callback=None): + # Unlike stock weakref, which preserves hash semantics of the + # original object but lazily defers hash calls until the first + # time the user attempts to hash the weakref, we can eagerly + # cache the id of the key as we know this is definitely the hash + # method + self._id = hash(key) + super().__init__(key, callback) # type: ignore[call-arg] + + def __call__(self): + r = super().__call__() + # Special logic for Tensor PyObject resurrection + if hasattr(r, "_fix_weakref"): + r._fix_weakref() # type: ignore[union-attr] + return r + + def __hash__(self): + return self._id + + def __eq__(self, other): + # Use hash equality to determine ref equality. + # ScriptObject implements __hash__ to return the wrapped IValue's id, so + # this is equivalent to doing an identity comparison. + a = self() + b = other() + if a is not None and b is not None: + return hash(a) == hash(b) + return self is other + + +# This is directly adapted from cpython/Lib/weakref.py +class WeakIdKeyDictionary(MutableMapping): + def __init__(self, dict=None, ref_type=WeakIdRef): # CHANGED + self.data = {} + + self.ref_type = ref_type # CHANGED + + def remove(k, selfref=ref(self)): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(k) + else: + try: + del self.data[k] + except KeyError: + pass + + self._remove = remove + # A list of dead weakrefs (keys to be removed) + self._pending_removals = [] + self._iterating = set() + self._dirty_len = False + if dict is not None: + self.update(dict) + + def _commit_removals(self): + # NOTE: We don't need to call this method before mutating the dict, + # because a dead weakref never compares equal to a live weakref, + # even if they happened to refer to equal objects. + # However, it means keys may already have been removed. + pop = self._pending_removals.pop + d = self.data + while True: + try: + key = pop() + except IndexError: + return + + try: + del d[key] + except KeyError: + pass + + def _scrub_removals(self): + d = self.data + self._pending_removals = [k for k in self._pending_removals if k in d] + self._dirty_len = False + + def __delitem__(self, key): + self._dirty_len = True + del self.data[self.ref_type(key)] # CHANGED + + def __getitem__(self, key): + return self.data[self.ref_type(key)] # CHANGED + + def __len__(self): + if self._dirty_len and self._pending_removals: + # self._pending_removals may still contain keys which were + # explicitly removed, we have to scrub them (see issue #21173). + self._scrub_removals() + return len(self.data) - len(self._pending_removals) + + def __repr__(self): + return f"<{self.__class__.__name__} at {id(self):#x}>" + + def __setitem__(self, key, value): + self.data[self.ref_type(key, self._remove)] = value # CHANGED + + def copy(self): + new = WeakIdKeyDictionary() + with _IterationGuard(self): + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = value + return new + + __copy__ = copy + + def __deepcopy__(self, memo): + from copy import deepcopy + + new = self.__class__() + with _IterationGuard(self): + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) + return new + + def get(self, key, default=None): + return self.data.get(self.ref_type(key), default) # CHANGED + + def __contains__(self, key): + try: + wr = self.ref_type(key) # CHANGED + except TypeError: + return False + return wr in self.data + + def items(self): + with _IterationGuard(self): + for wr, value in self.data.items(): + key = wr() + if key is not None: + yield key, value + + def keys(self): + with _IterationGuard(self): + for wr in self.data: + obj = wr() + if obj is not None: + yield obj + + __iter__ = keys + + def values(self): + with _IterationGuard(self): + for wr, value in self.data.items(): + if wr() is not None: + yield value + + def keyrefs(self): + """Return a list of weak references to the keys. + + The references are not guaranteed to be 'live' at the time + they are used, so the result of calling the references needs + to be checked before being used. This can be used to avoid + creating references that will cause the garbage collector to + keep the keys around longer than needed. + + """ + return list(self.data) + + def popitem(self): + self._dirty_len = True + while True: + key, value = self.data.popitem() + o = key() + if o is not None: + return o, value + + def pop(self, key, *args): + self._dirty_len = True + return self.data.pop(self.ref_type(key), *args) # CHANGED + + def setdefault(self, key, default=None): + return self.data.setdefault( + self.ref_type(key, self._remove), default + ) # CHANGED + + def update(self, dict=None, **kwargs): # type: ignore[override] + d = self.data + if dict is not None: + if not hasattr(dict, "items"): + dict = type({})(dict) + for key, value in dict.items(): + d[self.ref_type(key, self._remove)] = value # CHANGED + if len(kwargs): + self.update(kwargs) + + def __ior__(self, other): + self.update(other) + return self + + def __or__(self, other): + if isinstance(other, _collections_abc.Mapping): + c = self.copy() + c.update(other) + return c + return NotImplemented + + def __ror__(self, other): + if isinstance(other, _collections_abc.Mapping): + c = self.__class__() + c.update(other) + c.update(self) + return c + return NotImplemented + + # Default Mapping equality will tests keys for equality, but + # we want to test ids for equality + def __eq__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + return {id(k): v for k, v in self.items()} == { + id(k): v for k, v in other.items() + } + + +# Convenience alias +WeakTensorKeyDictionary = WeakIdKeyDictionary + + +class TensorWeakRef: + """Wrapper around a weak ref of a Tensor that handles the _fix_weakref() call required when unwrapping a Tensor weakref.""" + + ref: WeakRef[Tensor] + + def __init__(self, tensor: Tensor): + assert isinstance(tensor, Tensor) + self.ref = weakref.ref(tensor) + + def __call__(self): + out = self.ref() + if out is None: + return out + assert isinstance(out, Tensor) + # TODO, add _fix_weakref type binding + out._fix_weakref() # type: ignore[attr-defined] + return out \ No newline at end of file diff --git a/mindnlp/utils/torch_proxy.py b/mindnlp/utils/torch_proxy.py index 0cf1ae32d..51fe5e210 100644 --- a/mindnlp/utils/torch_proxy.py +++ b/mindnlp/utils/torch_proxy.py @@ -21,7 +21,7 @@ def find_spec(self, fullname, path, target=None): if fullname == proxy_prefix or fullname.startswith(proxy_prefix + "."): # 计算实际模块名 target_name = fullname.replace(proxy_prefix, target_prefix, 1) - if DEVICE_TARGET == 'Ascend': + if DEVICE_TARGET == 'Ascend' and 'common_cuda' not in target_name: target_name = target_name.replace('cuda', 'npu') try: importlib.import_module(target_name)