From a7273afa25a140bef9e100c480e81647ed02853d Mon Sep 17 00:00:00 2001 From: lvyufeng Date: Sat, 16 Aug 2025 12:30:50 +0800 Subject: [PATCH] fix qwen3/qwen3 moe --- .../peft/lora/Qwen2.5-7B-Instruct-Lora.ipynb | 11198 +++++++++++++++- mindnlp/core/__init__.py | 23 +- mindnlp/core/_bind.py | 17 +- mindnlp/core/_prims/ascend.py | 45 + mindnlp/core/_tensor.py | 6 +- mindnlp/core/amp/autocast_mode.py | 3 + mindnlp/core/amp/grad_scaler.py | 17 +- mindnlp/core/autograd/function.py | 2 + mindnlp/core/dispatcher.py | 109 +- mindnlp/core/nn/functional.py | 4 +- mindnlp/core/npu/__init__.py | 1 + mindnlp/core/ops/_inner.py | 8 +- mindnlp/core/ops/array.py | 6 +- mindnlp/core/ops/inplace.py | 3 + mindnlp/core/ops/pointwise.py | 4 +- mindnlp/core/optim/adam.py | 2 +- mindnlp/core/optim/lr_scheduler.py | 5 +- mindnlp/core/optim/optimizer.py | 18 +- mindnlp/transformers/trainer.py | 6 +- 19 files changed, 10917 insertions(+), 560 deletions(-) diff --git a/examples/transformers/peft/lora/Qwen2.5-7B-Instruct-Lora.ipynb b/examples/transformers/peft/lora/Qwen2.5-7B-Instruct-Lora.ipynb index d831932e7..50ceab397 100644 --- a/examples/transformers/peft/lora/Qwen2.5-7B-Instruct-Lora.ipynb +++ b/examples/transformers/peft/lora/Qwen2.5-7B-Instruct-Lora.ipynb @@ -20,8 +20,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[WARNING] DEVICE(3393766,ffffa6d05020,python):2025-08-15-13:37:22.811.651 [mindspore/ccsrc/utils/dlopen_macro.h:165] DlsymAscend] Dynamically load symbol aclprofGetSupportedFeaturesV2 failed, result = /usr/local/Ascend/ascend-toolkit/latest/lib64/libmsprofiler.so: undefined symbol: aclprofGetSupportedFeaturesV2\n", - "[WARNING] DEVICE(3393766,ffffa6d05020,python):2025-08-15-13:37:22.811.779 [mindspore/ccsrc/utils/dlopen_macro.h:165] DlsymAscend] Dynamically load symbol aclrtEventGetTimestamp failed, result = /usr/local/Ascend/ascend-toolkit/latest/lib64/libascendcl.so: undefined symbol: aclrtEventGetTimestamp\n", + "[WARNING] DEVICE(957136,ffffb13e8020,python):2025-08-16-12:18:37.758.703 [mindspore/ccsrc/utils/dlopen_macro.h:165] DlsymAscend] Dynamically load symbol aclprofGetSupportedFeaturesV2 failed, result = /usr/local/Ascend/ascend-toolkit/latest/lib64/libmsprofiler.so: undefined symbol: aclprofGetSupportedFeaturesV2\n", + "[WARNING] DEVICE(957136,ffffb13e8020,python):2025-08-16-12:18:37.758.835 [mindspore/ccsrc/utils/dlopen_macro.h:165] DlsymAscend] Dynamically load symbol aclrtEventGetTimestamp failed, result = /usr/local/Ascend/ascend-toolkit/latest/lib64/libascendcl.so: undefined symbol: aclrtEventGetTimestamp\n", "/home/lvyufeng/miniconda3/envs/mindspore/lib/python3.11/site-packages/numpy/core/getlimits.py:549: UserWarning: The value of the smallest subnormal for type is zero.\n", " setattr(self, word, getattr(machar, word).flat[0])\n", "/home/lvyufeng/miniconda3/envs/mindspore/lib/python3.11/site-packages/numpy/core/getlimits.py:89: UserWarning: The value of the smallest subnormal for type is zero.\n", @@ -59,16 +59,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "--2025-08-15 13:37:36-- https://openi.pcl.ac.cn/mindnlp/self-llm/raw/branch/master/dataset/huanhuan.json\n", - "Resolving openi.pcl.ac.cn (openi.pcl.ac.cn)... 120.237.18.54, 210.22.22.132, 218.17.115.247\n", - "Connecting to openi.pcl.ac.cn (openi.pcl.ac.cn)|120.237.18.54|:443... connected.\n", + "--2025-08-16 12:18:51-- https://openi.pcl.ac.cn/mindnlp/self-llm/raw/branch/master/dataset/huanhuan.json\n", + "Resolving openi.pcl.ac.cn (openi.pcl.ac.cn)... 210.22.22.132, 120.237.18.54, 218.17.115.247\n", + "Connecting to openi.pcl.ac.cn (openi.pcl.ac.cn)|210.22.22.132|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: unspecified [text/plain]\n", "Saving to: ‘huanhuan.json’\n", "\n", - "huanhuan.json [ <=> ] 914.84K 4.22MB/s in 0.2s \n", + "huanhuan.json [ <=> ] 914.84K 4.41MB/s in 0.2s \n", "\n", - "2025-08-15 13:37:37 (4.22 MB/s) - ‘huanhuan.json’ saved [936793]\n", + "2025-08-16 12:18:52 (4.41 MB/s) - ‘huanhuan.json’ saved [936793]\n", "\n" ] } @@ -214,7 +214,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Map: 100%|███████████████████████████████████████████████████████████████| 3729/3729 [00:13<00:00, 280.87 examples/s]\n" + "Map: 100%|██████████| 3729/3729 [00:13<00:00, 284.85 examples/s]\n" ] }, { @@ -302,7 +302,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Loading checkpoint shards: 100%|███████████████████████████████████████████████████████| 4/4 [00:32<00:00, 8.06s/it]\n" + "Loading checkpoint shards: 100%|██████████| 4/4 [00:34<00:00, 8.54s/it]\n" ] }, { @@ -316,7 +316,7 @@ "source": [ "import torch\n", "\n", - "model = AutoModelForCausalLM.from_pretrained('Qwen/Qwen2.5-7B-Instruct', torch_dtype=torch.bfloat16)\n", + "model = AutoModelForCausalLM.from_pretrained('Qwen/Qwen2.5-7B-Instruct', torch_dtype=torch.float16, attn_implementation='eager')\n", "model = model.npu()" ] }, @@ -374,7 +374,7 @@ { "data": { "text/plain": [ - "LoraConfig(task_type=, peft_type=, auto_mapping=None, base_model_name_or_path=None, revision=None, inference_mode=False, r=8, target_modules={'o_proj', 'up_proj', 'gate_proj', 'q_proj', 'k_proj', 'down_proj', 'v_proj'}, exclude_modules=None, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', trainable_token_indices=None, loftq_config={}, eva_config=None, corda_config=None, use_dora=False, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False), lora_bias=False)" + "LoraConfig(task_type=, peft_type=, auto_mapping=None, base_model_name_or_path=None, revision=None, inference_mode=False, r=8, target_modules={'v_proj', 'o_proj', 'up_proj', 'q_proj', 'k_proj', 'down_proj', 'gate_proj'}, exclude_modules=None, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', trainable_token_indices=None, loftq_config={}, eva_config=None, corda_config=None, use_dora=False, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False), lora_bias=False)" ] }, "execution_count": 13, @@ -407,7 +407,7 @@ { "data": { "text/plain": [ - "LoraConfig(task_type=, peft_type=, auto_mapping=None, base_model_name_or_path='Qwen/Qwen2.5-7B-Instruct', revision=None, inference_mode=False, r=8, target_modules={'o_proj', 'up_proj', 'gate_proj', 'q_proj', 'k_proj', 'down_proj', 'v_proj'}, exclude_modules=None, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', trainable_token_indices=None, loftq_config={}, eva_config=None, corda_config=None, use_dora=False, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False), lora_bias=False)" + "LoraConfig(task_type=, peft_type=, auto_mapping=None, base_model_name_or_path='Qwen/Qwen2.5-7B-Instruct', revision=None, inference_mode=False, r=8, target_modules={'v_proj', 'o_proj', 'up_proj', 'q_proj', 'k_proj', 'down_proj', 'gate_proj'}, exclude_modules=None, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', trainable_token_indices=None, loftq_config={}, eva_config=None, corda_config=None, use_dora=False, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False), lora_bias=False)" ] }, "execution_count": 14, @@ -466,7 +466,7 @@ " save_steps=100, \n", " learning_rate=1e-4,\n", " save_on_each_node=True,\n", - " # fp16=True,\n", + " fp16=True,\n", " # gradient_checkpointing=True\n", ")" ] @@ -483,8 +483,17 @@ "name": "stderr", "output_type": "stream", "text": [ + "/home/lvyufeng/miniconda3/envs/mindspore/lib/python3.11/site-packages/accelerate/utils/modeling.py:2086: FutureWarning: `torch.npu.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('npu', args...)` instead.\n", + " return torch.npu.amp.GradScaler(**kwargs)\n", "Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "get_grad_scaler {}\n" + ] } ], "source": [ @@ -511,7 +520,7 @@ "Local process index: 0\n", "Device: device(type=npu, index=0)\n", "\n", - "Mixed precision type: no" + "Mixed precision type: fp16" ] }, "execution_count": 18, @@ -535,43 +544,118 @@ "name": "stdout", "output_type": "stream", "text": [ - "create_optimizer\n", - "Optimizer.zero_grad#AdamW.zero_grad\n", - "/home/lvyufeng/miniconda3/envs/mindspore/lib/python3.11/site-packages/mindnlp/core/optim/optimizer.py\n", - "/home/lvyufeng/miniconda3/envs/mindspore/lib/python3.11/site-packages/mindnlp/core/optim/optimizer.py\n", - "create_scheduler /home/lvyufeng/miniconda3/envs/mindspore/lib/python3.11/site-packages/mindnlp/core/optim/lr_scheduler.py\n", - "Optimizer.zero_grad#AdamW.zero_grad\n", - "/home/lvyufeng/miniconda3/envs/mindspore/lib/python3.11/site-packages/mindnlp/core/optim/optimizer.py\n", - " \n", - "try func None\n" + "get_mixed_precision_context_manager True\n", + "fp16\n", + "torch.autocast\n", + "\n", + "\n", + "\n", + "起始行号: 55, 源代码: def __init__(\n", + " self,\n", + " device_type: str,\n", + " dtype: Optional[_dtype] = None,\n", + " enabled: bool = True,\n", + " cache_enabled: Optional[bool] = None,\n", + " ):\n", + " if not isinstance(device_type, str):\n", + " raise ValueError(\n", + " f\"Expected `device_type` of type `str`, got: `{type(device_type)}`\"\n", + " )\n", + " self.device_type = device_type\n", + " if dtype is None:\n", + " dtype = float32\n", + " self.dtype = dtype\n", + " self.amp_level = AmpLevel.AmpAuto if enabled else AmpLevel.AmpO0\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:05.938.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "54910.402 0.83786625\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] PRE_ACT(957136,fffd8bfff120,python):2025-08-16-12:20:06.449.385 [mindspore/ccsrc/memory/mem_pool/mem_dynamic_allocator.cc:700] FreeIdleMemsByEagerFree] Eager free count : 2, free memory : 12837284864, real free : 1186988032, not free size: 11650296832.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:09.614.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "112496.695 1.7165633\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:10.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "81901.17 1.2497127\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:11.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "71451.305 1.0902604\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [-1018540541 0 0 0 0 0\n", + " 0 0]\n", + "found_inf [-1018540541 0 0 0 0 0\n", + " 0 0]\n", + "found_inf [ True False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 1.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "1.0\n", + "scaler update\n", + "scaler update False\n", + "65536.0\n", + "32768.0\n" ] }, { @@ -580,8 +664,8 @@ "\n", "
\n", " \n", - " \n", - " [ 15/702 00:40 < 35:31, 0.32 it/s, Epoch 0.06/3]\n", + " \n", + " [145/702 09:43 < 37:52, 0.25 it/s, Epoch 0.62/3]\n", "
\n", " \n", " \n", @@ -593,7 +677,59 @@ " \n", " \n", " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
105.0050004.372900
203.147200
302.964200
403.193600
503.062600
602.995400
702.982700
803.067700
902.939100
1003.005000
1103.044500
1203.052100
1303.225000
1403.469700

" @@ -605,466 +741,10518 @@ "metadata": {}, "output_type": "display_data" }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:12.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n", - " \n", - "try func None\n" + "30739.01 0.93808013\n", + "all_finite forward False\n" ] - } - ], - "source": [ - "trainer.train()" - ] - }, - { - "cell_type": "markdown", - "id": "8abb2327-458e-4e96-ac98-2141b5b97c8e", - "metadata": {}, - "source": [ - "# 合并加载模型" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bd2a415a-a9ad-49ea-877f-243558a83bfc", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoModelForCausalLM, AutoTokenizer\n", - "import torch\n", - "from peft import PeftModel\n", - "\n", - "mode_path = 'Qwen/Qwen2.5-7B-Instruct'\n", - "lora_path = './output/Qwen2.5_instruct_lora/checkpoint-702' # 这里改称你的 lora 输出对应 checkpoint 地址\n", - "\n", - "# 加载tokenizer\n", - "tokenizer = AutoTokenizer.from_pretrained(mode_path, trust_remote_code=True)\n", - "\n", - "# 加载模型\n", - "model = AutoModelForCausalLM.from_pretrained(mode_path, device_map=\"auto\",torch_dtype=torch.bfloat16, trust_remote_code=True).eval()\n", - "\n", - "# 加载lora权重\n", - "model = PeftModel.from_pretrained(model, model_id=lora_path)\n", - "\n", - "prompt = \"你是谁？\"\n", - "inputs = tokenizer.apply_chat_template([{\"role\": \"user\", \"content\": \"假设你是皇帝身边的女人--甄嬛。\"},{\"role\": \"user\", \"content\": prompt}],\n", - " add_generation_prompt=True,\n", - " tokenize=True,\n", - " return_tensors=\"pt\",\n", - " return_dict=True\n", - " ).to('cuda')\n", - "\n", - "\n", - "gen_kwargs = {\"max_length\": 2500, \"do_sample\": True, \"top_k\": 1}\n", - "with torch.no_grad():\n", - " outputs = model.generate(**inputs, **gen_kwargs)\n", - " outputs = outputs[:, inputs['input_ids'].shape[1]:]\n", - " print(tokenizer.decode(outputs[0], skip_special_tokens=True))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a102696b", - "metadata": {}, - "outputs": [], - "source": [] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:13.394.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "88414.24 2.6981885\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:14.234.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "31042.045 0.94732803\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:15.440.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "28081.932 0.85699254\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [-803458813 0 0 0 0 0\n", + " 0 0]\n", + "found_inf [-803458813 0 0 0 0 0\n", + " 0 0]\n", + "found_inf [ True False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 1.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "1.0\n", + "scaler update\n", + "scaler update False\n", + "32768.0\n", + "16384.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:16.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "17063.844 1.0414944\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:16.936.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "20659.578 1.2609606\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:17.764.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "18347.51 1.1198431\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:18.581.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "18720.67 1.142619\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [-1164986109 0 0 0 0 0\n", + " 0 0]\n", + "found_inf [-1164986109 0 0 0 0 0\n", + " 0 0]\n", + "found_inf [ True False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 1.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "1.0\n", + "scaler update\n", + "scaler update False\n", + "16384.0\n", + "8192.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:19.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12079.898 1.474597\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:20.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "14997.002 1.8306887\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:21.277.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9899.985 1.2084943\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:22.870.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7154.4824 0.8733499\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:23.624.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9302.338 1.1355393\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:24.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "11410.603 1.3928958\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:25.344.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "13681.863 1.6701493\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:26.230.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7389.4536 0.9020329\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:27.821.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9701.396 1.1842525\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:28.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4588.254 0.5600896\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:29.512.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7891.192 0.96328026\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:30.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9734.306 1.1882697\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:31.953.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5672.57 0.6924524\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:32.776.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5531.1675 0.67519134\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:33.639.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9602.322 1.1721585\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:34.572.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "11391.412 1.3905532\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:36.158.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6778.675 0.82747495\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:36.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6709.415 0.8190204\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:37.830.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7294.2373 0.8904098\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:38.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7415.2695 0.90518427\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:40.357.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5251.7427 0.64108187\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:41.207.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "11432.913 1.3956193\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:42.830.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7051.7896 0.86081415\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:42.961.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5747.045 0.70154357\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:44.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7764.5986 0.947827\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:45.326.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6362.836 0.7767134\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:46.160.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7277.203 0.88833046\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:47.800.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7053.908 0.8610728\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:48.617.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5251.716 0.6410786\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:49.535.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9778.426 1.1936555\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:50.511.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4631.264 0.56533986\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:51.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6773.184 0.8268047\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:53.201.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2924.2185 0.35696027\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:54.127.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7816.322 0.95414084\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:55.132.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6861.905 0.83763486\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:56.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6842.7554 0.8352973\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:57.305.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5346.579 0.6526586\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:57.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5900.034 0.720219\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:58.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "13363.887 1.6313338\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:20:59.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3293.2358 0.40200633\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:00.882.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4303.8423 0.5253714\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:01.612.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "11541.967 1.4089315\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:02.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7780.0923 0.9497183\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:03.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4069.7551 0.49679628\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:04.553.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9629.118 1.1754295\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:05.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8123.1504 0.9915955\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:06.140.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2239.2522 0.27334622\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:06.767.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6092.8887 0.7437608\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:08.139.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5872.828 0.71689796\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:08.897.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9445.795 1.1530511\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:09.519.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3538.7412 0.43197525\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:10.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7384.433 0.90142006\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:11.596.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4692.3076 0.57279146\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:12.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6250.6396 0.76301754\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:13.290.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8471.654 1.0341375\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:14.196.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3702.066 0.45191234\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:15.826.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3078.8574 0.3758371\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:16.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5785.0063 0.70617753\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:17.641.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7677.179 0.93715566\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:18.574.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9333.895 1.1393914\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:20.378.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9388.088 1.1460068\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:21.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5197.871 0.63450575\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:22.286.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3747.0469 0.45740318\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:23.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7584.0938 0.9257927\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:24.853.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9502.698 1.1599973\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:25.697.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3334.9731 0.4071012\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:26.563.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6249.941 0.76293224\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:27.456.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5016.228 0.6123325\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:29.180.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8176.815 0.99814636\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:29.851.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8464.828 1.0333042\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:30.708.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2668.5417 0.32574973\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:31.573.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6561.969 0.80102164\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:33.128.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2891.233 0.3529337\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:34.740.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9314.32 1.137002\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:34.916.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4721.1777 0.57631564\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:35.740.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6200.0293 0.7568395\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:37.206.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3346.6133 0.40852213\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:37.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6461.0024 0.7886966\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:38.815.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6798.041 0.829839\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:39.637.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5975.327 0.72941005\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:41.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5864.8228 0.71592075\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:41.896.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4894.2925 0.5974478\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:42.676.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3699.2966 0.4515743\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:43.459.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "11721.943 1.4309013\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:44.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3283.5522 0.40082425\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:45.627.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7233.9663 0.8830525\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:46.448.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8731.289 1.0658312\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:47.275.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4651.584 0.5678203\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:48.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10831.513 1.3222061\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:49.631.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4985.6704 0.60860234\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:50.446.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6020.3394 0.7349047\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:51.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2673.9316 0.32640767\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:52.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9668.475 1.1802337\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:53.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5637.6772 0.688193\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:54.157.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5649.9814 0.689695\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:54.799.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4796.758 0.5855417\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:56.120.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3073.137 0.3751388\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:56.616.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5740.5127 0.7007462\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:57.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10597.762 1.2936721\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:57.932.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5920.4272 0.7227084\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:59.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5089.523 0.62127966\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:21:59.769.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3671.932 0.44823387\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:00.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4486.311 0.5476454\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:01.610.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9323.181 1.1380836\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:02.278.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5019.006 0.6126716\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:02.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6086.423 0.74297154\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:03.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6050.372 0.7385708\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:04.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5843.4897 0.7133166\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:06.560.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10407.959 1.2705028\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:06.806.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3190.8948 0.38951352\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:07.401.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5735.7183 0.7001609\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:08.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8044.478 0.98199195\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [3 0 0 0 0 0 0 0]\n", + "found_inf [3 0 0 0 0 0 0 0]\n", + "found_inf [ True False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 1.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "1.0\n", + "scaler update\n", + "scaler update False\n", + "8192.0\n", + "4096.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:08.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2063.103 0.50368726\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:09.406.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5106.7617 1.246768\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:10.160.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3206.504 0.78283787\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:10.629.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3586.835 0.8756921\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:11.798.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3134.63 0.7652905\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:12.376.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3094.2312 0.75542754\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:12.974.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2821.3284 0.6888009\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:13.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4547.082 1.1101274\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:14.711.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2537.3735 0.61947596\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:15.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2220.1116 0.5420194\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:15.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5356.7944 1.3078111\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:16.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2157.0803 0.52663094\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:18.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2994.0312 0.73096466\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:18.800.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3442.0366 0.840341\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:19.580.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3599.7903 0.87885505\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:20.362.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2163.684 0.5282432\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:21.880.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4498.332 1.0982256\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:22.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2629.19 0.6418921\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:23.489.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4061.885 0.99167114\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:24.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1987.3943 0.48520368\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:25.499.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3955.8167 0.96577555\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:26.279.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3548.518 0.8663374\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:27.200.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4050.5361 0.9889004\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:27.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1569.208 0.38310742\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:29.107.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "832.6935 0.2032943\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:29.906.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3535.163 0.8630769\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:30.750.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1524.4944 0.372191\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:31.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6841.27 1.6702319\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:33.108.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4650.126 1.1352847\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:33.934.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4189.2603 1.0227686\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:34.620.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1747.7362 0.4266934\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:35.417.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2132.2605 0.5205714\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:37.100.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4626.734 1.1295737\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:37.758.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1891.8018 0.46186566\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:38.566.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4587.7695 1.1200609\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:39.425.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2230.4863 0.5445523\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:41.191.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6230.9077 1.5212177\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:42.150.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2295.3296 0.5603832\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:43.148.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2321.9253 0.5668763\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:44.159.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3213.8562 0.78463286\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:45.909.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2609.2563 0.6370255\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:46.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4119.854 1.0058237\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:47.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1779.3223 0.43440485\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:48.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2661.3545 0.64974475\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:49.810.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3987.2744 0.97345567\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:50.673.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5199.431 1.2693924\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:51.534.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2221.0906 0.54225844\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:52.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1756.6427 0.42886785\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:53.600.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3740.6125 0.9132355\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:54.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1228.1027 0.29982975\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:55.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1765.4086 0.43100795\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:55.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5914.357 1.4439348\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:57.402.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1924.4934 0.46984702\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:58.231.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3857.4397 0.94175774\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:22:59.143.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5267.875 1.2861023\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:00.630.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "802.6415 0.19595739\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:01.661.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4382.6255 1.0699769\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:02.495.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1668.9064 0.40744784\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:03.385.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2848.946 0.69554347\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:04.274.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4386.1807 1.0708449\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:05.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3836.6562 0.93668365\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:06.671.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2283.4812 0.5574905\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:07.530.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2550.9177 0.62278265\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:08.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3712.3071 0.906325\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:09.959.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3248.1965 0.79301673\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:10.801.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3715.835 0.90718627\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:11.437.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1434.2343 0.35015485\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:12.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5075.508 1.2391376\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:13.515.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3918.3735 0.95663416\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:14.493.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4883.9463 1.1923697\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:15.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2761.7478 0.67425483\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:16.526.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "731.8806 0.17868179\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:18.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2371.8125 0.5790558\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:19.287.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3086.8704 0.75363046\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:20.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3252.5999 0.79409176\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:21.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2395.0474 0.58472836\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:23.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4134.273 1.009344\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:24.177.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3289.594 0.80312353\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:25.253.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2756.6907 0.6730202\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:26.321.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2028.4688 0.49523163\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:28.126.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1733.1372 0.4231292\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:28.925.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6022.089 1.4702365\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:29.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2203.7324 0.5380206\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:30.633.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2952.8098 0.72090083\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:32.151.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4729.455 1.1546521\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:32.948.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1020.94714 0.24925467\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:33.778.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2930.2244 0.7153868\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:34.592.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2463.5574 0.60145444\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:36.217.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3422.4927 0.8355695\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:37.860.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4271.732 1.0429033\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:37.994.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2430.3735 0.5933529\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:38.824.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3444.4534 0.840931\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:40.531.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6385.7764 1.5590274\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:41.503.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1240.1918 0.3027812\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:42.324.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2183.2522 0.53302056\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:43.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2284.8604 0.55782723\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:44.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2473.9578 0.6039936\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:45.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3331.0835 0.8132528\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:46.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1416.7529 0.34588695\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:47.240.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5735.763 1.4003328\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:48.929.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3121.3445 0.762047\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:49.848.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3999.0747 0.9763366\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:50.744.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1499.0521 0.36597952\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:51.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2203.596 0.5379873\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:52.978.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4022.6245 0.98208606\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:53.631.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2960.8364 0.72286046\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:54.461.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2717.0479 0.66334176\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:55.262.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2791.7627 0.6815827\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:56.729.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2090.3008 0.51032734\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:57.487.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1559.8855 0.38083142\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:58.395.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5144.402 1.2559575\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:23:59.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3353.7117 0.81877726\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:00.921.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3815.8193 0.9315965\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:01.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6458.0757 1.5766786\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:02.752.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "703.1942 0.17167827\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:03.665.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1366.9177 0.33372015\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:05.250.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3784.085 0.92384887\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:06.149.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1432.3479 0.3496943\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:07.860.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1503.9475 0.36717469\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:08.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5457.336 1.3323574\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:09.679.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2521.4412 0.6155862\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:10.501.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2520.7927 0.6154279\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:11.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4100.4893 1.001096\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:12.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3381.381 0.8255325\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:13.874.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1938.0549 0.47315794\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:14.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6447.8345 1.5741783\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:15.359.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2814.3816 0.6871049\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:16.188.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2760.7659 0.6740151\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:17.860.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1892.635 0.4620691\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:18.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2359.016 0.57593167\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:19.709.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3728.3625 0.91024476\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:20.651.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3748.4856 0.9151576\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:22.375.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2375.694 0.58000344\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:23.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2720.705 0.66423464\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:24.245.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2644.7095 0.645681\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:25.200.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3930.5056 0.9595961\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:26.904.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2034.2415 0.49664098\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:27.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2664.4976 0.6505121\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:28.797.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2996.8933 0.7316634\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:29.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3425.7595 0.8363671\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:31.438.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3279.7349 0.8007165\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:32.334.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1817.4989 0.44372532\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:33.295.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2916.2424 0.71197325\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:34.860.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3910.3855 0.95468396\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:35.607.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2448.4377 0.5977631\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:36.413.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3935.6804 0.9608595\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:37.251.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3086.025 0.75342405\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:38.880.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3903.8867 0.95309734\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:39.602.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1581.8405 0.38619152\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:40.398.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4406.353 1.0757698\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:41.235.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3575.5322 0.8729327\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:42.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3104.3206 0.75789076\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:43.590.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "661.32446 0.16145617\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:44.392.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2175.3613 0.5310941\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:45.233.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3228.7397 0.78826654\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:46.930.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4954.6807 1.2096388\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:47.660.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5022.986 1.2263149\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:48.464.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1163.1852 0.28398076\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:49.299.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2857.671 0.69767356\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:50.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3177.867 0.7758464\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:51.773.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4655.253 1.1365364\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:52.472.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3287.647 0.8026482\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:53.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2145.1516 0.52371866\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:54.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2959.772 0.7226006\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:55.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4290.3853 1.0474573\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:56.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3385.0151 0.8264197\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:57.193.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2108.6501 0.51480716\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:57.951.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2900.335 0.7080896\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:24:59.384.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4811.238 1.1746186\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:00.297.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1017.25867 0.24835417\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:01.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2914.7021 0.7115972\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:02.160.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3869.4255 0.94468397\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:03.570.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4281.2715 1.0452323\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:04.433.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3111.8303 0.7597242\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:05.306.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2724.7866 0.6652311\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:06.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2723.32 0.66487306\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:07.785.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3569.3816 0.87143105\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:08.608.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2030.474 0.4957212\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:09.475.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2737.8909 0.6684304\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:10.354.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4238.767 1.0348552\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:11.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1876.1686 0.45804897\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:12.638.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2766.6577 0.67545354\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:13.450.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4410.1436 1.0766952\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:14.273.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3249.0298 0.79322016\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:15.772.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3512.2156 0.8574745\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:16.551.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2828.2466 0.6904899\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:17.387.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4035.949 0.9853391\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:18.296.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2642.2598 0.64508295\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:19.887.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2404.0688 0.5869309\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:20.733.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3282.9634 0.80150473\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:21.632.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2404.595 0.5870593\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:22.533.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3850.3262 0.94002104\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:24.134.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3645.4087 0.88999236\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:24.975.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2321.8076 0.56684756\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:25.871.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3742.2166 0.9136271\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:26.769.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2697.6575 0.6586078\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:28.370.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2612.6794 0.6378612\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:29.217.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1409.8065 0.34419104\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:30.114.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4215.067 1.0290691\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:31.120.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2363.574 0.5770444\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:32.618.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4023.3787 0.9822702\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:33.468.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1198.9829 0.29272044\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:34.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4681.8955 1.1430409\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:35.292.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3139.9504 0.76658946\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:36.893.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3898.5327 0.9517902\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:37.749.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4852.447 1.1846794\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:38.640.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "775.9978 0.18945259\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:39.584.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3675.351 0.8973025\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:41.254.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4687.4033 1.1443856\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:42.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2975.3308 0.7263991\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:42.966.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1972.605 0.481593\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:43.870.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1634.4521 0.39903617\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:45.541.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4789.534 1.1693199\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:46.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2980.1025 0.7275641\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:47.325.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2221.8416 0.5424418\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:48.239.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2832.235 0.69146365\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:49.905.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3214.4805 0.7847853\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:50.767.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2249.228 0.54912794\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:51.686.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4381.9355 1.0698085\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:52.552.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3384.9075 0.8263934\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:54.820.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2204.1877 0.5381318\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:54.928.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3218.2935 0.7857162\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:55.834.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1773.498 0.43298292\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:56.733.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4461.385 1.0892053\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:58.371.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2602.352 0.63533986\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:25:59.195.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3387.674 0.82706887\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:00.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "760.3062 0.18562163\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:00.976.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5261.101 1.2844485\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:02.589.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2562.8271 0.6256902\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:03.419.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2316.9653 0.56566536\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:04.303.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2486.6904 0.60710216\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:05.184.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3944.2283 0.96294636\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:06.813.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3088.5564 0.7540421\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:07.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2681.675 0.6547058\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:08.536.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4771.17 1.1648364\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:09.416.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "694.8997 0.16965325\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:11.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1366.021 0.33350122\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:11.860.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3513.4495 0.85777575\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:12.739.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4784.0063 1.1679703\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:13.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3176.009 0.77539283\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:15.372.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2994.316 0.73103416\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:16.260.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3309.6448 0.80801874\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:17.161.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4509.583 1.1009724\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:18.130.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "708.0294 0.17285874\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:19.585.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3415.4304 0.8338453\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:20.364.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3763.2283 0.9187569\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:21.174.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3140.482 0.7667192\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:21.988.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2068.263 0.504947\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:23.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5703.475 1.39245\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:24.337.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2879.4458 0.7029897\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:25.153.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2471.863 0.6034822\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:25.976.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1287.2333 0.31426594\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:27.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3874.8025 0.9459967\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:28.257.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4476.9316 1.0930009\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:29.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2039.0051 0.497804\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:30.720.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2269.6277 0.5541083\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:31.786.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5291.4897 1.2918676\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:32.691.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2163.3503 0.5281617\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:33.642.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3316.262 0.80963427\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:34.595.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "609.1483 0.14871785\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:36.307.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3665.958 0.8950093\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:37.208.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4388.3657 1.0713784\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:38.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "818.99097 0.19994897\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:39.122.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3164.364 0.7725498\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:40.830.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2013.4603 0.49156746\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:41.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3498.887 0.85422045\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:42.692.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1435.1669 0.35038254\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:43.652.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5345.398 1.3050288\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:45.361.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4083.9087 0.997048\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:46.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2068.6528 0.5050422\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:47.249.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4617.3994 1.1272948\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:48.198.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2951.5132 0.7205843\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:49.888.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2527.65 0.617102\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:50.790.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3003.5598 0.733291\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:51.747.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2533.02 0.6184131\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:52.697.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3836.7646 0.9367101\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:57.819.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3517.1953 0.85869026\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:58.659.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3907.5168 0.9539836\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:26:59.569.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3199.6958 0.78117573\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:00.420.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2244.1816 0.5478959\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:01.968.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3987.194 0.97343606\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:02.782.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5922.999 1.4460447\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:03.653.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2669.573 0.6517512\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:04.524.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1339.6589 0.32706517\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:06.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4780.2915 1.1670634\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:06.950.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2808.8567 0.685756\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:07.823.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4319.5864 1.0545865\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:08.700.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1555.7203 0.37981454\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:10.272.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3972.7566 0.9699113\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:11.109.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3148.7598 0.7687402\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:11.982.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2567.216 0.62676173\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:12.859.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3792.02 0.92578614\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:14.456.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4074.2795 0.99469715\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:15.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2182.1865 0.5327604\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:16.590.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4822.094 1.1772691\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:16.807.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1527.0446 0.3728136\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:18.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2736.775 0.66815794\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:18.916.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "929.5039 0.22692966\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:19.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4356.846 1.0636832\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:20.612.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3247.2205 0.79277843\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:22.910.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1996.5781 0.48744583\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:22.875.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2653.3093 0.6477806\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:23.689.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3320.1025 0.8105719\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:24.613.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4271.953 1.0429573\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:26.150.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2766.1836 0.6753378\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:26.813.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2076.362 0.50692433\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:27.670.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3979.745 0.97161746\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:28.523.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2629.8696 0.642058\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:30.660.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1676.544 0.4093125\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:30.827.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1614.4132 0.39414385\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:31.638.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3877.262 0.94659716\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:32.444.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3491.2598 0.85235834\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:33.979.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4200.6865 1.0255582\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:34.807.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3300.4563 0.80577546\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:35.684.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3413.9983 0.8334957\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:36.559.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1825.1127 0.44558415\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:38.115.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2158.9978 0.5270991\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:38.973.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2280.8928 0.5568586\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:39.868.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3151.2498 0.7693481\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:40.706.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4769.5474 1.1644403\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:42.225.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1355.8026 0.3310065\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:43.170.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2831.3933 0.69125813\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:43.864.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4406.976 1.0759219\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:44.705.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3419.664 0.8348789\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:46.256.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2348.3533 0.57332844\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:47.118.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4425.2124 1.0803741\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:47.999.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2748.4243 0.67100203\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:48.878.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1977.3588 0.4827536\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:50.483.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5191.6143 1.267484\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:51.407.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2272.7595 0.55487293\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:52.374.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2765.5916 0.67519325\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:53.349.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3578.8508 0.8737429\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:55.130.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4559.232 1.1130937\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:55.899.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1396.2253 0.34087533\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:56.812.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4579.3677 1.1180097\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:57.727.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2098.8145 0.5124059\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:27:59.350.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1856.6504 0.4532838\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:00.237.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3838.4595 0.9371239\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:01.169.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4931.9004 1.2040772\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:02.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2437.27 0.5950366\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:03.761.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4280.9966 1.0451652\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:04.655.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1824.5895 0.44545642\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:05.576.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2574.7996 0.6286132\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:06.522.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2493.048 0.6086543\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:08.142.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4956.0977 1.2099848\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:08.980.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1881.2456 0.45928848\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:09.836.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3857.2769 0.941718\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:10.649.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1254.2574 0.3062152\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:12.970.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2241.2073 0.54716974\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:12.884.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2695.1318 0.6579912\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:13.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4728.833 1.1545002\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:14.517.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3484.7983 0.85078084\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [-1355665149 0 0 0 0 0\n", + " 0 0]\n", + "found_inf [-1355665149 0 0 0 0 0\n", + " 0 0]\n", + "found_inf [ True False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 1.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "1.0\n", + "scaler update\n", + "scaler update False\n", + "4096.0\n", + "2048.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:15.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2687.9365 1.312469\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:16.564.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1284.6727 0.6272816\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:17.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1028.1296 0.5020164\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:18.399.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1679.874 0.820251\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:20.250.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1269.6825 0.61996216\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:20.906.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1280.1108 0.6250541\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:21.823.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "752.4598 0.367412\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:22.750.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2832.8757 1.3832401\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:24.386.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1847.7007 0.9021976\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:25.271.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1878.5565 0.9172639\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:26.133.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1835.2625 0.89612424\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:27.900.0 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1685.8992 0.82319295\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:28.663.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1761.0078 0.8598671\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:29.575.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2128.3337 1.0392255\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:30.491.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1864.431 0.9103667\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:31.403.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "718.7248 0.35093984\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:33.440.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1000.2243 0.48839077\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:34.360.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2324.5261 1.1350225\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:34.818.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "600.19507 0.293064\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:35.516.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2589.2842 1.2642989\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:36.970.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1109.3081 0.54165435\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:37.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1103.2681 0.5387051\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:38.537.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1445.8438 0.7059784\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:39.339.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3183.7522 1.5545665\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:40.816.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1476.5559 0.72097456\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:41.609.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2311.9521 1.1288829\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:42.410.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1559.1323 0.7612951\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:43.210.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "855.9128 0.41792616\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:44.683.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "545.6742 0.26644248\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:45.436.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1966.5497 0.96022934\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:46.197.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2017.1786 0.9849505\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:46.946.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2230.986 1.0893487\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:48.482.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1629.3646 0.7955882\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:49.291.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1805.7606 0.88171905\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:50.163.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2679.648 1.3084219\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:51.260.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "545.60693 0.26640964\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:52.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1477.9883 0.72167397\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:53.408.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1786.734 0.8724287\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:54.268.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1263.3077 0.6168495\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:55.154.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1791.2345 0.8746262\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:56.762.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1689.6965 0.82504714\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:57.601.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1009.9577 0.4931434\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:58.490.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1871.7771 0.91395366\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:28:59.379.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2321.0906 1.133345\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:00.992.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1083.5278 0.5290663\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:01.832.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1789.0507 0.8735599\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:02.734.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2226.2102 1.0870167\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:03.630.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1832.8191 0.8949312\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:05.121.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3011.4976 1.4704578\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:05.914.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1144.1918 0.5586874\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:06.738.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1817.8107 0.88760287\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:07.545.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "950.64014 0.46417975\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:09.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "776.62964 0.3792137\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:09.794.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1523.0018 0.74365324\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:10.598.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1859.3116 0.907867\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:11.406.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2627.973 1.2831899\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:12.871.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1453.8527 0.709889\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:13.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3468.0308 1.6933744\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:14.457.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1243.8833 0.6073649\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:15.259.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1032.7545 0.50427467\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:16.730.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "745.72 0.36412108\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:17.498.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3014.853 1.4720962\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:18.308.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2728.3872 1.3322203\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:19.113.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "835.76514 0.40808845\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:20.558.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2196.3145 1.0724192\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:21.302.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1899.5171 0.9274986\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:22.810.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1024.5664 0.50027657\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:22.865.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1954.4487 0.95432067\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:24.280.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1415.517 0.6911704\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:25.280.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1292.7661 0.63123345\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:25.819.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2534.3506 1.2374759\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:26.643.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1985.1097 0.96929187\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:28.730.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1159.2529 0.56604147\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:28.952.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1769.1779 0.8638564\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:29.717.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "904.20416 0.44150594\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:30.571.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3243.373 1.5836782\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:32.146.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1253.8743 0.6122433\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:32.969.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2329.8455 1.1376199\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:33.809.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2122.3071 1.0362828\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:34.647.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1846.2656 0.9014969\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:36.198.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2907.9622 1.4199034\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:36.998.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1618.8727 0.7904652\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:37.839.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1014.78235 0.4954992\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:38.681.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1420.6028 0.6936537\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:40.236.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2495.4941 1.218503\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:41.240.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1886.6587 0.92122006\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:41.861.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1711.8558 0.8358671\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:42.698.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2554.4795 1.2473044\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:44.244.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3104.7605 1.5159963\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:45.370.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1519.2853 0.7418385\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:45.876.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2525.38 1.2330956\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:46.722.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1891.6155 0.9236404\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:48.264.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1249.3854 0.61005145\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:49.460.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3613.9075 1.7646033\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:49.887.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2038.497 0.99535984\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:50.724.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3012.2153 1.4708083\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:52.270.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1920.4708 0.9377299\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:53.570.00 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2902.804 1.4173847\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:53.802.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1897.9937 0.9267547\n", + "all_finite forward False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:54.543.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3229.8235 1.5770622\n", + "all_finite forward False\n", + "found_inf before 0.0\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [0 0 0 0 0 0 0 0]\n", + "found_inf [False False False False False False False False]\n", + "after clear [0 0 0 0 0 0 0 0]\n", + "found_inf after 0.0\n", + "grad scaler\n", + "False\n", + "_maybe_opt_step\n", + "0.0\n", + "patched_step\n", + "scaler update\n", + "scaler update False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[WARNING] ME(957136:281473655406624,MainProcess):2025-08-16-12:29:55.991.000 [mindspore/run_check/_check_version.py:305] The version 7.7 used for compiling the custom operator does not match Ascend AI software package version 7.6 in the current environment.\n" + ] + } + ], + "source": [ + "trainer.train()" + ] + }, + { + "cell_type": "markdown", + "id": "8abb2327-458e-4e96-ac98-2141b5b97c8e", + "metadata": {}, + "source": [ + "# 合并加载模型" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd2a415a-a9ad-49ea-877f-243558a83bfc", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoModelForCausalLM, AutoTokenizer\n", + "import torch\n", + "from peft import PeftModel\n", + "\n", + "mode_path = 'Qwen/Qwen2.5-7B-Instruct'\n", + "lora_path = './output/Qwen2.5_instruct_lora/checkpoint-702' # 这里改称你的 lora 输出对应 checkpoint 地址\n", + "\n", + "# 加载tokenizer\n", + "tokenizer = AutoTokenizer.from_pretrained(mode_path, trust_remote_code=True)\n", + "\n", + "# 加载模型\n", + "model = AutoModelForCausalLM.from_pretrained(mode_path, device_map=\"auto\",torch_dtype=torch.bfloat16, trust_remote_code=True).eval()\n", + "\n", + "# 加载lora权重\n", + "model = PeftModel.from_pretrained(model, model_id=lora_path)\n", + "\n", + "prompt = \"你是谁？\"\n", + "inputs = tokenizer.apply_chat_template([{\"role\": \"user\", \"content\": \"假设你是皇帝身边的女人--甄嬛。\"},{\"role\": \"user\", \"content\": prompt}],\n", + " add_generation_prompt=True,\n", + " tokenize=True,\n", + " return_tensors=\"pt\",\n", + " return_dict=True\n", + " ).to('cuda')\n", + "\n", + "\n", + "gen_kwargs = {\"max_length\": 2500, \"do_sample\": True, \"top_k\": 1}\n", + "with torch.no_grad():\n", + " outputs = model.generate(**inputs, **gen_kwargs)\n", + " outputs = outputs[:, inputs['input_ids'].shape[1]:]\n", + " print(tokenizer.decode(outputs[0], skip_special_tokens=True))" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "mindspore", "language": "python", "name": "python3" }, diff --git a/mindnlp/core/__init__.py b/mindnlp/core/__init__.py index 006ed4e75..71d1a45a6 100644 --- a/mindnlp/core/__init__.py +++ b/mindnlp/core/__init__.py @@ -50,7 +50,9 @@ from .autograd import * from .ops import * from .serialization import load, save -from ._bind import get_default_dtype, set_default_dtype, get_default_device +from ._bind import get_default_dtype, set_default_dtype, get_default_device, is_autocast_enabled, set_autocast_enabled, \ + set_autocast_dtype, get_autocast_dtype + from .amp import autocast, GradScaler from .func import vmap from .configs import set_pyboost @@ -90,25 +92,6 @@ def wrap_func(fn): return wrap_func(fn) return wrap_func -AUTO_CAST_DTYE = { - 'cuda': float16, - 'cpu': float16, - 'npu': float16, - 'Ascend': float16 -} - -def set_autocast_dtype(device_type, dtype): - assert device_type in AUTO_CAST_DTYE.keys(), f'{device_type} is not in {AUTO_CAST_DTYE.keys()}' - AUTO_CAST_DTYE[device_type] = dtype - -def get_autocast_dtype(device_type): - return AUTO_CAST_DTYE[device_type] - -def get_autocast_gpu_dtype(): - return AUTO_CAST_DTYE['cuda'] - -def is_autocast_enabled(): - return True def use_deterministic_algorithms(mode, *, warn_only=False): mindspore.set_context(deterministic='ON' if mode else 'OFF') diff --git a/mindnlp/core/_bind.py b/mindnlp/core/_bind.py index 60358145a..cad31e74b 100644 --- a/mindnlp/core/_bind.py +++ b/mindnlp/core/_bind.py @@ -10,10 +10,17 @@ AUTO_CAST_DTYE = { 'cuda': float16, - 'cpu': bfloat16, - 'npu': float16 + 'cpu': float16, + 'npu': float16, + 'Ascend': float16 } +AUTO_CAST_ENABLED = False + +def set_autocast_enabled(device, mode): + global AUTO_CAST_ENABLED + AUTO_CAST_ENABLED = mode + def set_autocast_dtype(device_type, dtype): assert device_type in AUTO_CAST_DTYE.keys(), f'{device_type} is not in {AUTO_CAST_DTYE.keys()}' AUTO_CAST_DTYE[device_type] = dtype @@ -21,6 +28,12 @@ def set_autocast_dtype(device_type, dtype): def get_autocast_dtype(device_type): return AUTO_CAST_DTYE[device_type] +def get_autocast_gpu_dtype(): + return AUTO_CAST_DTYE['cuda'] + +def is_autocast_enabled(device): + return AUTO_CAST_ENABLED + def set_default_dtype(dtype): """set default dtype""" global DEFAULT_DTYPE diff --git a/mindnlp/core/_prims/ascend.py b/mindnlp/core/_prims/ascend.py index 6466e385f..5058a9375 100644 --- a/mindnlp/core/_prims/ascend.py +++ b/mindnlp/core/_prims/ascend.py @@ -1,9 +1,11 @@ import numbers +import mindspore from mindspore import ops from mindspore.ops.auto_generate import gen_ops_prim from mindspore.ops.auto_generate import pyboost_inner_prim from mindspore._c_expression import _empty_instance from mindspore.ops.operations.math_ops import NPUGetFloatStatusV2, NPUClearFloatStatusV2 +from mindspore.ops.operations.nn_ops import AllFinite from mindnlp import core from mindnlp.core._C import default_generator @@ -163,6 +165,8 @@ def reverse_v2(input, dims): dims = (dims,) return pyboost_inner_prim.reverse_v2_impl(input, dims) +__all__.append('reverse_v2') + adam_op = ops.Adam().set_device('Ascend') def raw_adam(param, exp_avg, exp_avg_sq, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad): # var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad @@ -193,3 +197,44 @@ def stop_gradient(*args): return stop_gradient_op(*args) __all__.append('stop_gradient') + +# allfinite_op = AllFinite().set_device('Ascend') +def all_finite(inputs): + return allfinite_op(inputs) + +def rsqrt_fp32(input): + return rsqrt(input.astype(mindspore.float32)) + +__all__.append('rsqrt_fp32') + +def matmul_ext_fp16(input, other): + return matmul_ext(input.astype(mindspore.float16), other.astype(mindspore.float16)) + +__all__.append('matmul_ext_fp16') + +def dense_fp16(input, weight, bias): + input = input.astype(mindspore.float16) + weight = weight.astype(mindspore.float16) + if bias is not None: + bias = bias.astype(mindspore.float16) + return dense(input, weight, bias) + +__all__.append('dense_fp16') + +def softmax_fp32(input, dim): + return softmax(input.astype(mindspore.float32), dim) + +__all__.append('softmax_fp32') + +def log_softmax_ext_fp32(input, dim, dtype): + return log_softmax_ext(input.astype(mindspore.float32), dim, dtype) + +__all__.append('log_softmax_ext_fp32') + +def one_hot_ext(tensor, num_classes): + on_value = core.Tensor(1, dtype=tensor.dtype) + off_value = core.Tensor(0, dtype=tensor.dtype) + + return pyboost_inner_prim.one_hot_ext_impl(tensor, num_classes, on_value, off_value, -1) + +__all__.append('one_hot_ext') diff --git a/mindnlp/core/_tensor.py b/mindnlp/core/_tensor.py index 17f056207..b90a7504c 100644 --- a/mindnlp/core/_tensor.py +++ b/mindnlp/core/_tensor.py @@ -338,8 +338,8 @@ def __bool__(self): return bool(self.item()) def __index__(self): - if self.ndim > 0: - return self.tolist() + # if self.ndim > 0: + # return self.tolist() return int(self.item()) def __and__(self, other): @@ -1216,7 +1216,7 @@ def hardshrink(self, lambd=0.5): # Tensor.index_add_ def index_add_(self, dim, index, source, *, alpha=1): - return self.copy_(ops.index_add(self, dim, source, alpha=alpha)) + return self.copy_(ops.index_add(self, dim, index, source, alpha=alpha)) # Tensor.index_add def index_add(self, dim, index, source, *, alpha=1): diff --git a/mindnlp/core/amp/autocast_mode.py b/mindnlp/core/amp/autocast_mode.py index 5716167f4..ff922e0d6 100644 --- a/mindnlp/core/amp/autocast_mode.py +++ b/mindnlp/core/amp/autocast_mode.py @@ -10,6 +10,7 @@ from mindspore.common.dtype import TensorType as _dtype, float32 from mindspore.train.amp import AMP_AUTO_BLACK_LIST, AMP_AUTO_WHITE_LIST, AMP_PRIM_ARG_TABLE + try: import numpy as np @@ -48,6 +49,7 @@ def decorate_autocast(*args, **kwargs): return decorate_autocast + class autocast: def __init__( @@ -82,6 +84,7 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any): # type: ignore[ov def __call__(self, func): return autocast_decorator(self, func) + def _cast(value, device_type: str, dtype): if isinstance(value, core.Tensor): is_eligible = ( diff --git a/mindnlp/core/amp/grad_scaler.py b/mindnlp/core/amp/grad_scaler.py index 2a0fbddee..7a87a3119 100644 --- a/mindnlp/core/amp/grad_scaler.py +++ b/mindnlp/core/amp/grad_scaler.py @@ -21,9 +21,14 @@ def non_finite_check(inputs): status = core.tensor(np.array([0] * 8), dtype=core.int32, device='npu') status = core.depend(status, inputs) found_inf = core.npu_get_float_status_v2(status) + print('found_inf', found_inf) status = core.depend(status, found_inf) clear_status = core.npu_clear_float_status_v2(status) found_inf = core.depend(found_inf, clear_status) + print('found_inf', found_inf) + found_inf = core.not_equal(found_inf, 0) + print('found_inf', found_inf) + print('after clear', core.npu_get_float_status_v2(status)) return found_inf.sum() found_inf = core.all_finite(inputs) # pylint: disable=invalid-unary-operand-type @@ -302,7 +307,9 @@ def _unscale_grads_( # per_device_inv_scale.get(device), # ) found_inf = per_device_found_inf.get(device) + print('found_inf before', found_inf) found_inf.copy_(non_finite_check(grads).to(found_inf.dtype)) + print('found_inf after', found_inf) for grad in grads: grad *= per_device_inv_scale.get(device) @@ -372,6 +379,7 @@ def _maybe_opt_step( **kwargs: Any, ) -> Optional[float]: retval: Optional[float] = None + print(sum(v.item() for v in optimizer_state["found_inf_per_device"].values())) if not sum(v.item() for v in optimizer_state["found_inf_per_device"].values()): retval = optimizer.step(*args, **kwargs) return retval @@ -418,7 +426,8 @@ def step( ) retval: Optional[float] = None - + print('grad scaler') + print(getattr(optimizer, "_step_supports_amp_scaling", False)) if getattr(optimizer, "_step_supports_amp_scaling", False): # This optimizer has customized scale-handling logic, so we can call optimizer.step() directly. # The contract with custom optimizers is that their step() should accept an additional, @@ -478,6 +487,7 @@ def step( len(optimizer_state["found_inf_per_device"]) > 0 ), "No inf checks were recorded for this optimizer." + print('_maybe_opt_step') retval = self._maybe_opt_step(optimizer, optimizer_state, *args, **kwargs) optimizer_state["stage"] = OptState.STEPPED @@ -514,6 +524,8 @@ def update(self, new_scale: Optional[Union[float, core.Tensor]] = None) -> None: _scale, _growth_tracker = self._check_scale_growth_tracker("update") + print('scaler update') + print('scaler update', new_scale is not None) if new_scale is not None: assert self._scale is not None # Accept a new user-defined scale. @@ -551,7 +563,10 @@ def update(self, new_scale: Optional[Union[float, core.Tensor]] = None) -> None: # self._growth_interval, # ) if found_inf_combined > 0: + print(_scale) _scale.copy_(_scale * self._backoff_factor) + print(_scale) + _growth_tracker.copy_(_growth_tracker * 0) else: successful = self._growth_interval + 1 diff --git a/mindnlp/core/autograd/function.py b/mindnlp/core/autograd/function.py index d1b37c8f4..8735afc1d 100644 --- a/mindnlp/core/autograd/function.py +++ b/mindnlp/core/autograd/function.py @@ -43,6 +43,7 @@ def value_and_grad_f(*args, **kwargs): _pynative_executor.set_grad_flag(True) _pynative_executor.new_graph(fn_, *args, **kwargs) values = fn_(*args, **kwargs) + print('all_finite forward', mindspore.amp.all_finite(values)) _pynative_executor.end_graph(fn_, values, *args, **kwargs) run_args = args @@ -51,6 +52,7 @@ def value_and_grad_f(*args, **kwargs): grads = _pynative_executor.check_run(grad_, fn_, params_or_argnums, None, *run_args) grads = _pynative_executor.grad(fn_, grad_, params_or_argnums, None, *run_args) + # print('all_finite backward', mindspore.amp.all_finite(grads)) if attach_grads: for param, grad in zip(params_or_argnums, grads): grad = core.tensor(grad, device=param.device) diff --git a/mindnlp/core/dispatcher.py b/mindnlp/core/dispatcher.py index 376f9535a..752870e29 100644 --- a/mindnlp/core/dispatcher.py +++ b/mindnlp/core/dispatcher.py @@ -2,9 +2,98 @@ from .types import device as device_ from ._prims import ascend, cpu, numpy, meta, ascend_310b from .configs import DEVICE_TARGET, CPU_USE_NUMPY_OP, SOC +from ._bind import is_autocast_enabled device_map = {"cpu": "CPU", "npu": "Ascend", "cuda": "GPU"} +""" +__matmul__, addbmm, addmm, addmv, addr, baddbmm, bmm, chain_matmul, multi_dot, +conv1d, conv2d, conv3d, conv_transpose1d, conv_transpose2d, conv_transpose3d, GRUCell, +linear, LSTMCell, matmul, mm, mv, prelu, RNNCell +""" +AMP_AUTO_WHITE_LIST = [ + "dense", + "matmul", + "addbmm", + "addmm", + "addmv", + "addr", + "baddbmm", + "bmm", + "chain_matmul", + "multi_dot", + "conv1d", + "conv2d", + "conv3d", + "conv_transpose1d", + "conv_transpose2d", + "conv_transpose3d", + "mm", + "mv", + "prelu", +] + + +""" +__pow__, __rdiv__, __rpow__, __rtruediv__, acos, asin, binary_cross_entropy_with_logits, +cosh, cosine_embedding_loss, cdist, cosine_similarity, cross_entropy, +cumprod, cumsum, dist, erfinv, exp, expm1, group_norm, hinge_embedding_loss, +kl_div, l1_loss, layer_norm, log, log_softmax, log10, log1p, log2, margin_ranking_loss, mse_loss, +multilabel_margin_loss, multi_margin_loss, nll_loss, norm, normalize, pdist, poisson_nll_loss, +pow, prod, reciprocal, rsqrt, sinh, smooth_l1_loss, soft_margin_loss, softmax, softmin, softplus, +sum, renorm, tan, triplet_margin_loss +""" + +AMP_AUTO_BLACK_LIST = [ + 'acos', + 'asin', + 'binary_cross_entropy_with_logits', + 'cosh', + 'cosine_embedding_loss', + 'cdist', + 'cosine_similarity', + 'cross_entropy', + 'cumprod', + 'cumsum', + 'dist', + 'erfinv', + 'exp', + 'expm1', + 'group_norm', + 'hinge_embedding_loss', + 'kl_div', + 'l1_loss', + 'layer_norm', + 'log', + 'log_softmax', + 'log10', + 'log1p', + 'log2', + 'margin_ranking_loss', + 'mse_loss', + 'multilabel_margin_loss', + 'multi_margin_loss', + 'nll_loss', + 'norm', + 'normalize', + 'pdist', + 'poisson_nll_loss', + 'pow', + 'prod', + 'reciprocal', + 'rsqrt', + 'sinh', + 'smooth_l1_loss', + 'soft_margin_loss', + 'softmax', + 'softmin', + 'softplus', + 'sum', + 'renorm', + 'tan', + 'triplet_margin_loss', +] + class SingletonMeta(type): _instances = {} @@ -18,7 +107,7 @@ def __call__(cls, *args, **kwargs): class Dispatcher(metaclass=SingletonMeta): def __init__(self): - self._registry = {"cpu": {}, "npu": {}, "gpu": {}, 'numpy': {}, 'meta': {}} + self._registry = {"cpu": {}, "npu": {}, "gpu": {}, "numpy": {}, "meta": {}} def register(self, func_name, device, func): self._registry[device][func_name] = func @@ -46,13 +135,20 @@ def dispatch(self, func_name, *args, **kwargs): device = next(iter(devices), device_("cpu")) - if DEVICE_TARGET == 'Ascend' and device.type == 'cuda': - device.type = 'npu' + if DEVICE_TARGET == "Ascend" and device.type == "cuda": + device.type = "npu" device_type = device.type - if CPU_USE_NUMPY_OP and device_type == 'cpu': - device_type = 'numpy' + if CPU_USE_NUMPY_OP and device_type == "cpu": + device_type = "numpy" + + # if is_autocast_enabled(device_type): + # if func_name in AMP_AUTO_WHITE_LIST or func_name.replace('_ext', '') in AMP_AUTO_WHITE_LIST: + # func_name = func_name + "_fp16" + + # elif func_name in AMP_AUTO_BLACK_LIST or func_name.replace('_ext', '') in AMP_AUTO_BLACK_LIST: + # func_name = func_name + "_fp32" func = self._registry[device_type].get(func_name, None) if func is None: @@ -63,7 +159,7 @@ def dispatch(self, func_name, *args, **kwargs): dispatcher = Dispatcher() -if SOC == 'ascend310b': +if SOC == "ascend310b": for func_name in ascend_310b.__all__: dispatcher.register(func_name, "npu", getattr(ascend_310b, func_name)) else: @@ -78,4 +174,3 @@ def dispatch(self, func_name, *args, **kwargs): for func_name in meta.__all__: dispatcher.register(func_name, "meta", getattr(meta, func_name)) - diff --git a/mindnlp/core/nn/functional.py b/mindnlp/core/nn/functional.py index 68d132536..fa4668b92 100644 --- a/mindnlp/core/nn/functional.py +++ b/mindnlp/core/nn/functional.py @@ -1625,9 +1625,7 @@ def ctc_loss(log_probs, targets, input_lengths, target_lengths, blank=0, reducti return loss def one_hot(tensor, num_classes=-1): - if use_pyboost(): - return mint.nn.functional.one_hot(tensor, num_classes) - return ops.one_hot(tensor, num_classes) + return execute('one_hot_ext', tensor, num_classes) def pixel_shuffle(input, upscale_factor): return ops.pixel_shuffle(input, upscale_factor) diff --git a/mindnlp/core/npu/__init__.py b/mindnlp/core/npu/__init__.py index 0b55df9c8..afb40a047 100644 --- a/mindnlp/core/npu/__init__.py +++ b/mindnlp/core/npu/__init__.py @@ -8,6 +8,7 @@ memory_allocated as ms_memory_allocated, StreamCtx as StreamContext, Stream, empty_cache as ms_empty_cache, \ reset_peak_memory_stats, reset_max_memory_allocated, max_memory_allocated, synchronize, \ current_stream +from mindspore.hal import get_device_properties from mindspore.device_context.ascend import device_count as ms_device_count from mindspore.communication import GlobalComm, get_group_size diff --git a/mindnlp/core/ops/_inner.py b/mindnlp/core/ops/_inner.py index 4b59947e2..eb918cff8 100644 --- a/mindnlp/core/ops/_inner.py +++ b/mindnlp/core/ops/_inner.py @@ -16,5 +16,11 @@ def npu_get_float_status_v2(status): def npu_clear_float_status_v2(status): return execute('npu_clear_float_status_v2', status) +def all_finite(inputs): + return execute('all_finite', inputs) -__all__ = ['cast', 'assign', 'depend', 'npu_get_float_status_v2', 'npu_clear_float_status_v2'] +__all__ = [ + 'cast', 'assign', 'depend', + 'npu_get_float_status_v2', 'npu_clear_float_status_v2', + 'all_finite' +] diff --git a/mindnlp/core/ops/array.py b/mindnlp/core/ops/array.py index e7fdbd113..5ff97f758 100644 --- a/mindnlp/core/ops/array.py +++ b/mindnlp/core/ops/array.py @@ -78,7 +78,7 @@ def gather_nd(input, indices): # index_add def index_add(input, dim, index, source, *, alpha=1): - return execute("index_add_ext", input, index, source, dim, alpha) + return execute("index_add_ext", input, dim, index, source, alpha) # index_copy @@ -294,7 +294,9 @@ def unsqueeze(input, dim): # where -def where(condition, input, other): +def where(condition, input=None, other=None): + if input is None and other is None: + return nonzero(condition, as_tuple=True) return execute("select", condition, input, other) diff --git a/mindnlp/core/ops/inplace.py b/mindnlp/core/ops/inplace.py index ab6f40a5b..8076f4b92 100644 --- a/mindnlp/core/ops/inplace.py +++ b/mindnlp/core/ops/inplace.py @@ -1,3 +1,4 @@ +import numbers from mindnlp import core from mindnlp.core._C import default_generator from mindnlp.core.executor import execute @@ -64,6 +65,8 @@ def inplace_uniform(input, *args, **kwargs): return input def inplace_add(input, other, alpha): + if isinstance(other, numbers.Number): + other = core.tensor(other, dtype=input.dtype, device=input.device) execute('inplace_add_ext', input, other, alpha) return input diff --git a/mindnlp/core/ops/pointwise.py b/mindnlp/core/ops/pointwise.py index 9dfb61b53..d85c57e36 100644 --- a/mindnlp/core/ops/pointwise.py +++ b/mindnlp/core/ops/pointwise.py @@ -45,12 +45,12 @@ def add(input, other, *, alpha=1): # addcdiv def addcdiv(input, tensor1, tensor2, *, value=1): - return execute("addcdiv", input, tensor1, tensor2, value) + return execute("addcdiv_ext", input, tensor1, tensor2, value) # addcmul def addcmul(input, tensor1, tensor2, *, value=1): - return execute("addcmul", input, tensor1, tensor2, value) + return execute("addcmul_ext", input, tensor1, tensor2, value) # angle diff --git a/mindnlp/core/optim/adam.py b/mindnlp/core/optim/adam.py index 8dcabd462..32c57492d 100644 --- a/mindnlp/core/optim/adam.py +++ b/mindnlp/core/optim/adam.py @@ -209,11 +209,11 @@ def step(self, closure=None): closure (Callable, optional): A closure that reevaluates the model and returns the loss. """ - loss = None if closure is not None: with core.enable_grad(): loss = closure() + for group in self.param_groups: params_with_grad: list[Tensor] = [] grads: list[Tensor] = [] diff --git a/mindnlp/core/optim/lr_scheduler.py b/mindnlp/core/optim/lr_scheduler.py index dcd5cc629..3fbe614b8 100644 --- a/mindnlp/core/optim/lr_scheduler.py +++ b/mindnlp/core/optim/lr_scheduler.py @@ -126,11 +126,12 @@ def wrap_step(step_fn): opt_ref = ref(self.optimizer) func = step_fn.__func__ - # @wraps(func) + @wraps(func) def wrapper(*args, **kwargs): opt = opt_ref() opt._opt_called = True # type: ignore[union-attr] - return func.__get__(opt, opt.__class__)(*args, **kwargs) + out = func.__get__(opt, opt.__class__)(*args, **kwargs) + return out wrapper._wrapped_by_lr_sched = True # type: ignore[attr-defined] return wrapper diff --git a/mindnlp/core/optim/optimizer.py b/mindnlp/core/optim/optimizer.py index 8654ec0e8..4e5961032 100644 --- a/mindnlp/core/optim/optimizer.py +++ b/mindnlp/core/optim/optimizer.py @@ -385,17 +385,17 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> R: f"{func} must return None or a tuple of (new_args, new_kwargs), but got {result}." ) - out = func(*args, **kwargs) - self._optimizer_step_code() + out = func(*args, **kwargs) + self._optimizer_step_code() - # call optimizer step post hooks - for post_hook in chain( - self._optimizer_step_post_hooks.values(), - _global_optimizer_post_hooks.values(), - ): - post_hook(self, args, kwargs) + # call optimizer step post hooks + for post_hook in chain( + self._optimizer_step_post_hooks.values(), + _global_optimizer_post_hooks.values(), + ): + post_hook(self, args, kwargs) - return out + return out return wrapper diff --git a/mindnlp/transformers/trainer.py b/mindnlp/transformers/trainer.py index 178e6ab3b..2bb469525 100644 --- a/mindnlp/transformers/trainer.py +++ b/mindnlp/transformers/trainer.py @@ -85,10 +85,12 @@ def forward_fn(inputs, num_items_in_batch): else: loss.backward(**kwargs) + print(loss, loss_true) return loss, loss_true - grad_fn = autograd.value_and_grad(forward_fn, model.trainable_params(), has_aux=True) + if not hasattr(self, 'grad_fn'): + self.grad_fn = autograd.value_and_grad(forward_fn, model.trainable_params(), has_aux=True) - loss_scaled, (loss_true,) = grad_fn(inputs, num_items_in_batch) + loss_scaled, (loss_true,) = self.grad_fn(inputs, num_items_in_batch) return loss_true