chatglm2-sagemaker-fulltune.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f25d98fb-9a1e-4f98-b3fd-10b8ae565583",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 1. 准备环境(仅新建笔记本实例时初始化使用)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "751bc8af-7e6d-443c-9ee3-b4fd42575282",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.1 准备Python SDK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7794a87-f631-4925-bb2d-a8355225a097",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install --upgrade boto3\n",
    "!pip install --upgrade sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee05fcfc-89a1-45fb-85ea-1681a1316a77",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install huggingface_hub"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ff91a8e-44e6-47e3-8eb5-1a458476d28f",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.2 克隆ChatGLM2代码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "44baf3b5-8eed-4388-8ffe-005d53a3fa92",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning into 'ChatGLM2-6B'...\n"
     ]
    }
   ],
   "source": [
    "%%script bash\n",
    "rm -rf ChatGLM2-6B\n",
    "git clone https://github.com/THUDM/ChatGLM2-6B.git"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "924a0559-b1fe-4c90-9d6c-722ea2ffe839",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.3 准备s5cmd\n",
    "除了给笔记本实例使用外，还要提供给训练实例使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "41aedda3-1e27-4091-bf34-9310c1fccbbe",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "rm: cannot remove ‘/usr/local/bin/s5cmd’: No such file or directory\n",
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
      "100 4425k  100 4425k    0     0  7742k      0 --:--:-- --:--:-- --:--:-- 7742k\n"
     ]
    }
   ],
   "source": [
    "%%script bash\n",
    "sudo rm /usr/local/bin/s5cmd\n",
    "mkdir s5cmd_download\n",
    "cd s5cmd_download\n",
    "curl -L https://github.com/peak/s5cmd/releases/download/v2.1.0/s5cmd_2.1.0_Linux-64bit.tar.gz | tar -xz\n",
    "chmod 777 ./s5cmd\n",
    "cp ./s5cmd ../ChatGLM2-6B/ptuning/\n",
    "chmod 777 ./s5cmd\n",
    "cp ./s5cmd ../code\n",
    "sudo mv s5cmd /usr/local/bin/\n",
    "cd ..\n",
    "rm -rf s5cmd_download"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3fde6af-3414-478f-be27-6e28203c36c9",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.4 获取Runtime资源配置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9dfe8c4c-cb83-4306-a823-3b0eb119b47e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n",
      "Sagemaker Execution Role:\n",
      "arn:aws:iam::687912291502:role/service-role/AmazonSageMaker-ExecutionRole-20211013T113123\n",
      "Sagemaker Default Bucket:\n",
      "sagemaker-us-west-2-687912291502\n",
      "Sagemaker Boto Account:\n",
      "687912291502\n",
      "Sagemaker Boto Region:\n",
      "us-west-2\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "from sagemaker import get_execution_role\n",
    "\n",
    "sess = sagemaker.Session()\n",
    "role = get_execution_role()\n",
    "sagemaker_default_bucket = sess.default_bucket()\n",
    "\n",
    "account = sess.boto_session.client(\"sts\").get_caller_identity()[\"Account\"]\n",
    "region = sess.boto_session.region_name\n",
    "\n",
    "print(\"Sagemaker Execution Role:\")\n",
    "print(role)\n",
    "print(\"Sagemaker Default Bucket:\")\n",
    "print(sagemaker_default_bucket)\n",
    "print(\"Sagemaker Boto Account:\")\n",
    "print(account)\n",
    "print(\"Sagemaker Boto Region:\")\n",
    "print(region)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d1790e2-5800-411d-8c6c-af26618670a7",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 1.5 下载ChatGLM2原始模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3acd1f6f-6172-4326-8390-3f3a75361898",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.013374090194702148,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Fetching 15 files",
       "rate": null,
       "total": 15,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "058d7d43ba604f1d85c9db429ef32f0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 15 files:   0%|          | 0/15 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.012126684188842773,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)l-00002-of-00007.bin",
       "rate": null,
       "total": 1968299005,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a8d02b45ecb46d9a7f7e01fc00817b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00002-of-00007.bin:   0%|          | 0.00/1.97G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.020511627197265625,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)l-00001-of-00007.bin",
       "rate": null,
       "total": 1827780615,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bd8317130ea4d2a925982260b150aa8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00001-of-00007.bin:   0%|          | 0.00/1.83G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.0076563358306884766,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)l-00005-of-00007.bin",
       "rate": null,
       "total": 1968299069,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13d603ecdc674a179100426d03af4366",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00005-of-00007.bin:   0%|          | 0.00/1.97G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.011175155639648438,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)l-00003-of-00007.bin",
       "rate": null,
       "total": 1927414561,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3f84b1d032e44f9a1d00ac12e77b972",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00003-of-00007.bin:   0%|          | 0.00/1.93G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.010100841522216797,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)l-00004-of-00007.bin",
       "rate": null,
       "total": 1815225523,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d90acd8a62ca4708a0cdcb9039233be7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00004-of-00007.bin:   0%|          | 0.00/1.82G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.02055048942565918,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)/modeling_chatglm.py",
       "rate": null,
       "total": 50717,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6271b4fd1ca41f6b73d4d2198f42d8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/modeling_chatglm.py:   0%|          | 0.00/50.7k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.01530146598815918,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)2620ce9f/config.json",
       "rate": null,
       "total": 1223,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91054aa58ec34d389518a7d1763e3b7f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)2620ce9f/config.json:   0%|          | 0.00/1.22k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.012979745864868164,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)iguration_chatglm.py",
       "rate": null,
       "total": 2246,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a66826e24d1b4fd38c166f53373b4155",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)iguration_chatglm.py:   0%|          | 0.00/2.25k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.0077168941497802734,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)l-00006-of-00007.bin",
       "rate": null,
       "total": 1927414561,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20acd7bca77f4f128331cc9a3a7ac5a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00006-of-00007.bin:   0%|          | 0.00/1.93G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.00623631477355957,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)l-00007-of-00007.bin",
       "rate": null,
       "total": 1052808067,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a612629fa14b4fd792d681bd9ab9b443",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00007-of-00007.bin:   0%|          | 0.00/1.05G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.009282827377319336,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)model.bin.index.json",
       "rate": null,
       "total": 20438,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bdcabf2dfdbf4690a3ff16d81c81550f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)model.bin.index.json:   0%|          | 0.00/20.4k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.00754547119140625,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)ce9f/quantization.py",
       "rate": null,
       "total": 14692,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3e06f7355924b67a40ffc6d9118f265",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)ce9f/quantization.py:   0%|          | 0.00/14.7k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.006563663482666016,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)enization_chatglm.py",
       "rate": null,
       "total": 10061,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef7d90671eb44b6f95efb8807cf3c2e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)enization_chatglm.py:   0%|          | 0.00/10.1k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.007886648178100586,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading tokenizer.model",
       "rate": null,
       "total": 1018370,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79700ee796114761a3766750d8ce3428",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading tokenizer.model:   0%|          | 0.00/1.02M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.009075164794921875,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)okenizer_config.json",
       "rate": null,
       "total": 244,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a5b8d1d5818417081eeea094031a42c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/244 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "\n",
    "local_cache_path = Path(\"./model\")\n",
    "local_cache_path.mkdir(exist_ok=True)\n",
    "\n",
    "model_name = \"THUDM/chatglm2-6b\"\n",
    "\n",
    "# Only download pytorch checkpoint files\n",
    "allow_patterns = [\"*.json\", \"*.pt\", \"*.bin\", \"*.model\", \"*.py\"]\n",
    "\n",
    "model_download_path = snapshot_download(\n",
    "    repo_id=model_name,\n",
    "    cache_dir=local_cache_path,\n",
    "    allow_patterns=allow_patterns,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "42c0ce80-883b-46bf-a642-e8f69a5fdaab",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/\n"
     ]
    }
   ],
   "source": [
    "# Get the model files path\n",
    "import os\n",
    "from glob import glob\n",
    "\n",
    "local_model_path = None\n",
    "\n",
    "paths = os.walk(r'./model')\n",
    "for root, dirs, files in paths:\n",
    "    for file in files:\n",
    "        if file == 'config.json':\n",
    "            # print(os.path.join(root, file))\n",
    "            local_model_path = str(os.path.join(root, file))[0:-11]\n",
    "            print(local_model_path)\n",
    "if local_model_path == None:\n",
    "    print(\"Model download may failed, please check prior step!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2fa337d-f4ab-4182-884a-0d1b36c7e8c6",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.6 将ChatGLM2原始模型拷贝到 S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "827f7e34-dd87-4f69-9206-6dda7093037c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cp s3://sagemaker-us-west-2-687912291502/llm/datasets/chatglm2/KuaFuTrainData/dev.json dev.json\n",
      "cp s3://sagemaker-us-west-2-687912291502/llm/datasets/chatglm2/KuaFuTrainData/train.json train.json\n"
     ]
    }
   ],
   "source": [
    "!s5cmd sync s3://sagemaker-us-west-2-687912291502/llm/datasets/chatglm2/KuaFuTrainData/* ./"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "47937a52-ca54-4041-b2da-b3f02735d6a9",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/pytorch_model.bin.index.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model.bin.index.json\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/tokenization_chatglm.py s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/tokenization_chatglm.py\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/configuration_chatglm.py s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/configuration_chatglm.py\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/tokenizer.model s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/tokenizer.model\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/config.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/config.json\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/quantization.py s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/quantization.py\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/tokenizer_config.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/tokenizer_config.json\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/modeling_chatglm.py s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/modeling_chatglm.py\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/pytorch_model-00007-of-00007.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00007-of-00007.bin\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/pytorch_model-00004-of-00007.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00004-of-00007.bin\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/pytorch_model-00001-of-00007.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00001-of-00007.bin\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/pytorch_model-00002-of-00007.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00002-of-00007.bin\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/pytorch_model-00003-of-00007.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00003-of-00007.bin\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/pytorch_model-00006-of-00007.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00006-of-00007.bin\n",
      "cp model/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f/pytorch_model-00005-of-00007.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00005-of-00007.bin\n"
     ]
    }
   ],
   "source": [
    "%%script env sagemaker_default_bucket=$sagemaker_default_bucket local_model_path=$local_model_path bash\n",
    "\n",
    "s5cmd sync ${local_model_path} s3://${sagemaker_default_bucket}/llm/models/chatglm2/original-6B/\n",
    "\n",
    "# rm -rf model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23a16e22-ad1a-44b3-a174-b0f6a8a96c80",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# 2. 准备数据集"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19d9a1eb-f83f-4b18-9a0b-ee83adbbb621",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 2.1 准备数据目录\n",
    "\n",
    "夸父健康数据集:根据输入,输出广告词,如下所示:\n",
    "```JSON\n",
    "{\n",
    "  \"content\":\"需求:完成用户的运动建议的意见修改|数据信息:增加核心训练次数;参考信息:undefined\",\n",
    "  \"summary\":\"根据您的修改意见，我增加了核心训练次数，以帮助您更好地改善体态异常。\"\n",
    "}\n",
    "```\n",
    "建立`KuaFuTrainData` 目录放到本目录下，并上传 `train.json` 与 `dev.json`。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "41c50a39-37e9-4308-b7c0-66693862b729",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘KuaFuTrainData’: File exists\n"
     ]
    }
   ],
   "source": [
    "# 创建文件夹\n",
    "!mkdir KuaFuTrainData\n",
    "\n",
    "# 自行上传数据"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53a7c26c-952d-4c07-83e5-646efed110fb",
   "metadata": {},
   "source": [
    "## 2.2 将数据集到S3\n",
    "也可以忽略2.1与2.2，在本地直接将数据上传至 s3://${sagemaker_default_bucket}/llm/datasets/chatglm2/KuaFuTrainData/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "088797c9-5995-4c6e-bded-d373e58cad80",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%script env sagemaker_default_bucket=$sagemaker_default_bucket bash\n",
    "\n",
    "s5cmd sync ./KuaFuTrainData/ s3://${sagemaker_default_bucket}/llm/datasets/chatglm2/KuaFuTrainData/\n",
    "\n",
    "# rm -rf KuaFuTrainData"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "800caa83-69d8-433b-980a-23cd2e6cb50b",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 3. 开始微调模型准备"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7052d461-6f2b-497e-83cc-d90517465e4f",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 3.1 准备微调代码\n",
    "### 复写 ChatGLM2-6B/ptuning/arguments.py 文件\n",
    "与原始文件相比，增加了 ```model_output_s3_path``` 参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "87c80278-69f6-42c8-a15e-22e64d240b76",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ChatGLM2-6B/ptuning/arguments.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile ChatGLM2-6B/ptuning/arguments.py\n",
    "\n",
    "from dataclasses import dataclass, field\n",
    "from typing import Optional\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class ModelArguments:\n",
    "    \"\"\"\n",
    "    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.\n",
    "    \"\"\"\n",
    "        \n",
    "    model_output_s3_path: str = field(\n",
    "        metadata={\"help\": \"Path to model saved in s3 path using s5cmd utily\"}\n",
    "    )\n",
    "    \n",
    "        \n",
    "    model_name_or_path: str = field(\n",
    "        metadata={\"help\": \"Path to pretrained model or model identifier from huggingface.co/models\"}\n",
    "    )\n",
    "    ptuning_checkpoint: str = field(\n",
    "        default=None, metadata={\"help\": \"Path to p-tuning v2 checkpoints\"}\n",
    "    )\n",
    "    config_name: Optional[str] = field(\n",
    "        default=None, metadata={\"help\": \"Pretrained config name or path if not the same as model_name\"}\n",
    "    )\n",
    "    tokenizer_name: Optional[str] = field(\n",
    "        default=None, metadata={\"help\": \"Pretrained tokenizer name or path if not the same as model_name\"}\n",
    "    )\n",
    "    cache_dir: Optional[str] = field(\n",
    "        default=None,\n",
    "        metadata={\"help\": \"Where to store the pretrained models downloaded from huggingface.co\"},\n",
    "    )\n",
    "    use_fast_tokenizer: bool = field(\n",
    "        default=True,\n",
    "        metadata={\"help\": \"Whether to use one of the fast tokenizer (backed by the tokenizers library) or not.\"},\n",
    "    )\n",
    "    model_revision: str = field(\n",
    "        default=\"main\",\n",
    "        metadata={\"help\": \"The specific model version to use (can be a branch name, tag name or commit id).\"},\n",
    "    )\n",
    "    use_auth_token: bool = field(\n",
    "        default=False,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"Will use the token generated when running `huggingface-cli login` (necessary to use this script \"\n",
    "                \"with private models).\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    resize_position_embeddings: Optional[bool] = field(\n",
    "        default=None,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"Whether to automatically resize the position embeddings if `max_source_length` exceeds \"\n",
    "                \"the model's position embeddings.\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    quantization_bit: Optional[int] = field(\n",
    "        default=None\n",
    "    )\n",
    "    pre_seq_len: Optional[int] = field(\n",
    "        default=None\n",
    "    )\n",
    "    prefix_projection: bool = field(\n",
    "        default=False\n",
    "    )\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class DataTrainingArguments:\n",
    "    \"\"\"\n",
    "    Arguments pertaining to what data we are going to input our model for training and eval.\n",
    "    \"\"\"\n",
    "\n",
    "    lang: Optional[str] = field(default=None, metadata={\"help\": \"Language id for summarization.\"})\n",
    "\n",
    "    dataset_name: Optional[str] = field(\n",
    "        default=None, metadata={\"help\": \"The name of the dataset to use (via the datasets library).\"}\n",
    "    )\n",
    "    dataset_config_name: Optional[str] = field(\n",
    "        default=None, metadata={\"help\": \"The configuration name of the dataset to use (via the datasets library).\"}\n",
    "    )\n",
    "    prompt_column: Optional[str] = field(\n",
    "        default=None,\n",
    "        metadata={\"help\": \"The name of the column in the datasets containing the full texts (for summarization).\"},\n",
    "    )\n",
    "    response_column: Optional[str] = field(\n",
    "        default=None,\n",
    "        metadata={\"help\": \"The name of the column in the datasets containing the summaries (for summarization).\"},\n",
    "    )\n",
    "    history_column: Optional[str] = field(\n",
    "        default=None,\n",
    "        metadata={\"help\": \"The name of the column in the datasets containing the history of chat.\"},\n",
    "    )\n",
    "    train_file: Optional[str] = field(\n",
    "        default=None, metadata={\"help\": \"The input training data file (a jsonlines or csv file).\"}\n",
    "    )\n",
    "    validation_file: Optional[str] = field(\n",
    "        default=None,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"An optional input evaluation data file to evaluate the metrics (rouge) on (a jsonlines or csv file).\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    test_file: Optional[str] = field(\n",
    "        default=None,\n",
    "        metadata={\n",
    "            \"help\": \"An optional input test data file to evaluate the metrics (rouge) on (a jsonlines or csv file).\"\n",
    "        },\n",
    "    )\n",
    "    overwrite_cache: bool = field(\n",
    "        default=False, metadata={\"help\": \"Overwrite the cached training and evaluation sets\"}\n",
    "    )\n",
    "    train_simple: bool = field(\n",
    "        default=False, metadata={\"help\": \"whether use single node single GPU fine tuning\"}\n",
    "    )\n",
    "    train_mutipl: bool = field(\n",
    "        default=False, metadata={\"help\": \"whether use mutiple node mutiple GPU fine tuning\"}\n",
    "    )\n",
    "    preprocessing_num_workers: Optional[int] = field(\n",
    "        default=None,\n",
    "        metadata={\"help\": \"The number of processes to use for the preprocessing.\"},\n",
    "    )\n",
    "    max_source_length: Optional[int] = field(\n",
    "        default=1024,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"The maximum total input sequence length after tokenization. Sequences longer \"\n",
    "                \"than this will be truncated, sequences shorter will be padded.\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    max_target_length: Optional[int] = field(\n",
    "        default=128,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"The maximum total sequence length for target text after tokenization. Sequences longer \"\n",
    "                \"than this will be truncated, sequences shorter will be padded.\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    val_max_target_length: Optional[int] = field(\n",
    "        default=None,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"The maximum total sequence length for validation target text after tokenization. Sequences longer \"\n",
    "                \"than this will be truncated, sequences shorter will be padded. Will default to `max_target_length`.\"\n",
    "                \"This argument is also used to override the ``max_length`` param of ``model.generate``, which is used \"\n",
    "                \"during ``evaluate`` and ``predict``.\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    pad_to_max_length: bool = field(\n",
    "        default=False,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"Whether to pad all samples to model maximum sentence length. \"\n",
    "                \"If False, will pad the samples dynamically when batching to the maximum length in the batch. More \"\n",
    "                \"efficient on GPU but very bad for TPU.\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    max_train_samples: Optional[int] = field(\n",
    "        default=None,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"For debugging purposes or quicker training, truncate the number of training examples to this \"\n",
    "                \"value if set.\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    max_eval_samples: Optional[int] = field(\n",
    "        default=None,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"For debugging purposes or quicker training, truncate the number of evaluation examples to this \"\n",
    "                \"value if set.\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    max_predict_samples: Optional[int] = field(\n",
    "        default=None,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"For debugging purposes or quicker training, truncate the number of prediction examples to this \"\n",
    "                \"value if set.\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    num_beams: Optional[int] = field(\n",
    "        default=None,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"Number of beams to use for evaluation. This argument will be passed to ``model.generate``, \"\n",
    "                \"which is used during ``evaluate`` and ``predict``.\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "    ignore_pad_token_for_loss: bool = field(\n",
    "        default=True,\n",
    "        metadata={\n",
    "            \"help\": \"Whether to ignore the tokens corresponding to padded labels in the loss computation or not.\"\n",
    "        },\n",
    "    )\n",
    "    source_prefix: Optional[str] = field(\n",
    "        default=\"\", metadata={\"help\": \"A prefix to add before every source text (useful for T5 models).\"}\n",
    "    )\n",
    "\n",
    "    forced_bos_token: Optional[str] = field(\n",
    "        default=None,\n",
    "        metadata={\n",
    "            \"help\": (\n",
    "                \"The token to force as the first generated token after the decoder_start_token_id.\"\n",
    "                \"Useful for multilingual models like mBART where the first generated token\"\n",
    "                \"needs to be the target language token (Usually it is the target language token)\"\n",
    "            )\n",
    "        },\n",
    "    )\n",
    "\n",
    "    \n",
    "\n",
    "    def __post_init__(self):\n",
    "        if self.dataset_name is None and self.train_file is None and self.validation_file is None and self.test_file is None:\n",
    "            raise ValueError(\"Need either a dataset name or a training/validation/test file.\")\n",
    "        else:\n",
    "            if self.train_file is not None:\n",
    "                extension = self.train_file.split(\".\")[-1]\n",
    "                assert extension in [\"csv\", \"json\"], \"`train_file` should be a csv or a json file.\"\n",
    "            if self.validation_file is not None:\n",
    "                extension = self.validation_file.split(\".\")[-1]\n",
    "                assert extension in [\"csv\", \"json\"], \"`validation_file` should be a csv or a json file.\"\n",
    "        if self.val_max_target_length is None:\n",
    "            self.val_max_target_length = self.max_target_length"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a792579b-930e-479a-9f4a-4316c80808aa",
   "metadata": {},
   "source": [
    "### 增加 ChatGLM2-6B/ptuning/main_tuning.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7489ee67-d15f-4d03-a846-8caaced2d428",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ChatGLM2-6B/ptuning/main_tuning.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile ChatGLM2-6B/ptuning/main_tuning.py\n",
    "\n",
    "#!/usr/bin/env python\n",
    "# coding=utf-8\n",
    "# Copyright 2021 The HuggingFace Team. All rights reserved.\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License.\n",
    "\"\"\"\n",
    "Fine-tuning the library models for sequence to sequence.\n",
    "\"\"\"\n",
    "# You can also adapt this script on your own sequence to sequence task. Pointers for this are left as comments.\n",
    "\n",
    "import logging\n",
    "import os\n",
    "import sys\n",
    "import json\n",
    "\n",
    "import numpy as np\n",
    "from datasets import load_dataset\n",
    "import jieba \n",
    "from rouge_chinese import Rouge\n",
    "from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction\n",
    "import torch\n",
    "import deepspeed\n",
    "import torch.distributed as dist\n",
    "\n",
    "import transformers\n",
    "from transformers import (\n",
    "    AutoConfig,\n",
    "    AutoModel,\n",
    "    AutoTokenizer,\n",
    "    DataCollatorForSeq2Seq,\n",
    "    HfArgumentParser,\n",
    "    Seq2SeqTrainingArguments,\n",
    "    set_seed,\n",
    ")\n",
    "from trainer_seq2seq import Seq2SeqTrainer\n",
    "\n",
    "from arguments import ModelArguments, DataTrainingArguments\n",
    "\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "def main():\n",
    "    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, Seq2SeqTrainingArguments))\n",
    "    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n",
    "        # If we pass only one argument to the script and it's the path to a json file,\n",
    "        # let's parse it to get our arguments.\n",
    "        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n",
    "    else:\n",
    "        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n",
    "        \n",
    "    ## if mutil-node train , set torch.distribute initial ###########\n",
    "    #if data_args.train_mutipl:\n",
    "    #    # Environment variables set by torch.distributed.launch\n",
    "    #    LOCAL_RANK = int(os.environ['LOCAL_RANK'])\n",
    "    #    WORLD_SIZE = int(os.environ['WORLD_SIZE'])\n",
    "    #    WORLD_RANK = int(os.environ['RANK'])\n",
    "    #    \n",
    "    #    dist.init_process_group(backend='nccl', rank=WORLD_RANK, world_size=WORLD_SIZE)\n",
    "\n",
    "    # Setup logging\n",
    "    logging.basicConfig(\n",
    "        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n",
    "        datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
    "        handlers=[logging.StreamHandler(sys.stdout)],\n",
    "    )\n",
    "\n",
    "    if training_args.should_log:\n",
    "        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n",
    "        transformers.utils.logging.set_verbosity_info()\n",
    "\n",
    "    log_level = training_args.get_process_log_level()\n",
    "    logger.setLevel(log_level)\n",
    "    # datasets.utils.logging.set_verbosity(log_level)\n",
    "    transformers.utils.logging.set_verbosity(log_level)\n",
    "    transformers.utils.logging.enable_default_handler()\n",
    "    transformers.utils.logging.enable_explicit_format()\n",
    "\n",
    "    # Log on each process the small summary:\n",
    "    logger.warning(\n",
    "        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}\"\n",
    "        + f\"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}\"\n",
    "    )\n",
    "    logger.info(f\"Training/evaluation parameters {training_args}\")\n",
    "\n",
    "    # Set seed before initializing model.\n",
    "    set_seed(training_args.seed)\n",
    "\n",
    "    # Load dataset\n",
    "    data_files = {}\n",
    "    if data_args.train_file is not None:\n",
    "        data_files[\"train\"] = data_args.train_file\n",
    "        extension = data_args.train_file.split(\".\")[-1]\n",
    "    if data_args.validation_file is not None:\n",
    "        data_files[\"validation\"] = data_args.validation_file\n",
    "        extension = data_args.validation_file.split(\".\")[-1]\n",
    "    if data_args.test_file is not None:\n",
    "        data_files[\"test\"] = data_args.test_file\n",
    "        extension = data_args.test_file.split(\".\")[-1]\n",
    "\n",
    "    raw_datasets = load_dataset(\n",
    "        extension,\n",
    "        data_files=data_files,\n",
    "        cache_dir=model_args.cache_dir,\n",
    "        #use_auth_token=True if model_args.use_auth_token else None,\n",
    "    )\n",
    "    \n",
    "    # if s3 path model, use s5cmd to download the model to /tmp/orignal/ for model load\n",
    "    if model_args.model_name_or_path.startswith(\"s3\"):\n",
    "        #Note: if deepspeed fine tuning ,we just use the rank 0 process to download the model assets to S3 by s5cmd command.\n",
    "        if data_args.train_simple == False:\n",
    "            WORLD_RANK = int(os.environ['RANK'])\n",
    "            print(\"world_rank===\"+str(WORLD_RANK))\n",
    "            if WORLD_RANK == 0:\n",
    "                os.system(\"cp ./s5cmd  /tmp/ && chmod +x /tmp/s5cmd\")\t\n",
    "                os.system(\"/tmp/s5cmd sync {0} {1}\".format(model_args.model_name_or_path + \"*\", \"/tmp/orignal/\"))\n",
    "\n",
    "                print(\"sync s3 model \" + model_args.model_name_or_path + \"to /tmp/orignal:\")\n",
    "                model_args.model_name_or_path = \"/tmp/orignal/\"\n",
    "                print(os.listdir(model_args.model_name_or_path))\n",
    "            else: \n",
    "                model_args.model_name_or_path = \"/tmp/orignal/\"\n",
    "            #Note: we should sync with every ranker and ensure rank 0 uploading the model assets successfully. \n",
    "            torch.distributed.barrier()\n",
    "        else:\n",
    "            os.system(\"cp ./s5cmd  /tmp/ && chmod +x /tmp/s5cmd\")\t\n",
    "            os.system(\"/tmp/s5cmd sync {0} {1}\".format(model_args.model_name_or_path + \"*\", \"/tmp/orignal/\"))\n",
    "\n",
    "            print(\"sync s3 model \" + model_args.model_name_or_path + \"to /tmp/orignal:\")\n",
    "            model_args.model_name_or_path = \"/tmp/orignal/\"\n",
    "            print(os.listdir(model_args.model_name_or_path))\n",
    "\n",
    "    # Load pretrained model and tokenizer\n",
    "    config = AutoConfig.from_pretrained(model_args.model_name_or_path, trust_remote_code=True)\n",
    "    config.pre_seq_len = model_args.pre_seq_len\n",
    "    config.prefix_projection = model_args.prefix_projection\n",
    "\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, trust_remote_code=True)\n",
    "\n",
    "    if model_args.ptuning_checkpoint is not None:\n",
    "        # Evaluation\n",
    "        # Loading extra state dict of prefix encoder\n",
    "        model = AutoModel.from_pretrained(model_args.model_name_or_path, config=config, trust_remote_code=True)\n",
    "        prefix_state_dict = torch.load(os.path.join(model_args.ptuning_checkpoint, \"pytorch_model.bin\"))\n",
    "        new_prefix_state_dict = {}\n",
    "        for k, v in prefix_state_dict.items():\n",
    "            if k.startswith(\"transformer.prefix_encoder.\"):\n",
    "                new_prefix_state_dict[k[len(\"transformer.prefix_encoder.\"):]] = v\n",
    "        model.transformer.prefix_encoder.load_state_dict(new_prefix_state_dict)\n",
    "    else:\n",
    "        model = AutoModel.from_pretrained(model_args.model_name_or_path, config=config, trust_remote_code=True)\n",
    "        #model = AutoModel.from_pretrained(model_args.model_name_or_path, trust_remote_code=True)\n",
    "\n",
    "    if model_args.quantization_bit is not None:\n",
    "        print(f\"Quantized to {model_args.quantization_bit} bit\")\n",
    "        model = model.quantize(model_args.quantization_bit)\n",
    "    if model_args.pre_seq_len is not None:\n",
    "        # P-tuning v2\n",
    "        model = model.half()\n",
    "        model.transformer.prefix_encoder.float()\n",
    "    else:\n",
    "        # Finetune\n",
    "        model = model.float()\n",
    "\n",
    "    prefix = data_args.source_prefix if data_args.source_prefix is not None else \"\"\n",
    "\n",
    "    # Preprocessing the datasets.\n",
    "    # We need to tokenize inputs and targets.\n",
    "    if training_args.do_train:\n",
    "        column_names = raw_datasets[\"train\"].column_names\n",
    "    elif training_args.do_eval:\n",
    "        column_names = raw_datasets[\"validation\"].column_names\n",
    "    elif training_args.do_predict:\n",
    "        column_names = raw_datasets[\"test\"].column_names\n",
    "    else:\n",
    "        logger.info(\"There is nothing to do. Please pass `do_train`, `do_eval` and/or `do_predict`.\")\n",
    "        return\n",
    "\n",
    "    # Get the column names for input/target.\n",
    "    prompt_column = data_args.prompt_column\n",
    "    response_column = data_args.response_column\n",
    "    history_column = data_args.history_column\n",
    "    \n",
    "    # Temporarily set max_target_length for training.\n",
    "    max_target_length = data_args.max_target_length\n",
    "\n",
    "    def preprocess_function_eval(examples):\n",
    "        inputs, targets = [], []\n",
    "        for i in range(len(examples[prompt_column])):\n",
    "            if examples[prompt_column][i] and examples[response_column][i]:\n",
    "                query = examples[prompt_column][i]\n",
    "                history = examples[history_column][i] if history_column is not None else None\n",
    "                prompt = tokenizer.build_prompt(query, history)\n",
    "                inputs.append(prompt)\n",
    "                targets.append(examples[response_column][i])\n",
    "\n",
    "        inputs = [prefix + inp for inp in inputs]\n",
    "        model_inputs = tokenizer(inputs, max_length=data_args.max_source_length, truncation=True, padding=True)\n",
    "        labels = tokenizer(text_target=targets, max_length=max_target_length, truncation=True)\n",
    "\n",
    "        if data_args.ignore_pad_token_for_loss:\n",
    "            labels[\"input_ids\"] = [\n",
    "                [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels[\"input_ids\"]\n",
    "            ]\n",
    "        model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "\n",
    "        return model_inputs\n",
    "\n",
    "    def preprocess_function_train(examples):\n",
    "        max_seq_length = data_args.max_source_length + data_args.max_target_length + 1\n",
    "\n",
    "        model_inputs = {\n",
    "            \"input_ids\": [],\n",
    "            \"labels\": [],\n",
    "        }\n",
    "        for i in range(len(examples[prompt_column])):\n",
    "            if examples[prompt_column][i] and examples[response_column][i]:\n",
    "                query, answer = examples[prompt_column][i], examples[response_column][i]\n",
    "\n",
    "                history = examples[history_column][i] if history_column is not None else None\n",
    "                prompt = tokenizer.build_prompt(query, history)\n",
    "\n",
    "                prompt = prefix + prompt\n",
    "                a_ids = tokenizer.encode(text=prompt, add_special_tokens=True, truncation=True,\n",
    "                                         max_length=data_args.max_source_length)\n",
    "                b_ids = tokenizer.encode(text=answer, add_special_tokens=False, truncation=True,\n",
    "                                         max_length=data_args.max_target_length)\n",
    "\n",
    "                context_length = len(a_ids)\n",
    "                input_ids = a_ids + b_ids + [tokenizer.eos_token_id]\n",
    "                labels = [tokenizer.pad_token_id] * context_length + b_ids + [tokenizer.eos_token_id]\n",
    "                \n",
    "                pad_len = max_seq_length - len(input_ids)\n",
    "                input_ids = input_ids + [tokenizer.pad_token_id] * pad_len\n",
    "                labels = labels + [tokenizer.pad_token_id] * pad_len\n",
    "                if data_args.ignore_pad_token_for_loss:\n",
    "                    labels = [(l if l != tokenizer.pad_token_id else -100) for l in labels]\n",
    "\n",
    "                model_inputs[\"input_ids\"].append(input_ids)\n",
    "                model_inputs[\"labels\"].append(labels)\n",
    "\n",
    "        return model_inputs\n",
    "    \n",
    "    def print_dataset_example(example):\n",
    "        print(\"input_ids\", example[\"input_ids\"])\n",
    "        print(\"inputs\", tokenizer.decode(example[\"input_ids\"]))\n",
    "        print(\"label_ids\", example[\"labels\"])\n",
    "        print(\"labels\", tokenizer.decode(example[\"labels\"]))\n",
    "\n",
    "    if training_args.do_train:\n",
    "        if \"train\" not in raw_datasets:\n",
    "            raise ValueError(\"--do_train requires a train dataset\")\n",
    "        train_dataset = raw_datasets[\"train\"]\n",
    "        if data_args.max_train_samples is not None:\n",
    "            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n",
    "            train_dataset = train_dataset.select(range(max_train_samples))\n",
    "        with training_args.main_process_first(desc=\"train dataset map pre-processing\"):\n",
    "            train_dataset = train_dataset.map(\n",
    "                preprocess_function_train,\n",
    "                batched=True,\n",
    "                num_proc=data_args.preprocessing_num_workers,\n",
    "                remove_columns=column_names,\n",
    "                load_from_cache_file=not data_args.overwrite_cache,\n",
    "                desc=\"Running tokenizer on train dataset\",\n",
    "            )\n",
    "        print_dataset_example(train_dataset[0])\n",
    "\n",
    "    if training_args.do_eval:\n",
    "        max_target_length = data_args.val_max_target_length\n",
    "        if \"validation\" not in raw_datasets:\n",
    "            raise ValueError(\"--do_eval requires a validation dataset\")\n",
    "        eval_dataset = raw_datasets[\"validation\"]\n",
    "        if data_args.max_eval_samples is not None:\n",
    "            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)\n",
    "            eval_dataset = eval_dataset.select(range(max_eval_samples))\n",
    "        with training_args.main_process_first(desc=\"validation dataset map pre-processing\"):\n",
    "            eval_dataset = eval_dataset.map(\n",
    "                preprocess_function_eval,\n",
    "                batched=True,\n",
    "                num_proc=data_args.preprocessing_num_workers,\n",
    "                remove_columns=column_names,\n",
    "                load_from_cache_file=not data_args.overwrite_cache,\n",
    "                desc=\"Running tokenizer on validation dataset\",\n",
    "            )\n",
    "        print_dataset_example(eval_dataset[0])\n",
    "\n",
    "    if training_args.do_predict:\n",
    "        max_target_length = data_args.val_max_target_length\n",
    "        if \"test\" not in raw_datasets:\n",
    "            raise ValueError(\"--do_predict requires a test dataset\")\n",
    "        predict_dataset = raw_datasets[\"test\"]\n",
    "        if data_args.max_predict_samples is not None:\n",
    "            max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples)\n",
    "            predict_dataset = predict_dataset.select(range(max_predict_samples))\n",
    "        with training_args.main_process_first(desc=\"prediction dataset map pre-processing\"):\n",
    "            predict_dataset = predict_dataset.map(\n",
    "                preprocess_function_eval,\n",
    "                batched=True,\n",
    "                num_proc=data_args.preprocessing_num_workers,\n",
    "                remove_columns=column_names,\n",
    "                load_from_cache_file=not data_args.overwrite_cache,\n",
    "                desc=\"Running tokenizer on prediction dataset\",\n",
    "            )\n",
    "        print_dataset_example(predict_dataset[0])\n",
    "\n",
    "    # Data collator\n",
    "    label_pad_token_id = -100 if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id\n",
    "    data_collator = DataCollatorForSeq2Seq(\n",
    "        tokenizer,\n",
    "        model=model,\n",
    "        label_pad_token_id=label_pad_token_id,\n",
    "        pad_to_multiple_of=None,\n",
    "        padding=False\n",
    "    )\n",
    "\n",
    "    # Metric\n",
    "    def compute_metrics(eval_preds):\n",
    "        preds, labels = eval_preds\n",
    "        if isinstance(preds, tuple):\n",
    "            preds = preds[0]\n",
    "        decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n",
    "        if data_args.ignore_pad_token_for_loss:\n",
    "            # Replace -100 in the labels as we can't decode them.\n",
    "            labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
    "        decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
    "\n",
    "        score_dict = {\n",
    "            \"rouge-1\": [],\n",
    "            \"rouge-2\": [],\n",
    "            \"rouge-l\": [],\n",
    "            \"bleu-4\": []\n",
    "        }\n",
    "        for pred, label in zip(decoded_preds, decoded_labels):\n",
    "            hypothesis = list(jieba.cut(pred))\n",
    "            reference = list(jieba.cut(label))\n",
    "            rouge = Rouge()\n",
    "            scores = rouge.get_scores(' '.join(hypothesis) , ' '.join(reference))\n",
    "            result = scores[0]\n",
    "            \n",
    "            for k, v in result.items():\n",
    "                score_dict[k].append(round(v[\"f\"] * 100, 4))\n",
    "            bleu_score = sentence_bleu([list(label)], list(pred), smoothing_function=SmoothingFunction().method3)\n",
    "            score_dict[\"bleu-4\"].append(round(bleu_score * 100, 4))\n",
    "\n",
    "        for k, v in score_dict.items():\n",
    "            score_dict[k] = float(np.mean(v))\n",
    "        return score_dict\n",
    "\n",
    "    # Override the decoding parameters of Seq2SeqTrainer\n",
    "    training_args.generation_max_length = (\n",
    "        training_args.generation_max_length\n",
    "        if training_args.generation_max_length is not None\n",
    "        else data_args.val_max_target_length\n",
    "    )\n",
    "    training_args.generation_num_beams = (\n",
    "        data_args.num_beams if data_args.num_beams is not None else training_args.generation_num_beams\n",
    "    )\n",
    "    # Initialize our Trainer\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model=model,\n",
    "        args=training_args,\n",
    "        train_dataset=train_dataset if training_args.do_train else None,\n",
    "        eval_dataset=eval_dataset if training_args.do_eval else None,\n",
    "        tokenizer=tokenizer,\n",
    "        data_collator=data_collator,\n",
    "        compute_metrics=compute_metrics if training_args.predict_with_generate else None,\n",
    "        save_changed=model_args.pre_seq_len is not None\n",
    "    )\n",
    "\n",
    "    # Training\n",
    "    if training_args.do_train:\n",
    "        checkpoint = None\n",
    "        if training_args.resume_from_checkpoint is not None:\n",
    "            checkpoint = training_args.resume_from_checkpoint\n",
    "        # elif last_checkpoint is not None:\n",
    "        #     checkpoint = last_checkpoint\n",
    "        model.gradient_checkpointing_enable()\n",
    "        model.enable_input_require_grads()\n",
    "        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
    "        # trainer.save_model()  # Saves the tokenizer too for easy upload\n",
    "\n",
    "        metrics = train_result.metrics\n",
    "        max_train_samples = (\n",
    "            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)\n",
    "        )\n",
    "        metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n",
    "\n",
    "        trainer.log_metrics(\"train\", metrics)\n",
    "        trainer.save_metrics(\"train\", metrics)\n",
    "        trainer.save_state()\n",
    "        \n",
    "        print(\"------saving model!-----\")\t\n",
    "        \t\n",
    "        save_model_dir = os.environ['OUTPUT_DIR']\t\n",
    "        tokenizer.save_pretrained(save_model_dir)\t\n",
    "        trainer.save_model(save_model_dir)\t\n",
    "        print(\"save_model_dir : {}\".format(save_model_dir))\t\n",
    "        print(\"------model is saved!-----\")\n",
    "        \n",
    "        print(\"model sync to : {}\".format(os.environ['MODEL_OUTPUT_S3_PATH']))\n",
    "        \n",
    "        #Note: if deepspeed fine tuning ,we just use the rank 0 process to upload the trained model assets to S3 by s5cmd command.\n",
    "        if data_args.train_simple == False:\n",
    "            WORLD_RANK = int(os.environ['RANK'])\n",
    "            print(\"world_rank===\"+str(WORLD_RANK))\n",
    "            if WORLD_RANK == 0:\n",
    "                os.system(\"./s5cmd sync {0} {1}\".format(save_model_dir, os.environ['MODEL_OUTPUT_S3_PATH']))\n",
    "            #Note: we should sync with every ranker and ensure rank 0 uploading the model assets successfully. \n",
    "            torch.distributed.barrier()\n",
    "        else:\n",
    "            os.system(\"./s5cmd sync {0} {1}\".format(save_model_dir, os.environ['MODEL_OUTPUT_S3_PATH']))\n",
    "\n",
    "        \n",
    "\n",
    "    # Evaluation\n",
    "    results = {}\n",
    "    max_seq_length = data_args.max_source_length + data_args.max_target_length + 1\n",
    "    if training_args.do_eval:\n",
    "        logger.info(\"*** Evaluate ***\")\n",
    "        metrics = trainer.evaluate(metric_key_prefix=\"eval\", do_sample=True, top_p=0.7, max_length=max_seq_length, temperature=0.95)\n",
    "        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)\n",
    "        metrics[\"eval_samples\"] = min(max_eval_samples, len(eval_dataset))\n",
    "\n",
    "        trainer.log_metrics(\"eval\", metrics)\n",
    "        trainer.save_metrics(\"eval\", metrics)\n",
    "\n",
    "    if training_args.do_predict:\n",
    "        logger.info(\"*** Predict ***\")\n",
    "        predict_results = trainer.predict(predict_dataset, metric_key_prefix=\"predict\", max_length=max_seq_length, do_sample=True, top_p=0.7, temperature=0.95)\n",
    "        metrics = predict_results.metrics\n",
    "        max_predict_samples = (\n",
    "            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)\n",
    "        )\n",
    "        metrics[\"predict_samples\"] = min(max_predict_samples, len(predict_dataset))\n",
    "\n",
    "        trainer.log_metrics(\"predict\", metrics)\n",
    "        trainer.save_metrics(\"predict\", metrics)\n",
    "\n",
    "        if trainer.is_world_process_zero():\n",
    "            if training_args.predict_with_generate:\n",
    "                predictions = tokenizer.batch_decode(\n",
    "                    predict_results.predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True\n",
    "                )\n",
    "                predictions = [pred.strip() for pred in predictions]\n",
    "                labels = tokenizer.batch_decode(\n",
    "                    predict_results.label_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True\n",
    "                )\n",
    "                labels = [label.strip() for label in labels]\n",
    "                output_prediction_file = os.path.join(training_args.output_dir, \"generated_predictions.txt\")\n",
    "                with open(output_prediction_file, \"w\", encoding=\"utf-8\") as writer:\n",
    "                    for p, l in zip(predictions, labels):\n",
    "                        res = json.dumps({\"labels\": l, \"predict\": p}, ensure_ascii=False)\n",
    "                        writer.write(f\"{res}\\n\")\n",
    "    return results\n",
    "\n",
    "\n",
    "def _mp_fn(index):\n",
    "    # For xla_spawn (TPUs)\n",
    "    main()\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29d0dc95-b7fb-45a7-a0b8-32bb29fb50bf",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 增加 ChatGLM2-6B/ptuning/deepspeed.json 文件\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76f6e541-072a-48d7-a89b-fdee398213cf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile ChatGLM2-6B/ptuning/deepspeed.json\n",
    "{\n",
    "  \"train_micro_batch_size_per_gpu\": \"auto\",\n",
    "  \"zero_allow_untested_optimizer\": true,\n",
    "  \"fp16\": {\n",
    "    \"enabled\": \"auto\",\n",
    "    \"loss_scale\": 0,\n",
    "    \"initial_scale_power\": 16,\n",
    "    \"loss_scale_window\": 1000,\n",
    "    \"hysteresis\": 2,\n",
    "    \"min_loss_scale\": 1\n",
    "  },\n",
    "  \"optimizer\": {\n",
    "    \"type\": \"Adam\",\n",
    "    \"params\": {\n",
    "      \"lr\": \"auto\",\n",
    "      \"betas\": \"auto\",\n",
    "      \"eps\": \"auto\",\n",
    "      \"weight_decay\": \"auto\"\n",
    "    }\n",
    "  },\n",
    "  \"scheduler\": {\n",
    "    \"type\": \"WarmupLR\",\n",
    "    \"params\": {\n",
    "      \"warmup_min_lr\": \"auto\",\n",
    "      \"warmup_max_lr\": \"auto\",\n",
    "      \"warmup_num_steps\": \"auto\"\n",
    "    }\n",
    "  },\n",
    "  \"zero_optimization\": {\n",
    "    \"stage\": 2,\n",
    "    \"allgather_partitions\": true,\n",
    "    \"allgather_bucket_size\": 5e8,\n",
    "    \"overlap_comm\": false,\n",
    "    \"reduce_scatter\": true,\n",
    "    \"reduce_bucket_size\": 5e8,\n",
    "    \"contiguous_gradients\" : true\n",
    "  }\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5feb7b01-a789-45ab-9214-4d5dd32593bd",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 增加 ChatGLM2-6B/ptuning/sm_fulltuning_train.sh 文件\n",
    "官方 LR=1e-4  \n",
    "CUDA_VISIBLE_DEVICES 为GPU参数  \n",
    "参考以下文件：  \n",
    "CChatGLM2-6B/ptuning/ds_train_finetune.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3666cd75-cdbf-4572-b5a8-85c9ce156ba7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ChatGLM2-6B/ptuning/sm_fulltuning_train.sh\n"
     ]
    }
   ],
   "source": [
    "%%writefile ChatGLM2-6B/ptuning/sm_fulltuning_train.sh\n",
    "\n",
    "LR=1e-4\n",
    "MASTER_PORT=\"23456\"\n",
    "\n",
    "deepspeed --num_gpus=$NUM_GPUS  --master_port $MASTER_PORT main_tuning.py \\\n",
    "    --deepspeed deepspeed.json \\\n",
    "    --do_train \\\n",
    "    --train_file $TRAIN_DATASET \\\n",
    "    --validation_file $TEST_DATASET \\\n",
    "    --prompt_column ${PROMPT_COLUMN} \\\n",
    "    --response_column ${RESPONSE_COLUMN} \\\n",
    "    --overwrite_cache \\\n",
    "    --model_name_or_path ${MODEL_NAME_OR_PATH}   \\\n",
    "    --output_dir ${OUTPUT_DIR} \\\n",
    "    --model_output_s3_path ${MODEL_OUTPUT_S3_PATH} \\\n",
    "    --overwrite_output_dir \\\n",
    "    --max_source_length 64 \\\n",
    "    --max_target_length 64 \\\n",
    "    --per_device_train_batch_size 1 \\\n",
    "    --per_device_eval_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 1 \\\n",
    "    --predict_with_generate \\\n",
    "    --max_steps ${TRAIN_STEPS} \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps ${TRAIN_STEPS} \\\n",
    "    --learning_rate $LR \\\n",
    "    --fp16"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "511a51f1-2b91-44cd-bbbc-e85f3b33b86c",
   "metadata": {},
   "source": [
    "### 增加 sm_fulltune_train.py 文件\n",
    "此文件为启动训练的 Entry Point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ee01918e-50b5-4738-9059-8c6375de5619",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing ChatGLM2-6B/ptuning/sm_fulltune_train.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile ChatGLM2-6B/ptuning/sm_fulltune_train.py\n",
    "\n",
    "import os\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    os.environ['PYTORCH_CUDA_ALLOC_CONF'] = str(os.environ['PYTORCH_CUDA_ALLOC_CONF'])\n",
    "    os.environ['LD_LIBRARY_PATH'] = str(os.environ['LD_LIBRARY_PATH'])\n",
    "    os.environ['TRAIN_DATASET'] = str(os.environ['TRAIN_DATASET'])\n",
    "    os.environ['TEST_DATASET'] = str(os.environ['TEST_DATASET'])\n",
    "    os.environ['PROMPT_COLUMN'] = str(os.environ['PROMPT_COLUMN'])\n",
    "    os.environ['RESPONSE_COLUMN'] = str(os.environ['RESPONSE_COLUMN'])\n",
    "    os.environ['MODEL_NAME_OR_PATH'] = str(os.environ['MODEL_NAME_OR_PATH'])\n",
    "    os.environ['OUTPUT_DIR'] = str(os.environ['OUTPUT_DIR'])\n",
    "    os.environ['MODEL_OUTPUT_S3_PATH'] = str(os.environ['MODEL_OUTPUT_S3_PATH'])\n",
    "\n",
    "    # os.system(\"chmod +x ./s5cmd\")\n",
    "    os.system(\"/bin/bash sm_fulltuning_train.sh\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f30c3df5-341d-4fe3-9b1c-6b7acbf3d39b",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 增加 ChatGLM2-6B/ptuning/requirements.txt 文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "bc67f35a-28d4-4f38-bb65-196b188feda4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ChatGLM2-6B/ptuning/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile ChatGLM2-6B/ptuning/requirements.txt\n",
    "\n",
    "protobuf\n",
    "#git+https://github.com/huggingface/transformers.git@68d640f7c368bcaaaecfc678f11908ebbd3d6176\n",
    "transformers==4.28.1\n",
    "cpm_kernels\n",
    "#torch>=2.0\n",
    "gradio\n",
    "mdtex2html\n",
    "sentencepiece\n",
    "accelerate>=0.20.1\n",
    "sse-starlette\n",
    "streamlit>=1.24.0\n",
    "datasets\n",
    "huggingface\n",
    "jieba\n",
    "rouge_chinese\n",
    "nltk\n",
    "deepspeed==0.9.5"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b5548d6-dd07-4044-b86a-8e48eba67cfa",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 3.2 获取Runtime资源配置(未执行 1.4 可执行这里)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0f49d710-968f-433f-ae08-f1b0c247a75b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sagemaker Execution Role:\n",
      "arn:aws:iam::687912291502:role/service-role/AmazonSageMaker-ExecutionRole-20211013T113123\n",
      "Sagemaker Default Bucket:\n",
      "sagemaker-us-west-2-687912291502\n",
      "Sagemaker Boto Account:\n",
      "687912291502\n",
      "Sagemaker Boto Region:\n",
      "us-west-2\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "from sagemaker import get_execution_role\n",
    "\n",
    "sess = sagemaker.Session()\n",
    "role = get_execution_role()\n",
    "sagemaker_default_bucket = sess.default_bucket()\n",
    "\n",
    "account = sess.boto_session.client(\"sts\").get_caller_identity()[\"Account\"]\n",
    "region = sess.boto_session.region_name\n",
    "\n",
    "print(\"Sagemaker Execution Role:\")\n",
    "print(role)\n",
    "print(\"Sagemaker Default Bucket:\")\n",
    "print(sagemaker_default_bucket)\n",
    "print(\"Sagemaker Boto Account:\")\n",
    "print(account)\n",
    "print(\"Sagemaker Boto Region:\")\n",
    "print(region)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37dfe385-35fa-4b47-8a79-3984cf4e51f4",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 3.2 Full-Tuning 训练\n",
    "### 定义微调参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3f14956c-693f-4544-b930-f25b9efa52b0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define Training Job Name\n",
    "import time\n",
    "from sagemaker.huggingface import HuggingFace\n",
    "\n",
    "job_name = f'huggingface-chatglm-finetune-fulltuning-{time.strftime(\"%Y-%m-%d-%H-%M-%S\", time.localtime())}'\n",
    "\n",
    "instance_type  = \"ml.p4d.24xlarge\"\n",
    "instance_count = 1\n",
    "processes_per_host = 8\n",
    "\n",
    "# 基础模型存放地址\n",
    "model_name_or_path = 's3://{}/llm/models/chatglm2/original-6B/'.format(sagemaker_default_bucket)\n",
    "\n",
    "# 微调模型输出地址\n",
    "output_dir         = '/opt/ml/model/adgen-chatglm2-6b-ft'\n",
    "model_s3_path      = 's3://{}/llm/models/chatglm2/finetune-fulltuning-adgen/'.format(sagemaker_default_bucket)\n",
    "\n",
    "# 模型环境变量设置\n",
    "environment = {\n",
    "    'PYTORCH_CUDA_ALLOC_CONF': 'max_split_size_mb:32',\n",
    "    'NUM_GPUS'               : str(processes_per_host),\n",
    "    'TRAIN_DATASET'          : '/opt/ml/input/data/KuaFuTrainData/train.json',\n",
    "    'TEST_DATASET'           : '/opt/ml/input/data/KuaFuTrainData/dev.json',\n",
    "    'PROMPT_COLUMN'          : 'content',\n",
    "    'RESPONSE_COLUMN'        : 'summary',\n",
    "    'MODEL_NAME_OR_PATH'     : model_name_or_path,\n",
    "    'OUTPUT_DIR'             : output_dir,\n",
    "    'MODEL_OUTPUT_S3_PATH'   : model_s3_path,\n",
    "    'TRAIN_STEPS'            : '50'\n",
    "}\n",
    "\n",
    "inputs = {\n",
    "   'KuaFuTrainData': f\"s3://{sagemaker_default_bucket}/llm/datasets/chatglm2/KuaFuTrainData/\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "9dc327f8-7bf6-499b-a0c1-cbf6ab90e994",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'PYTORCH_CUDA_ALLOC_CONF': 'max_split_size_mb:32',\n",
       " 'NUM_GPUS': '8',\n",
       " 'TRAIN_DATASET': '/opt/ml/input/data/KuaFuTrainData/train.json',\n",
       " 'TEST_DATASET': '/opt/ml/input/data/KuaFuTrainData/dev.json',\n",
       " 'PROMPT_COLUMN': 'content',\n",
       " 'RESPONSE_COLUMN': 'summary',\n",
       " 'MODEL_NAME_OR_PATH': 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/',\n",
       " 'OUTPUT_DIR': '/opt/ml/model/adgen-chatglm2-6b-ft',\n",
       " 'MODEL_OUTPUT_S3_PATH': 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/',\n",
       " 'TRAIN_STEPS': '50'}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "environment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18d79de1-2307-4824-9b7f-7734ce46afb2",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 启动微调训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "aaa9eab7-005d-454c-b88b-8996d67d7439",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# create the Estimator\n",
    "huggingface_estimator = HuggingFace(\n",
    "    entry_point          = 'sm_fulltune_train.py',\n",
    "    source_dir           = './ChatGLM2-6B/ptuning',\n",
    "    instance_type        = instance_type,\n",
    "    instance_count       = instance_count,\n",
    "    base_job_name        = job_name,\n",
    "    role                 = role,\n",
    "    script_mode          = True,\n",
    "    transformers_version = '4.28',\n",
    "    pytorch_version      = '2.0',\n",
    "    py_version           = 'py310',\n",
    "    environment          = environment\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "632dcb56-2e12-4b06-aad6-ab7560eb13c7",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n",
      "INFO:sagemaker:Creating training-job with name: huggingface-chatglm-finetune-fulltuning-2024-09-26-10-18-23-050\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-09-26 10:18:26 Starting - Starting the training job\n",
      "2024-09-26 10:18:26 Pending - Training job waiting for capacity......\n",
      "2024-09-26 10:19:09 Pending - Preparing the instances for training........................\n",
      "2024-09-26 10:23:28 Downloading - Downloading input data...\n",
      "2024-09-26 10:23:48 Downloading - Downloading the training image........................\n",
      "2024-09-26 10:27:46 Training - Training image download completed. Training in progress......\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2024-09-26 10:28:39,312 sagemaker-training-toolkit INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2024-09-26 10:28:39,427 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-09-26 10:28:39,434 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2024-09-26 10:28:39,436 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2024-09-26 10:28:41,021 sagemaker-training-toolkit INFO     Installing dependencies from requirements.txt:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.10 -m pip install -r requirements.txt\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: protobuf in /opt/conda/lib/python3.10/site-packages (from -r requirements.txt (line 1)) (3.20.2)\u001b[0m\n",
      "\u001b[34mCollecting transformers==4.30.1 (from -r requirements.txt (line 3))\u001b[0m\n",
      "\u001b[34mDownloading transformers-4.30.1-py3-none-any.whl.metadata (113 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 113.6/113.6 kB 4.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting cpm_kernels (from -r requirements.txt (line 4))\u001b[0m\n",
      "\u001b[34mDownloading cpm_kernels-1.0.11-py3-none-any.whl.metadata (1.2 kB)\u001b[0m\n",
      "\u001b[34mCollecting gradio (from -r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading gradio-4.44.0-py3-none-any.whl.metadata (15 kB)\u001b[0m\n",
      "\u001b[34mCollecting mdtex2html (from -r requirements.txt (line 7))\u001b[0m\n",
      "\u001b[34mDownloading mdtex2html-1.3.0-py3-none-any.whl.metadata (4.1 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.10/site-packages (from -r requirements.txt (line 8)) (0.1.99)\u001b[0m\n",
      "\u001b[34mCollecting accelerate>=0.20.1 (from -r requirements.txt (line 9))\u001b[0m\n",
      "\u001b[34mDownloading accelerate-0.34.2-py3-none-any.whl.metadata (19 kB)\u001b[0m\n",
      "\u001b[34mCollecting sse-starlette (from -r requirements.txt (line 10))\u001b[0m\n",
      "\u001b[34mDownloading sse_starlette-2.1.3-py3-none-any.whl.metadata (5.8 kB)\u001b[0m\n",
      "\u001b[34mCollecting streamlit>=1.24.0 (from -r requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading streamlit-1.38.0-py2.py3-none-any.whl.metadata (8.5 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (from -r requirements.txt (line 12)) (2.16.1)\u001b[0m\n",
      "\u001b[34mCollecting huggingface (from -r requirements.txt (line 13))\u001b[0m\n",
      "\u001b[34mDownloading huggingface-0.0.1-py3-none-any.whl.metadata (2.9 kB)\u001b[0m\n",
      "\u001b[34mCollecting jieba (from -r requirements.txt (line 14))\u001b[0m\n",
      "\u001b[34mDownloading jieba-0.42.1.tar.gz (19.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 19.2/19.2 MB 57.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting rouge_chinese (from -r requirements.txt (line 15))\u001b[0m\n",
      "\u001b[34mDownloading rouge_chinese-1.0.3-py3-none-any.whl.metadata (7.6 kB)\u001b[0m\n",
      "\u001b[34mCollecting nltk (from -r requirements.txt (line 16))\u001b[0m\n",
      "\u001b[34mDownloading nltk-3.9.1-py3-none-any.whl.metadata (2.9 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r requirements.txt (line 3)) (3.12.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r requirements.txt (line 3)) (0.20.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r requirements.txt (line 3)) (1.24.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r requirements.txt (line 3)) (23.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r requirements.txt (line 3)) (6.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r requirements.txt (line 3)) (2023.12.25)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r requirements.txt (line 3)) (2.31.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r requirements.txt (line 3)) (0.13.3)\u001b[0m\n",
      "\u001b[34mCollecting safetensors>=0.3.1 (from transformers==4.30.1->-r requirements.txt (line 3))\u001b[0m\n",
      "\u001b[34mDownloading safetensors-0.4.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r requirements.txt (line 3)) (4.65.0)\u001b[0m\n",
      "\u001b[34mCollecting aiofiles<24.0,>=22.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading aiofiles-23.2.1-py3-none-any.whl.metadata (9.7 kB)\u001b[0m\n",
      "\u001b[34mCollecting anyio<5.0,>=3.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading anyio-4.6.0-py3-none-any.whl.metadata (4.6 kB)\u001b[0m\n",
      "\u001b[34mCollecting fastapi<1.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading fastapi-0.115.0-py3-none-any.whl.metadata (27 kB)\u001b[0m\n",
      "\u001b[34mCollecting ffmpy (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading ffmpy-0.4.0-py3-none-any.whl.metadata (2.9 kB)\u001b[0m\n",
      "\u001b[34mCollecting gradio-client==1.3.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading gradio_client-1.3.0-py3-none-any.whl.metadata (7.1 kB)\u001b[0m\n",
      "\u001b[34mCollecting httpx>=0.24.1 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading httpx-0.27.2-py3-none-any.whl.metadata (7.1 kB)\u001b[0m\n",
      "\u001b[34mCollecting importlib-resources<7.0,>=1.3 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading importlib_resources-6.4.5-py3-none-any.whl.metadata (4.0 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jinja2<4.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r requirements.txt (line 6)) (3.1.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markupsafe~=2.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r requirements.txt (line 6)) (2.1.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: matplotlib~=3.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r requirements.txt (line 6)) (3.7.2)\u001b[0m\n",
      "\u001b[34mCollecting orjson~=3.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (50 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 50.4/50.4 kB 9.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pandas<3.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r requirements.txt (line 6)) (2.0.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pillow<11.0,>=8.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r requirements.txt (line 6)) (10.0.0)\u001b[0m\n",
      "\u001b[34mCollecting pydantic>=2.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading pydantic-2.9.2-py3-none-any.whl.metadata (149 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 149.4/149.4 kB 36.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting pydub (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading pydub-0.25.1-py2.py3-none-any.whl.metadata (1.4 kB)\u001b[0m\n",
      "\u001b[34mCollecting python-multipart>=0.0.9 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading python_multipart-0.0.10-py3-none-any.whl.metadata (1.9 kB)\u001b[0m\n",
      "\u001b[34mCollecting ruff>=0.2.2 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading ruff-0.6.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (25 kB)\u001b[0m\n",
      "\u001b[34mCollecting semantic-version~=2.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading semantic_version-2.10.0-py2.py3-none-any.whl.metadata (9.7 kB)\u001b[0m\n",
      "\u001b[34mCollecting tomlkit==0.12.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading tomlkit-0.12.0-py3-none-any.whl.metadata (2.7 kB)\u001b[0m\n",
      "\u001b[34mCollecting typer<1.0,>=0.12 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading typer-0.12.5-py3-none-any.whl.metadata (15 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typing-extensions~=4.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r requirements.txt (line 6)) (4.7.1)\u001b[0m\n",
      "\u001b[34mCollecting urllib3~=2.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading urllib3-2.2.3-py3-none-any.whl.metadata (6.5 kB)\u001b[0m\n",
      "\u001b[34mCollecting uvicorn>=0.14.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading uvicorn-0.30.6-py3-none-any.whl.metadata (6.6 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from gradio-client==1.3.0->gradio->-r requirements.txt (line 6)) (2023.6.0)\u001b[0m\n",
      "\u001b[34mCollecting websockets<13.0,>=10.0 (from gradio-client==1.3.0->gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markdown in /opt/conda/lib/python3.10/site-packages (from mdtex2html->-r requirements.txt (line 7)) (3.5.2)\u001b[0m\n",
      "\u001b[34mCollecting latex2mathml (from mdtex2html->-r requirements.txt (line 7))\u001b[0m\n",
      "\u001b[34mDownloading latex2mathml-3.77.0-py3-none-any.whl.metadata (14 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.20.1->-r requirements.txt (line 9)) (5.9.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.20.1->-r requirements.txt (line 9)) (2.0.0)\u001b[0m\n",
      "\u001b[34mCollecting huggingface-hub<1.0,>=0.14.1 (from transformers==4.30.1->-r requirements.txt (line 3))\u001b[0m\n",
      "\u001b[34mDownloading huggingface_hub-0.25.1-py3-none-any.whl.metadata (13 kB)\u001b[0m\n",
      "\u001b[34mCollecting starlette (from sse-starlette->-r requirements.txt (line 10))\u001b[0m\n",
      "\u001b[34mDownloading starlette-0.39.1-py3-none-any.whl.metadata (6.0 kB)\u001b[0m\n",
      "\u001b[34mCollecting altair<6,>=4.0 (from streamlit>=1.24.0->-r requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading altair-5.4.1-py3-none-any.whl.metadata (9.4 kB)\u001b[0m\n",
      "\u001b[34mCollecting blinker<2,>=1.0.0 (from streamlit>=1.24.0->-r requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading blinker-1.8.2-py3-none-any.whl.metadata (1.6 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cachetools<6,>=4.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r requirements.txt (line 11)) (5.3.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: click<9,>=7.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r requirements.txt (line 11)) (8.1.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyarrow>=7.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r requirements.txt (line 11)) (14.0.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: rich<14,>=10.14.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r requirements.txt (line 11)) (13.4.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tenacity<9,>=8.1.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r requirements.txt (line 11)) (8.2.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: toml<2,>=0.10.1 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r requirements.txt (line 11)) (0.10.2)\u001b[0m\n",
      "\u001b[34mCollecting gitpython!=3.1.19,<4,>=3.0.7 (from streamlit>=1.24.0->-r requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading GitPython-3.1.43-py3-none-any.whl.metadata (13 kB)\u001b[0m\n",
      "\u001b[34mCollecting pydeck<1,>=0.8.0b4 (from streamlit>=1.24.0->-r requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading pydeck-0.9.1-py2.py3-none-any.whl.metadata (4.1 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tornado<7,>=6.0.3 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r requirements.txt (line 11)) (6.4)\u001b[0m\n",
      "\u001b[34mCollecting watchdog<5,>=2.1.5 (from streamlit>=1.24.0->-r requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading watchdog-4.0.2-py3-none-manylinux2014_x86_64.whl.metadata (38 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyarrow-hotfix in /opt/conda/lib/python3.10/site-packages (from datasets->-r requirements.txt (line 12)) (0.6)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: dill<0.3.8,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets->-r requirements.txt (line 12)) (0.3.6)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets->-r requirements.txt (line 12)) (3.4.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets->-r requirements.txt (line 12)) (0.70.14)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets->-r requirements.txt (line 12)) (3.9.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from rouge_chinese->-r requirements.txt (line 15)) (1.16.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: joblib in /opt/conda/lib/python3.10/site-packages (from nltk->-r requirements.txt (line 16)) (1.3.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jsonschema>=3.0 in /opt/conda/lib/python3.10/site-packages (from altair<6,>=4.0->streamlit>=1.24.0->-r requirements.txt (line 11)) (4.18.3)\u001b[0m\n",
      "\u001b[34mCollecting narwhals>=1.5.2 (from altair<6,>=4.0->streamlit>=1.24.0->-r requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading narwhals-1.8.3-py3-none-any.whl.metadata (6.8 kB)\u001b[0m\n",
      "\u001b[34mCollecting typing-extensions~=4.0 (from gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: idna>=2.8 in /opt/conda/lib/python3.10/site-packages (from anyio<5.0,>=3.0->gradio->-r requirements.txt (line 6)) (3.4)\u001b[0m\n",
      "\u001b[34mCollecting sniffio>=1.1 (from anyio<5.0,>=3.0->gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading sniffio-1.3.1-py3-none-any.whl.metadata (3.9 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: exceptiongroup>=1.0.2 in /opt/conda/lib/python3.10/site-packages (from anyio<5.0,>=3.0->gradio->-r requirements.txt (line 6)) (1.2.0)\u001b[0m\n",
      "\u001b[34mCollecting starlette (from sse-starlette->-r requirements.txt (line 10))\u001b[0m\n",
      "\u001b[34mDownloading starlette-0.38.6-py3-none-any.whl.metadata (6.0 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r requirements.txt (line 12)) (1.3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r requirements.txt (line 12)) (23.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r requirements.txt (line 12)) (1.4.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r requirements.txt (line 12)) (6.0.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r requirements.txt (line 12)) (1.9.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r requirements.txt (line 12)) (4.0.3)\u001b[0m\n",
      "\u001b[34mCollecting gitdb<5,>=4.0.1 (from gitpython!=3.1.19,<4,>=3.0.7->streamlit>=1.24.0->-r requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading gitdb-4.0.11-py3-none-any.whl.metadata (1.2 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: certifi in /opt/conda/lib/python3.10/site-packages (from httpx>=0.24.1->gradio->-r requirements.txt (line 6)) (2024.2.2)\u001b[0m\n",
      "\u001b[34mCollecting httpcore==1.* (from httpx>=0.24.1->gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading httpcore-1.0.5-py3-none-any.whl.metadata (20 kB)\u001b[0m\n",
      "\u001b[34mCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx>=0.24.1->gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 6)) (1.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 6)) (0.11.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 6)) (4.47.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 6)) (1.4.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyparsing<3.1,>=2.3.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 6)) (3.0.9)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 6)) (2.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas<3.0,>=1.0->gradio->-r requirements.txt (line 6)) (2023.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas<3.0,>=1.0->gradio->-r requirements.txt (line 6)) (2023.3)\u001b[0m\n",
      "\u001b[34mCollecting annotated-types>=0.6.0 (from pydantic>=2.0->gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading annotated_types-0.7.0-py3-none-any.whl.metadata (15 kB)\u001b[0m\n",
      "\u001b[34mCollecting pydantic-core==2.23.4 (from pydantic>=2.0->gradio->-r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.30.1->-r requirements.txt (line 3)) (3.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markdown-it-py>=2.2.0 in /opt/conda/lib/python3.10/site-packages (from rich<14,>=10.14.0->streamlit>=1.24.0->-r requirements.txt (line 11)) (3.0.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from rich<14,>=10.14.0->streamlit>=1.24.0->-r requirements.txt (line 11)) (2.15.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate>=0.20.1->-r requirements.txt (line 9)) (1.12)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate>=0.20.1->-r requirements.txt (line 9)) (3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: shellingham>=1.3.0 in /opt/conda/lib/python3.10/site-packages (from typer<1.0,>=0.12->gradio->-r requirements.txt (line 6)) (1.5.1)\u001b[0m\n",
      "\u001b[34mCollecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=3.1.19,<4,>=3.0.7->streamlit>=1.24.0->-r requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading smmap-5.0.1-py3-none-any.whl.metadata (4.3 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit>=1.24.0->-r requirements.txt (line 11)) (2023.6.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: referencing>=0.28.4 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit>=1.24.0->-r requirements.txt (line 11)) (0.29.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: rpds-py>=0.7.1 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit>=1.24.0->-r requirements.txt (line 11)) (0.8.10)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: mdurl~=0.1 in /opt/conda/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich<14,>=10.14.0->streamlit>=1.24.0->-r requirements.txt (line 11)) (0.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate>=0.20.1->-r requirements.txt (line 9)) (1.3.0)\u001b[0m\n",
      "\u001b[34mDownloading transformers-4.30.1-py3-none-any.whl (7.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.2/7.2 MB 94.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading cpm_kernels-1.0.11-py3-none-any.whl (416 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 416.6/416.6 kB 75.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading gradio-4.44.0-py3-none-any.whl (18.1 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 18.1/18.1 MB 103.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading gradio_client-1.3.0-py3-none-any.whl (318 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 318.7/318.7 kB 62.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading tomlkit-0.12.0-py3-none-any.whl (37 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdtex2html-1.3.0-py3-none-any.whl (13 kB)\u001b[0m\n",
      "\u001b[34mDownloading accelerate-0.34.2-py3-none-any.whl (324 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 324.4/324.4 kB 67.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading sse_starlette-2.1.3-py3-none-any.whl (9.4 kB)\u001b[0m\n",
      "\u001b[34mDownloading streamlit-1.38.0-py2.py3-none-any.whl (8.7 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 8.7/8.7 MB 157.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading huggingface-0.0.1-py3-none-any.whl (2.5 kB)\u001b[0m\n",
      "\u001b[34mDownloading rouge_chinese-1.0.3-py3-none-any.whl (21 kB)\u001b[0m\n",
      "\u001b[34mDownloading nltk-3.9.1-py3-none-any.whl (1.5 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB 147.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\u001b[0m\n",
      "\u001b[34mDownloading altair-5.4.1-py3-none-any.whl (658 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 658.1/658.1 kB 101.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading anyio-4.6.0-py3-none-any.whl (89 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 89.6/89.6 kB 27.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading blinker-1.8.2-py3-none-any.whl (9.5 kB)\u001b[0m\n",
      "\u001b[34mDownloading fastapi-0.115.0-py3-none-any.whl (94 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 94.6/94.6 kB 27.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading GitPython-3.1.43-py3-none-any.whl (207 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 207.3/207.3 kB 50.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading httpx-0.27.2-py3-none-any.whl (76 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 76.4/76.4 kB 19.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading httpcore-1.0.5-py3-none-any.whl (77 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 77.9/77.9 kB 18.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading huggingface_hub-0.25.1-py3-none-any.whl (436 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 436.4/436.4 kB 76.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading importlib_resources-6.4.5-py3-none-any.whl (36 kB)\u001b[0m\n",
      "\u001b[34mDownloading orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (141 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 141.9/141.9 kB 35.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading pydantic-2.9.2-py3-none-any.whl (434 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 434.9/434.9 kB 73.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 146.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading pydeck-0.9.1-py2.py3-none-any.whl (6.9 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 6.9/6.9 MB 160.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading python_multipart-0.0.10-py3-none-any.whl (22 kB)\u001b[0m\n",
      "\u001b[34mDownloading ruff-0.6.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.8 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.8/10.8 MB 155.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading safetensors-0.4.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (435 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 435.0/435.0 kB 79.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\u001b[0m\n",
      "\u001b[34mDownloading starlette-0.38.6-py3-none-any.whl (71 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 71.5/71.5 kB 22.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading typer-0.12.5-py3-none-any.whl (47 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 47.3/47.3 kB 12.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading typing_extensions-4.12.2-py3-none-any.whl (37 kB)\u001b[0m\n",
      "\u001b[34mDownloading urllib3-2.2.3-py3-none-any.whl (126 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 126.3/126.3 kB 33.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading uvicorn-0.30.6-py3-none-any.whl (62 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.8/62.8 kB 17.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading watchdog-4.0.2-py3-none-manylinux2014_x86_64.whl (82 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 82.9/82.9 kB 24.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading ffmpy-0.4.0-py3-none-any.whl (5.8 kB)\u001b[0m\n",
      "\u001b[34mDownloading latex2mathml-3.77.0-py3-none-any.whl (73 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 73.7/73.7 kB 22.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\u001b[0m\n",
      "\u001b[34mDownloading annotated_types-0.7.0-py3-none-any.whl (13 kB)\u001b[0m\n",
      "\u001b[34mDownloading gitdb-4.0.11-py3-none-any.whl (62 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.7/62.7 kB 19.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading h11-0.14.0-py3-none-any.whl (58 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 58.3/58.3 kB 16.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading narwhals-1.8.3-py3-none-any.whl (169 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 169.1/169.1 kB 37.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading sniffio-1.3.1-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[34mDownloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 130.2/130.2 kB 35.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading smmap-5.0.1-py3-none-any.whl (24 kB)\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: jieba\u001b[0m\n",
      "\u001b[34mBuilding wheel for jieba (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for jieba (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for jieba: filename=jieba-0.42.1-py3-none-any.whl size=19314458 sha256=613db9159ab8949a90ba4d2cbbb3554e5f941e172c112ff51de062ed15822ccd\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/c9/69/31/d56d90b22a1777b0b231e234b00302a55be255930f8bd92dcd\u001b[0m\n",
      "\u001b[34mSuccessfully built jieba\u001b[0m\n",
      "\u001b[34mInstalling collected packages: pydub, jieba, huggingface, cpm_kernels, websockets, watchdog, urllib3, typing-extensions, tomlkit, sniffio, smmap, semantic-version, safetensors, ruff, rouge_chinese, python-multipart, orjson, nltk, narwhals, latex2mathml, importlib-resources, h11, ffmpy, blinker, annotated-types, aiofiles, uvicorn, pydeck, pydantic-core, mdtex2html, httpcore, gitdb, anyio, typer, starlette, pydantic, huggingface-hub, httpx, gitpython, transformers, sse-starlette, gradio-client, fastapi, altair, accelerate, streamlit, gradio\u001b[0m\n",
      "\u001b[34mAttempting uninstall: urllib3\u001b[0m\n",
      "\u001b[34mFound existing installation: urllib3 1.26.15\u001b[0m\n",
      "\u001b[34mUninstalling urllib3-1.26.15:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled urllib3-1.26.15\u001b[0m\n",
      "\u001b[34mAttempting uninstall: typing-extensions\u001b[0m\n",
      "\u001b[34mFound existing installation: typing_extensions 4.7.1\u001b[0m\n",
      "\u001b[34mUninstalling typing_extensions-4.7.1:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled typing_extensions-4.7.1\u001b[0m\n",
      "\u001b[34mAttempting uninstall: typer\u001b[0m\n",
      "\u001b[34mFound existing installation: typer 0.9.0\u001b[0m\n",
      "\u001b[34mUninstalling typer-0.9.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled typer-0.9.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: pydantic\u001b[0m\n",
      "\u001b[34mFound existing installation: pydantic 1.10.11\u001b[0m\n",
      "\u001b[34mUninstalling pydantic-1.10.11:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled pydantic-1.10.11\u001b[0m\n",
      "\u001b[34mAttempting uninstall: huggingface-hub\u001b[0m\n",
      "\u001b[34mFound existing installation: huggingface-hub 0.20.3\u001b[0m\n",
      "\u001b[34mUninstalling huggingface-hub-0.20.3:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled huggingface-hub-0.20.3\u001b[0m\n",
      "\u001b[34mAttempting uninstall: transformers\u001b[0m\n",
      "\u001b[34mFound existing installation: transformers 4.28.1\u001b[0m\n",
      "\u001b[34mUninstalling transformers-4.28.1:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled transformers-4.28.1\u001b[0m\n",
      "\u001b[34mAttempting uninstall: accelerate\u001b[0m\n",
      "\u001b[34mFound existing installation: accelerate 0.19.0\u001b[0m\n",
      "\u001b[34mUninstalling accelerate-0.19.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled accelerate-0.19.0\u001b[0m\n",
      "\u001b[34mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\u001b[0m\n",
      "\u001b[34mbotocore 1.34.34 requires urllib3<2.1,>=1.25.4; python_version >= \"3.10\", but you have urllib3 2.2.3 which is incompatible.\u001b[0m\n",
      "\u001b[34mconfection 0.1.0 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mspacy 3.6.0 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mspacy 3.6.0 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.5 which is incompatible.\u001b[0m\n",
      "\u001b[34mthinc 8.1.10 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mSuccessfully installed accelerate-0.34.2 aiofiles-23.2.1 altair-5.4.1 annotated-types-0.7.0 anyio-4.6.0 blinker-1.8.2 cpm_kernels-1.0.11 fastapi-0.115.0 ffmpy-0.4.0 gitdb-4.0.11 gitpython-3.1.43 gradio-4.44.0 gradio-client-1.3.0 h11-0.14.0 httpcore-1.0.5 httpx-0.27.2 huggingface-0.0.1 huggingface-hub-0.25.1 importlib-resources-6.4.5 jieba-0.42.1 latex2mathml-3.77.0 mdtex2html-1.3.0 narwhals-1.8.3 nltk-3.9.1 orjson-3.10.7 pydantic-2.9.2 pydantic-core-2.23.4 pydeck-0.9.1 pydub-0.25.1 python-multipart-0.0.10 rouge_chinese-1.0.3 ruff-0.6.7 safetensors-0.4.5 semantic-version-2.10.0 smmap-5.0.1 sniffio-1.3.1 sse-starlette-2.1.3 starlette-0.38.6 streamlit-1.38.0 tomlkit-0.12.0 transformers-4.30.1 typer-0.12.5 typing-extensions-4.12.2 urllib3-2.2.3 uvicorn-0.30.6 watchdog-4.0.2 websockets-12.0\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[34m[notice] A new release of pip is available: 23.3.2 -> 24.2\u001b[0m\n",
      "\u001b[34m[notice] To update, run: pip install --upgrade pip\u001b[0m\n",
      "\u001b[34m2024-09-26 10:29:04,647 sagemaker-training-toolkit INFO     Waiting for the process to finish and give a return code.\u001b[0m\n",
      "\u001b[34m2024-09-26 10:29:04,648 sagemaker-training-toolkit INFO     Done waiting for a return code. Received 0 from exiting process.\u001b[0m\n",
      "\u001b[34m2024-09-26 10:29:04,793 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-09-26 10:29:04,926 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-09-26 10:29:05,052 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-09-26 10:29:05,061 sagemaker-training-toolkit INFO     Invoking user script\u001b[0m\n",
      "\u001b[34mTraining Env:\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {\n",
      "        \"KuaFuTrainData\": \"/opt/ml/input/data/KuaFuTrainData\"\n",
      "    },\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"current_instance_group\": \"homogeneousCluster\",\n",
      "    \"current_instance_group_hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "    \"distribution_hosts\": [],\n",
      "    \"distribution_instance_groups\": [],\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {},\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {\n",
      "        \"KuaFuTrainData\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        }\n",
      "    },\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"instance_groups\": [\n",
      "        \"homogeneousCluster\"\n",
      "    ],\n",
      "    \"instance_groups_dict\": {\n",
      "        \"homogeneousCluster\": {\n",
      "            \"instance_group_name\": \"homogeneousCluster\",\n",
      "            \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "            \"hosts\": [\n",
      "                \"algo-1\"\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"is_hetero\": false,\n",
      "    \"is_master\": true,\n",
      "    \"is_modelparallel_enabled\": null,\n",
      "    \"is_smddpmprun_installed\": true,\n",
      "    \"job_name\": \"huggingface-chatglm-finetune-fulltuning-2024-09-26-10-18-23-050\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-us-west-2-687912291502/huggingface-chatglm-finetune-fulltuning-2024-09-26-10-18-23-050/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"sm_fulltune_train\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 96,\n",
      "    \"num_gpus\": 8,\n",
      "    \"num_neurons\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"sm_fulltune_train.py\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mEnvironment variables:\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=sm_fulltune_train.py\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={\"KuaFuTrainData\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[\"KuaFuTrainData\"]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_TYPE=ml.p4d.24xlarge\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP=homogeneousCluster\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}}\u001b[0m\n",
      "\u001b[34mSM_DISTRIBUTION_INSTANCE_GROUPS=[]\u001b[0m\n",
      "\u001b[34mSM_IS_HETERO=false\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=sm_fulltune_train\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=96\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=8\u001b[0m\n",
      "\u001b[34mSM_NUM_NEURONS=0\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-west-2-687912291502/huggingface-chatglm-finetune-fulltuning-2024-09-26-10-18-23-050/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"KuaFuTrainData\":\"/opt/ml/input/data/KuaFuTrainData\"},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.p4d.24xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"KuaFuTrainData\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":true,\"job_name\":\"huggingface-chatglm-finetune-fulltuning-2024-09-26-10-18-23-050\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-west-2-687912291502/huggingface-chatglm-finetune-fulltuning-2024-09-26-10-18-23-050/source/sourcedir.tar.gz\",\"module_name\":\"sm_fulltune_train\",\"network_interface_name\":\"eth0\",\"num_cpus\":96,\"num_gpus\":8,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"sm_fulltune_train.py\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_KUAFUTRAINDATA=/opt/ml/input/data/KuaFuTrainData\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python310.zip:/opt/conda/lib/python3.10:/opt/conda/lib/python3.10/lib-dynload:/opt/conda/lib/python3.10/site-packages\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.10 sm_fulltune_train.py\u001b[0m\n",
      "\u001b[34m2024-09-26 10:29:05,089 sagemaker-training-toolkit INFO     Exceptions not imported for SageMaker TF as Tensorflow is not installed.\u001b[0m\n",
      "\u001b[34mFound existing installation: deepspeed 0.6.1+1ea3d4b\u001b[0m\n",
      "\u001b[34mUninstalling deepspeed-0.6.1+1ea3d4b:\u001b[0m\n",
      "\u001b[34mWould remove:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/deepspeed\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/deepspeed.pt\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/ds\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/ds_elastic\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/ds_report\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/ds_ssh\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.10/site-packages/deepspeed-0.6.1+1ea3d4b.dist-info/*\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.10/site-packages/deepspeed/*\u001b[0m\n",
      "\u001b[34mProceed (Y/n)?\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled deepspeed-0.6.1+1ea3d4b\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[34mCollecting pydantic==1.9.2\u001b[0m\n",
      "\u001b[34mDownloading pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (127 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 127.1/127.1 kB 5.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from pydantic==1.9.2) (4.12.2)\u001b[0m\n",
      "\u001b[34mDownloading pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.0 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.0/12.0 MB 122.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mInstalling collected packages: pydantic\u001b[0m\n",
      "\u001b[34mAttempting uninstall: pydantic\u001b[0m\n",
      "\u001b[34mFound existing installation: pydantic 2.9.2\u001b[0m\n",
      "\u001b[34mUninstalling pydantic-2.9.2:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled pydantic-2.9.2\u001b[0m\n",
      "\u001b[34mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\u001b[0m\n",
      "\u001b[34mgradio 4.44.0 requires pydantic>=2.0, but you have pydantic 1.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mspacy 3.6.0 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.5 which is incompatible.\u001b[0m\n",
      "\u001b[34mSuccessfully installed pydantic-1.9.2\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[34m[notice] A new release of pip is available: 23.3.2 -> 24.2\u001b[0m\n",
      "\u001b[34m[notice] To update, run: pip install --upgrade pip\u001b[0m\n",
      "\u001b[34mCollecting deepspeed\u001b[0m\n",
      "\u001b[34mDownloading deepspeed-0.15.1.tar.gz (1.4 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.4/1.4 MB 23.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: deepspeed\u001b[0m\n",
      "\u001b[34mBuilding wheel for deepspeed (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for deepspeed (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for deepspeed: filename=deepspeed-0.15.1-py3-none-any.whl size=1483877 sha256=83a1052e209147811d8450bae684a9c82969a3df3af3bb1dc23f8b68b78d1e25\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/da/cb/14/9cbba50c73df044eb32a7ca29e34844c5f8959e12d22ae8b60\u001b[0m\n",
      "\u001b[34mSuccessfully built deepspeed\u001b[0m\n",
      "\u001b[34mInstalling collected packages: deepspeed\u001b[0m\n",
      "\u001b[34mSuccessfully installed deepspeed-0.15.1\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[34m[notice] A new release of pip is available: 23.3.2 -> 24.2\u001b[0m\n",
      "\u001b[34m[notice] To update, run: pip install --upgrade pip\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: protobuf in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 1)) (3.20.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: transformers==4.30.1 in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 3)) (4.30.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cpm_kernels in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 4)) (1.0.11)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: gradio in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 6)) (4.44.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: mdtex2html in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 7)) (1.3.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 8)) (0.1.99)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: accelerate>=0.20.1 in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 9)) (0.34.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sse-starlette in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 10)) (2.1.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: streamlit>=1.24.0 in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 11)) (1.38.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 12)) (2.16.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: huggingface in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 13)) (0.0.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jieba in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 14)) (0.42.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: rouge_chinese in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 15)) (1.0.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: nltk in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 16)) (3.9.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (3.12.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (0.25.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (1.24.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (23.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (6.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (2023.12.25)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (2.31.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (0.13.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (0.4.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (4.65.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiofiles<24.0,>=22.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (23.2.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: anyio<5.0,>=3.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (4.6.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fastapi<1.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (0.115.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: ffmpy in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (0.4.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: gradio-client==1.3.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (1.3.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: httpx>=0.24.1 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (0.27.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: importlib-resources<7.0,>=1.3 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (6.4.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jinja2<4.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (3.1.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markupsafe~=2.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (2.1.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: matplotlib~=3.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (3.7.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: orjson~=3.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (3.10.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pandas<3.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (2.0.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pillow<11.0,>=8.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (10.0.0)\u001b[0m\n",
      "\u001b[34mCollecting pydantic>=2.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mUsing cached pydantic-2.9.2-py3-none-any.whl.metadata (149 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pydub in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (0.25.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-multipart>=0.0.9 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (0.0.10)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: ruff>=0.2.2 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (0.6.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: semantic-version~=2.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (2.10.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tomlkit==0.12.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (0.12.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typer<1.0,>=0.12 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (0.12.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typing-extensions~=4.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (4.12.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: urllib3~=2.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (2.2.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: uvicorn>=0.14.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (0.30.6)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from gradio-client==1.3.0->gradio->-r ./requirements.txt (line 6)) (2023.6.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: websockets<13.0,>=10.0 in /opt/conda/lib/python3.10/site-packages (from gradio-client==1.3.0->gradio->-r ./requirements.txt (line 6)) (12.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markdown in /opt/conda/lib/python3.10/site-packages (from mdtex2html->-r ./requirements.txt (line 7)) (3.5.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: latex2mathml in /opt/conda/lib/python3.10/site-packages (from mdtex2html->-r ./requirements.txt (line 7)) (3.77.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.20.1->-r ./requirements.txt (line 9)) (5.9.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.20.1->-r ./requirements.txt (line 9)) (2.0.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: starlette in /opt/conda/lib/python3.10/site-packages (from sse-starlette->-r ./requirements.txt (line 10)) (0.38.6)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: altair<6,>=4.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (5.4.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: blinker<2,>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (1.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cachetools<6,>=4.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (5.3.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: click<9,>=7.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (8.1.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyarrow>=7.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (14.0.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: rich<14,>=10.14.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (13.4.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tenacity<9,>=8.1.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (8.2.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: toml<2,>=0.10.1 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (0.10.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: gitpython!=3.1.19,<4,>=3.0.7 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (3.1.43)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pydeck<1,>=0.8.0b4 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (0.9.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tornado<7,>=6.0.3 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (6.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: watchdog<5,>=2.1.5 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (4.0.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyarrow-hotfix in /opt/conda/lib/python3.10/site-packages (from datasets->-r ./requirements.txt (line 12)) (0.6)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: dill<0.3.8,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets->-r ./requirements.txt (line 12)) (0.3.6)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets->-r ./requirements.txt (line 12)) (3.4.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets->-r ./requirements.txt (line 12)) (0.70.14)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets->-r ./requirements.txt (line 12)) (3.9.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from rouge_chinese->-r ./requirements.txt (line 15)) (1.16.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: joblib in /opt/conda/lib/python3.10/site-packages (from nltk->-r ./requirements.txt (line 16)) (1.3.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jsonschema>=3.0 in /opt/conda/lib/python3.10/site-packages (from altair<6,>=4.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (4.18.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: narwhals>=1.5.2 in /opt/conda/lib/python3.10/site-packages (from altair<6,>=4.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (1.8.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: idna>=2.8 in /opt/conda/lib/python3.10/site-packages (from anyio<5.0,>=3.0->gradio->-r ./requirements.txt (line 6)) (3.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sniffio>=1.1 in /opt/conda/lib/python3.10/site-packages (from anyio<5.0,>=3.0->gradio->-r ./requirements.txt (line 6)) (1.3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: exceptiongroup>=1.0.2 in /opt/conda/lib/python3.10/site-packages (from anyio<5.0,>=3.0->gradio->-r ./requirements.txt (line 6)) (1.2.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r ./requirements.txt (line 12)) (1.3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r ./requirements.txt (line 12)) (23.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r ./requirements.txt (line 12)) (1.4.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r ./requirements.txt (line 12)) (6.0.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r ./requirements.txt (line 12)) (1.9.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r ./requirements.txt (line 12)) (4.0.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: gitdb<5,>=4.0.1 in /opt/conda/lib/python3.10/site-packages (from gitpython!=3.1.19,<4,>=3.0.7->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (4.0.11)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: certifi in /opt/conda/lib/python3.10/site-packages (from httpx>=0.24.1->gradio->-r ./requirements.txt (line 6)) (2024.2.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: httpcore==1.* in /opt/conda/lib/python3.10/site-packages (from httpx>=0.24.1->gradio->-r ./requirements.txt (line 6)) (1.0.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: h11<0.15,>=0.13 in /opt/conda/lib/python3.10/site-packages (from httpcore==1.*->httpx>=0.24.1->gradio->-r ./requirements.txt (line 6)) (0.14.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (1.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (0.11.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (4.47.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (1.4.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyparsing<3.1,>=2.3.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (3.0.9)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (2.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas<3.0,>=1.0->gradio->-r ./requirements.txt (line 6)) (2023.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas<3.0,>=1.0->gradio->-r ./requirements.txt (line 6)) (2023.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: annotated-types>=0.6.0 in /opt/conda/lib/python3.10/site-packages (from pydantic>=2.0->gradio->-r ./requirements.txt (line 6)) (0.7.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pydantic-core==2.23.4 in /opt/conda/lib/python3.10/site-packages (from pydantic>=2.0->gradio->-r ./requirements.txt (line 6)) (2.23.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.30.1->-r ./requirements.txt (line 3)) (3.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markdown-it-py>=2.2.0 in /opt/conda/lib/python3.10/site-packages (from rich<14,>=10.14.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (3.0.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from rich<14,>=10.14.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (2.15.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate>=0.20.1->-r ./requirements.txt (line 9)) (1.12)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate>=0.20.1->-r ./requirements.txt (line 9)) (3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: shellingham>=1.3.0 in /opt/conda/lib/python3.10/site-packages (from typer<1.0,>=0.12->gradio->-r ./requirements.txt (line 6)) (1.5.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: smmap<6,>=3.0.1 in /opt/conda/lib/python3.10/site-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.19,<4,>=3.0.7->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (5.0.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (2023.6.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: referencing>=0.28.4 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (0.29.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: rpds-py>=0.7.1 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (0.8.10)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: mdurl~=0.1 in /opt/conda/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich<14,>=10.14.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (0.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate>=0.20.1->-r ./requirements.txt (line 9)) (1.3.0)\u001b[0m\n",
      "\u001b[34mUsing cached pydantic-2.9.2-py3-none-any.whl (434 kB)\u001b[0m\n",
      "\u001b[34mInstalling collected packages: pydantic\u001b[0m\n",
      "\u001b[34mAttempting uninstall: pydantic\u001b[0m\n",
      "\u001b[34mFound existing installation: pydantic 1.9.2\u001b[0m\n",
      "\u001b[34mUninstalling pydantic-1.9.2:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled pydantic-1.9.2\u001b[0m\n",
      "\u001b[34mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\u001b[0m\n",
      "\u001b[34mdeepspeed 0.15.1 requires nvidia-ml-py, which is not installed.\u001b[0m\n",
      "\u001b[34mconfection 0.1.0 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mspacy 3.6.0 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mspacy 3.6.0 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.5 which is incompatible.\u001b[0m\n",
      "\u001b[34mthinc 8.1.10 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mSuccessfully installed pydantic-2.9.2\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[34m[notice] A new release of pip is available: 23.3.2 -> 24.2\u001b[0m\n",
      "\u001b[34m[notice] To update, run: pip install --upgrade pip\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:37,623] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:40,379] [WARNING] [runner.py:212:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only.\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:40,379] [INFO] [runner.py:585:main] cmd = /opt/conda/bin/python3.10 -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMCwgMSwgMiwgMywgNCwgNSwgNiwgN119 --master_addr=127.0.0.1 --master_port=23456 --enable_each_rank_log=None main_tuning.py --deepspeed deepspeed.json --do_train --train_file /opt/ml/input/data/KuaFuTrainData/train.json --validation_file /opt/ml/input/data/KuaFuTrainData/dev.json --prompt_column content --response_column summary --overwrite_cache --model_name_or_path s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/ --output_dir /opt/ml/model/adgen-chatglm2-6b-ft --model_output_s3_path s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/ --overwrite_output_dir --max_source_length 64 --max_target_length 64 --per_device_train_batch_size 1 --per_device_eval_batch_size 4 --gradient_accumulation_steps 1 --predict_with_generate --max_steps 50 --logging_steps 10 --save_steps 50 --learning_rate 1e-4 --fp16\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:45,399] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,118] [INFO] [launch.py:139:main] 0 NCCL_VERSION=2.16.2\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,118] [INFO] [launch.py:139:main] 0 NCCL_SOCKET_IFNAME=eth0\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,118] [INFO] [launch.py:139:main] 0 NCCL_DEBUG=WARN\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,118] [INFO] [launch.py:139:main] 0 NCCL_IB_DISABLE=1\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,118] [INFO] [launch.py:146:main] WORLD INFO DICT: {'localhost': [0, 1, 2, 3, 4, 5, 6, 7]}\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,118] [INFO] [launch.py:152:main] nnodes=1, num_local_procs=8, node_rank=0\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,118] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0, 1, 2, 3, 4, 5, 6, 7]})\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,118] [INFO] [launch.py:164:main] dist_world_size=8\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,118] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,119] [INFO] [launch.py:256:main] process 305 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=0', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/opt/ml/model/adgen-chatglm2-6b-ft', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,119] [INFO] [launch.py:256:main] process 306 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=1', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/opt/ml/model/adgen-chatglm2-6b-ft', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,120] [INFO] [launch.py:256:main] process 307 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=2', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/opt/ml/model/adgen-chatglm2-6b-ft', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,120] [INFO] [launch.py:256:main] process 308 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=3', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/opt/ml/model/adgen-chatglm2-6b-ft', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,121] [INFO] [launch.py:256:main] process 309 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=4', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/opt/ml/model/adgen-chatglm2-6b-ft', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,121] [INFO] [launch.py:256:main] process 310 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=5', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/opt/ml/model/adgen-chatglm2-6b-ft', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,122] [INFO] [launch.py:256:main] process 311 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=6', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/opt/ml/model/adgen-chatglm2-6b-ft', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:48,122] [INFO] [launch.py:256:main] process 312 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=7', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/opt/ml/model/adgen-chatglm2-6b-ft', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:59,816] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:59,903] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:59,957] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:59,960] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:59,970] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:59,971] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:59,975] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:29:59,981] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:30:03,590] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:30:03,825] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:30:03,853] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:30:03,921] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:30:03,922] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:30:03,943] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:30:03,957] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:30:03,957] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:30:03,981] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m09/26/2024 10:30:05 - WARNING - __main__ - Process rank: 7, device: cuda:7, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/26/2024 10:30:05 - WARNING - __main__ - Process rank: 4, device: cuda:4, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/26/2024 10:30:05 - WARNING - __main__ - Process rank: 5, device: cuda:5, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/26/2024 10:30:05 - WARNING - __main__ - Process rank: 1, device: cuda:1, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/26/2024 10:30:05 - WARNING - __main__ - Process rank: 6, device: cuda:6, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/26/2024 10:30:05 - WARNING - __main__ - Process rank: 3, device: cuda:3, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/26/2024 10:30:05 - WARNING - __main__ - Process rank: 2, device: cuda:2, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/26/2024 10:30:05 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/26/2024 10:30:05 - INFO - __main__ - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=None,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=deepspeed.json,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=1,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=0.0001,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=0,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/opt/ml/model/adgen-chatglm2-6b-ft/runs/Sep26_10-30-03_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=linear,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=50,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=3.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_hf,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/opt/ml/model/adgen-chatglm2-6b-ft,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=True,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=4,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=1,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=True,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=['tensorboard'],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/opt/ml/model/adgen-chatglm2-6b-ft,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=50,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34mGenerating train split: 0 examples [00:00, ? examples/s]\u001b[0m\n",
      "\u001b[34mGenerating train split: 26273 examples [00:00, 249098.52 examples/s]\u001b[0m\n",
      "\u001b[34mGenerating train split: 42351 examples [00:00, 277816.74 examples/s]\u001b[0m\n",
      "\u001b[34mGenerating validation split: 0 examples [00:00, ? examples/s]\u001b[0m\n",
      "\u001b[34mGenerating validation split: 1000 examples [00:00, 210917.43 examples/s]\u001b[0m\n",
      "\u001b[34mworld_rank===7\u001b[0m\n",
      "\u001b[34mworld_rank===5\u001b[0m\n",
      "\u001b[34mworld_rank===6\u001b[0m\n",
      "\u001b[34mworld_rank===1\u001b[0m\n",
      "\u001b[34mworld_rank===0\u001b[0m\n",
      "\u001b[34mworld_rank===2\u001b[0m\n",
      "\u001b[34mworld_rank===3\u001b[0m\n",
      "\u001b[34mworld_rank===4\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/configuration_chatglm.py /tmp/orignal/configuration_chatglm.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/config.json /tmp/orignal/config.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model.bin.index.json /tmp/orignal/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/tokenizer_config.json /tmp/orignal/tokenizer_config.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/modeling_chatglm.py /tmp/orignal/modeling_chatglm.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/quantization.py /tmp/orignal/quantization.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/tokenization_chatglm.py /tmp/orignal/tokenization_chatglm.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/tokenizer.model /tmp/orignal/tokenizer.model\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00007-of-00007.bin /tmp/orignal/pytorch_model-00007-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00004-of-00007.bin /tmp/orignal/pytorch_model-00004-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00006-of-00007.bin /tmp/orignal/pytorch_model-00006-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00005-of-00007.bin /tmp/orignal/pytorch_model-00005-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00003-of-00007.bin /tmp/orignal/pytorch_model-00003-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00002-of-00007.bin /tmp/orignal/pytorch_model-00002-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00001-of-00007.bin /tmp/orignal/pytorch_model-00001-of-00007.bin\u001b[0m\n",
      "\u001b[34msync s3 model s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/to /tmp/orignal:\u001b[0m\n",
      "\u001b[34m['pytorch_model-00007-of-00007.bin', 'tokenization_chatglm.py', 'pytorch_model-00004-of-00007.bin', 'pytorch_model-00006-of-00007.bin', 'quantization.py', 'pytorch_model-00001-of-00007.bin', 'modeling_chatglm.py', 'configuration_chatglm.py', 'tokenizer.model', 'pytorch_model.bin.index.json', 'pytorch_model-00003-of-00007.bin', 'config.json', 'pytorch_model-00005-of-00007.bin', 'tokenizer_config.json', 'pytorch_model-00002-of-00007.bin']\u001b[0m\n",
      "\u001b[34mNCCL version 2.16.2+cuda11.8\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:667] 2024-09-26 10:30:17,311 >> loading configuration file /tmp/orignal/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:667] 2024-09-26 10:30:17,311 >> loading configuration file /tmp/orignal/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:667] 2024-09-26 10:30:17,316 >> loading configuration file /tmp/orignal/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:667] 2024-09-26 10:30:17,316 >> loading configuration file /tmp/orignal/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:725] 2024-09-26 10:30:17,317 >> Model config ChatGLMConfig {\n",
      "  \"_name_or_path\": \"/tmp/orignal/\",\n",
      "  \"add_bias_linear\": false,\n",
      "  \"add_qkv_bias\": true,\n",
      "  \"apply_query_key_layer_scaling\": true,\n",
      "  \"apply_residual_connection_post_layernorm\": false,\n",
      "  \"architectures\": [\n",
      "    \"ChatGLMModel\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"attention_softmax_in_fp32\": true,\n",
      "  \"auto_map\": {\n",
      "    \"AutoConfig\": \"configuration_chatglm.ChatGLMConfig\",\n",
      "    \"AutoModel\": \"modeling_chatglm.ChatGLMForConditionalGeneration\",\n",
      "    \"AutoModelForCausalLM\": \"modeling_chatglm.ChatGLMForConditionalGeneration\",\n",
      "    \"AutoModelForSeq2SeqLM\": \"modeling_chatglm.ChatGLMForConditionalGeneration\"\n",
      "  },\n",
      "  \"bias_dropout_fusion\": true,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"ffn_hidden_size\": 13696,\n",
      "  \"fp32_residual_connection\": false,\n",
      "  \"hidden_dropout\": 0.0,\n",
      "  \"hidden_size\": 4096,\n",
      "  \"kv_channels\": 128,\n",
      "  \"layernorm_epsilon\": 1e-05,\n",
      "  \"model_type\": \"chatglm\",\n",
      "  \"multi_query_attention\": true,\n",
      "  \"multi_query_group_num\": 2,\n",
      "  \"num_attention_heads\": 32,\n",
      "  \"num_layers\": 28,\n",
      "  \"original_rope\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"padded_vocab_size\": 65024,\n",
      "  \"post_layer_norm\": true,\n",
      "  \"pre_seq_len\": null,\n",
      "  \"prefix_projection\": false,\n",
      "  \"quantization_bit\": 0,\n",
      "  \"rmsnorm\": true,\n",
      "  \"seq_length\": 32768,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"float16\",\n",
      "  \"transformers_version\": \"4.30.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 65024\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:725] 2024-09-26 10:30:17,317 >> Model config ChatGLMConfig {\n",
      "  \"_name_or_path\": \"/tmp/orignal/\",\n",
      "  \"add_bias_linear\": false,\n",
      "  \"add_qkv_bias\": true,\n",
      "  \"apply_query_key_layer_scaling\": true,\n",
      "  \"apply_residual_connection_post_layernorm\": false,\n",
      "  \"architectures\": [\n",
      "    \"ChatGLMModel\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"attention_softmax_in_fp32\": true,\n",
      "  \"auto_map\": {\n",
      "    \"AutoConfig\": \"configuration_chatglm.ChatGLMConfig\",\n",
      "    \"AutoModel\": \"modeling_chatglm.ChatGLMForConditionalGeneration\",\n",
      "    \"AutoModelForCausalLM\": \"modeling_chatglm.ChatGLMForConditionalGeneration\",\n",
      "    \"AutoModelForSeq2SeqLM\": \"modeling_chatglm.ChatGLMForConditionalGeneration\"\n",
      "  },\n",
      "  \"bias_dropout_fusion\": true,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"ffn_hidden_size\": 13696,\n",
      "  \"fp32_residual_connection\": false,\n",
      "  \"hidden_dropout\": 0.0,\n",
      "  \"hidden_size\": 4096,\n",
      "  \"kv_channels\": 128,\n",
      "  \"layernorm_epsilon\": 1e-05,\n",
      "  \"model_type\": \"chatglm\",\n",
      "  \"multi_query_attention\": true,\n",
      "  \"multi_query_group_num\": 2,\n",
      "  \"num_attention_heads\": 32,\n",
      "  \"num_layers\": 28,\n",
      "  \"original_rope\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"padded_vocab_size\": 65024,\n",
      "  \"post_layer_norm\": true,\n",
      "  \"pre_seq_len\": null,\n",
      "  \"prefix_projection\": false,\n",
      "  \"quantization_bit\": 0,\n",
      "  \"rmsnorm\": true,\n",
      "  \"seq_length\": 32768,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"float16\",\n",
      "  \"transformers_version\": \"4.30.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 65024\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-26 10:30:17,320 >> loading file tokenizer.model\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-26 10:30:17,320 >> loading file tokenizer.model\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-26 10:30:17,320 >> loading file added_tokens.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-26 10:30:17,320 >> loading file special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-26 10:30:17,320 >> loading file tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-26 10:30:17,320 >> loading file added_tokens.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-26 10:30:17,320 >> loading file special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-26 10:30:17,320 >> loading file tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2575] 2024-09-26 10:30:17,439 >> loading weights file /tmp/orignal/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2575] 2024-09-26 10:30:17,439 >> loading weights file /tmp/orignal/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:577] 2024-09-26 10:30:17,440 >> Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.30.1\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:577] 2024-09-26 10:30:17,440 >> Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.30.1\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]#015Loading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:01<00:10,  1.67s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:01<00:11,  1.88s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:12,  2.07s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:13,  2.28s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:13,  2.31s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:13,  2.32s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:14,  2.34s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:14,  2.34s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:03<00:08,  1.63s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:03<00:09,  1.86s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:03<00:09,  1.95s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:11,  2.20s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:11,  2.22s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:11,  2.26s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:11,  2.25s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:11,  2.28s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:04<00:06,  1.66s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:05<00:07,  1.87s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:05<00:07,  1.85s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.12s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.14s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.15s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.21s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.24s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:06<00:05,  1.73s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:07<00:05,  1.91s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:07<00:05,  1.91s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:05,  1.95s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:05,  1.98s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:06,  2.04s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:06,  2.07s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:06,  2.14s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:08<00:03,  1.81s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:09<00:03,  1.90s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:09<00:04,  2.02s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.03s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.04s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.08s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.07s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.09s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:10<00:01,  1.87s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:11<00:00,  1.64s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:11<00:00,  1.70s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:11<00:02,  2.08s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.12s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.03s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.16s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.18s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.15s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:13<00:02,  2.16s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:13<00:00,  1.98s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:13<00:00,  1.95s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:13<00:00,  1.92s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:13<00:00,  1.96s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:13<00:00,  1.82s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:13<00:00,  1.97s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  1.98s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.07s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  1.96s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.07s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.11s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.12s/it]\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3295] 2024-09-26 10:30:32,594 >> All model checkpoint weights were used when initializing ChatGLMForConditionalGeneration.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3295] 2024-09-26 10:30:32,594 >> All model checkpoint weights were used when initializing ChatGLMForConditionalGeneration.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3303] 2024-09-26 10:30:32,594 >> All the weights of ChatGLMForConditionalGeneration were initialized from the model checkpoint at /tmp/orignal/.\u001b[0m\n",
      "\u001b[34mIf your task is similar to the task the model of the checkpoint was trained on, you can already use ChatGLMForConditionalGeneration for predictions without further training.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3303] 2024-09-26 10:30:32,594 >> All the weights of ChatGLMForConditionalGeneration were initialized from the model checkpoint at /tmp/orignal/.\u001b[0m\n",
      "\u001b[34mIf your task is similar to the task the model of the checkpoint was trained on, you can already use ChatGLMForConditionalGeneration for predictions without further training.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2927] 2024-09-26 10:30:32,599 >> Generation config file not found, using a generation config created from the model config.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2927] 2024-09-26 10:30:32,599 >> Generation config file not found, using a generation config created from the model config.\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:17<00:00,  2.85s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:17<00:00,  2.47s/it]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:02<01:47, 383.93 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:04<01:23, 481.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:06<01:15, 519.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:10, 544.03 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:09<01:06, 558.15 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:11<01:04, 566.47 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:01, 572.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:14<00:59, 577.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:16<00:57, 576.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:18<00:55, 579.83 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 581.43 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:21<00:52, 580.40 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:23<00:50, 579.95 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:49, 576.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:26<00:47, 577.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:28<00:45, 579.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:30<00:43, 578.58 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:42, 578.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:33<00:40, 580.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:35<00:38, 579.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:37<00:36, 582.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:35, 580.24 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:40<00:33, 579.40 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:42<00:31, 578.98 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:30, 578.14 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:45<00:28, 579.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:47<00:26, 579.21 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:49<00:24, 579.54 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:22, 581.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:52<00:21, 581.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:54<00:19, 584.61 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 582.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:16, 583.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:59<00:14, 583.39 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:01<00:12, 584.76 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:10, 583.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:04<00:09, 584.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:06<00:07, 584.21 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 582.98 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:04, 582.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:11<00:02, 582.24 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:13<00:00, 582.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 580.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 574.54 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids\u001b[0m\n",
      "\u001b[34m[64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]#015Running tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]#015Running tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]#015Running tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:577] 2024-09-26 10:31:53,269 >> max_steps is given, it will override any value given in num_train_epochs\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:577] 2024-09-26 10:31:53,269 >> max_steps is given, it will override any value given in num_train_epochs\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:31:53,474] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed info: version=0.15.1, git-hash=unknown, git-branch=unknown\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:31:53,474] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 8\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:01<01:13, 565.09 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:01<01:13, 564.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:01<01:13, 559.61 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:01<01:14, 557.57 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:01<01:15, 549.21 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:02<01:50, 374.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:02<01:50, 374.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:03<01:10, 573.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:03<01:11, 568.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:03<01:11, 567.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:03<01:11, 564.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:03<01:11, 563.37 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:04<01:25, 473.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:04<01:25, 472.59 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:05<01:08, 576.85 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:05<01:09, 564.58 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:05<01:09, 564.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:05<01:09, 566.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:05<01:10, 560.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:06<01:16, 513.59 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:06<01:16, 512.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:06<01:05, 583.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:07, 571.58 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:07, 571.10 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:06, 572.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:07, 567.21 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:10, 542.09 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:11, 539.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:08<01:03, 586.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:08<01:04, 574.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:08<01:05, 574.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:08<01:04, 575.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:08<01:05, 570.59 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:09<01:06, 557.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:09<01:07, 555.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:10<01:02, 585.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:10<01:03, 575.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:10<01:03, 575.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:10<01:03, 575.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:10<01:03, 572.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:11<01:04, 567.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:11<01:04, 565.57 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:11<01:00, 587.63 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:01, 578.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:01, 577.45 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:01, 577.64 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:01, 574.45 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:01, 575.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:01, 572.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:13<00:58, 590.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:13<00:59, 579.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:13<00:59, 579.62 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:13<00:59, 579.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:14<00:59, 575.63 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:14<00:59, 580.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:14<00:59, 577.99 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:15<00:56, 588.94 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:15<00:57, 578.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:15<00:57, 577.83 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:15<00:57, 578.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:15<00:58, 574.03 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:16<00:57, 581.24 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:16<00:57, 578.59 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:17<00:54, 591.02 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:17<00:55, 580.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:17<00:55, 579.89 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:17<00:55, 580.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:17<00:56, 576.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:18<00:55, 585.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:18<00:55, 582.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:18<00:52, 592.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 581.39 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 581.14 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 581.48 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:54, 576.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 587.16 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 584.47 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:20<00:51, 590.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:20<00:52, 579.68 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:20<00:52, 579.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:20<00:52, 580.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:20<00:52, 575.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:21<00:51, 586.62 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:21<00:51, 583.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:22<00:49, 590.85 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:22<00:50, 579.93 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:22<00:50, 579.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:22<00:50, 580.25 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:22<00:50, 575.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:23<00:49, 587.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:23<00:50, 584.47 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:23<00:48, 589.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:48, 579.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:48, 579.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:49, 578.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:49, 574.89 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:48, 586.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:48, 583.21 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:25<00:46, 591.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:25<00:47, 579.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:25<00:47, 580.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:25<00:47, 579.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:26<00:47, 576.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:26<00:46, 587.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:26<00:46, 583.56 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:27<00:44, 591.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:27<00:45, 580.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:27<00:45, 580.93 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:27<00:45, 580.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:27<00:45, 576.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:28<00:44, 588.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:28<00:45, 584.84 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:28<00:42, 590.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:29<00:43, 579.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:29<00:43, 580.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:29<00:43, 579.15 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:29<00:44, 575.14 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:29<00:43, 587.28 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:30<00:43, 583.58 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:30<00:41, 591.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:41, 580.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:42, 579.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:42, 578.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:42, 575.44 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:41, 587.39 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:41, 584.01 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:32<00:39, 592.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:32<00:40, 581.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:32<00:40, 580.30 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:32<00:40, 580.16 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:33<00:40, 576.48 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:33<00:39, 588.54 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:33<00:39, 585.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:33<00:37, 591.28 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:34<00:38, 580.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:34<00:38, 579.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:34<00:38, 579.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:34<00:38, 575.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:35<00:38, 587.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:35<00:38, 584.40 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:35<00:35, 593.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:36, 581.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:36, 579.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:36, 580.56 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:37, 576.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:36, 588.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:36, 584.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:37<00:34, 591.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:35, 579.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:35, 577.98 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:35, 578.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:35, 575.01 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:34, 586.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:34, 583.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:39<00:32, 590.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:39<00:33, 579.33 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:39<00:33, 577.37 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:39<00:33, 577.87 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:40<00:33, 574.42 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:40<00:33, 586.13 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:40<00:33, 582.45 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:40<00:31, 590.00 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:41<00:31, 578.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:41<00:31, 576.99 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:41<00:31, 577.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:41<00:31, 574.03 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:41<00:31, 585.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:42<00:31, 582.37 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:42<00:29, 589.10 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:30, 577.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:30, 576.37 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:30, 576.48 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:29, 584.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:30, 573.45 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:29, 582.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:44<00:27, 589.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:44<00:28, 578.64 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:45<00:28, 576.91 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:45<00:28, 577.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:45<00:27, 585.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:45<00:28, 573.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:45<00:28, 582.84 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:45<00:26, 589.21 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:46<00:26, 577.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:46<00:26, 577.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:46<00:26, 576.03 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:46<00:26, 585.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:47<00:26, 573.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:47<00:26, 582.44 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:47<00:24, 589.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:24, 578.09 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:24, 577.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:24, 576.88 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:24, 585.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:25, 573.28 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:24, 582.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:49<00:22, 590.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:23, 579.25 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:23, 578.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:23, 578.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:22, 586.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:23, 574.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:22, 584.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:50<00:20, 590.48 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:51<00:21, 579.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:51<00:21, 579.08 examples/s]#015Running tokenizer on train dataset:  71%|███████   | 30000/42351 [00:51<00:21, 579.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:52<00:21, 587.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:52<00:21, 575.63 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:52<00:21, 584.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:52<00:19, 593.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:53<00:19, 582.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:53<00:19, 582.13 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:53<00:19, 582.01 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:53<00:19, 590.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:53<00:19, 578.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:54<00:19, 588.11 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:54<00:17, 592.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 581.15 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 580.54 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 580.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 588.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 576.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 586.42 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:55<00:15, 592.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:16, 581.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:16, 580.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:16, 580.58 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:15, 588.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:15, 586.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:16, 576.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:57<00:14, 592.85 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:58<00:14, 582.14 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:58<00:14, 581.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:58<00:14, 581.24 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:58<00:14, 589.39 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:59<00:14, 586.56 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:59<00:14, 577.44 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [00:59<00:12, 594.42 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 583.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 583.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 582.64 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 590.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 588.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 579.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:01<00:10, 592.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:10, 581.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:10, 582.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:10, 580.76 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:10, 588.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:10, 586.48 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:11, 577.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:02<00:08, 594.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:03<00:09, 583.24 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:03<00:09, 583.85 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:03<00:09, 590.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:03<00:09, 582.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:04<00:09, 587.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:04<00:09, 578.78 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:04<00:07, 594.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:05<00:07, 582.78 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:05<00:07, 589.93 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:05<00:07, 583.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:05<00:07, 581.74 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:05<00:07, 587.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:06<00:05, 592.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:06<00:07, 578.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 580.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 588.74 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 582.42 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 580.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 585.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:07<00:03, 592.44 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 576.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:03, 588.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:04, 580.20 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:04, 581.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:04, 580.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:04, 584.48 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:09<00:02, 590.59 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:04, 576.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:10<00:02, 586.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:10<00:02, 578.85 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:10<00:02, 579.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:10<00:02, 578.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:11<00:02, 582.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:11<00:00, 589.01 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:11<00:02, 574.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:11<00:00, 587.71 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:11<00:00, 589.97 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids\u001b[0m\n",
      "\u001b[34m[64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:12<00:00, 585.10 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:12<00:00, 578.14 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:12<00:00, 576.87 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:12<00:00, 566.10 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:12<00:00, 581.33 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:13<00:00, 571.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 581.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 579.29 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids [64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 574.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 578.55 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids [64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 573.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 578.18 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids [64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 561.20 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 577.66 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids [64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 570.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 575.96 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids\u001b[0m\n",
      "\u001b[34m[64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:14<00:00, 521.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:14<00:00, 571.09 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids\u001b[0m\n",
      "\u001b[34m[64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:33:21,291] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py310_cu118 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mCreating extension directory /root/.cache/torch_extensions/py310_cu118/fused_adam...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py310_cu118 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mCreating extension directory /root/.cache/torch_extensions/py310_cu118/fused_adam...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py310_cu118 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py310_cu118 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py310_cu118 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py310_cu118 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py310_cu118 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py310_cu118 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mDetected CUDA files, patching ldflags\u001b[0m\n",
      "\u001b[34mEmitting ninja build file /root/.cache/torch_extensions/py310_cu118/fused_adam/build.ninja...\u001b[0m\n",
      "\u001b[34mBuilding extension module fused_adam...\u001b[0m\n",
      "\u001b[34mAllowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\u001b[0m\n",
      "\u001b[34m[1/3] /opt/conda/bin/nvcc  -DTORCH_EXTENSION_NAME=fused_adam -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -I/opt/conda/lib/python3.10/site-packages/deepspeed/ops/csrc/includes -I/opt/conda/lib/python3.10/site-packages/deepspeed/ops/csrc/adam -isystem /opt/conda/lib/python3.10/site-packages/torch/include -isystem /opt/conda/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.10/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.10/site-packages/torch/include/THC -isystem /opt/conda/include -isystem /opt/conda/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_80,code=compute_80 -gencode=arch=compute_80,code=sm_80 --compiler-options '-fPIC' -O3 -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DVERSION_GE_1_5 -lineinfo --use_fast_math -gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_80,code=compute_80 -DBF16_AVAILABLE -U__CUDA_NO_BFLOAT16_OPERATORS__ -U__CUDA_NO_BFLOAT162_OPERATORS__ -U__CUDA_NO_BFLOAT16_CONVERSIONS__ -std=c++17 -c /opt/conda/lib/python3.10/site-packages/deepspeed/ops/csrc/adam/multi_tensor_adam.cu -o multi_tensor_adam.cuda.o\u001b[0m\n",
      "\u001b[34m[2/3] c++ -MMD -MF fused_adam_frontend.o.d -DTORCH_EXTENSION_NAME=fused_adam -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -I/opt/conda/lib/python3.10/site-packages/deepspeed/ops/csrc/includes -I/opt/conda/lib/python3.10/site-packages/deepspeed/ops/csrc/adam -isystem /opt/conda/lib/python3.10/site-packages/torch/include -isystem /opt/conda/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.10/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.10/site-packages/torch/include/THC -isystem /opt/conda/include -isystem /opt/conda/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++17 -O3 -std=c++17 -g -Wno-reorder -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DVERSION_GE_1_5 -DBF16_AVAILABLE -c /opt/conda/lib/python3.10/site-packages/deepspeed/ops/csrc/adam/fused_adam_frontend.cpp -o fused_adam_frontend.o\u001b[0m\n",
      "\u001b[34m[3/3] c++ fused_adam_frontend.o multi_tensor_adam.cuda.o -shared -L/opt/conda/lib/python3.10/site-packages/torch/lib -lc10 -lc10_cuda -ltorch_cpu -ltorch_cuda -ltorch -ltorch_python -L/opt/conda/lib64 -lcudart -o fused_adam.so\u001b[0m\n",
      "\u001b[34mFAILED: fused_adam.so \u001b[0m\n",
      "\u001b[34mc++ fused_adam_frontend.o multi_tensor_adam.cuda.o -shared -L/opt/conda/lib/python3.10/site-packages/torch/lib -lc10 -lc10_cuda -ltorch_cpu -ltorch_cuda -ltorch -ltorch_python -L/opt/conda/lib64 -lcudart -o fused_adam.so\u001b[0m\n",
      "\u001b[34m/usr/bin/ld: cannot find -lcudart\u001b[0m\n",
      "\u001b[34mcollect2: error: ld returned 1 exit status\u001b[0m\n",
      "\u001b[34mninja: build stopped: subcommand failed.\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1893, in _run_ninja_build\u001b[0m\n",
      "\u001b[34msubprocess.run(\n",
      "  File \"/opt/conda/lib/python3.10/subprocess.py\", line 526, in run\u001b[0m\n",
      "\u001b[34mraise CalledProcessError(retcode, process.args,\u001b[0m\n",
      "\u001b[34msubprocess.CalledProcessError\u001b[0m\n",
      "\u001b[34m: Command '['ninja', '-v']' returned non-zero exit status 1.\u001b[0m\n",
      "\u001b[34mThe above exception was the direct cause of the following exception:\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 472, in <module>\u001b[0m\n",
      "\u001b[34mmain()\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 388, in main\u001b[0m\n",
      "\u001b[34mtrain_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1645, in train\u001b[0m\n",
      "\u001b[34mreturn inner_training_loop(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1759, in _inner_training_loop\u001b[0m\n",
      "\u001b[34mmodel, self.optimizer, self.lr_scheduler = self.accelerator.prepare(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1318, in prepare\u001b[0m\n",
      "\u001b[34mresult = self._prepare_deepspeed(*args)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1815, in _prepare_deepspeed\u001b[0m\n",
      "\u001b[34mengine, optimizer, _, lr_scheduler = deepspeed.initialize(**kwargs)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/__init__.py\", line 193, in initialize\u001b[0m\n",
      "\u001b[34mengine = DeepSpeedEngine(args=args,\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 313, in __init__\u001b[0m\n",
      "\u001b[34mself._configure_optimizer(optimizer, model_parameters)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1276, in _configure_optimizer\u001b[0m\n",
      "\u001b[34mbasic_optimizer = self._configure_basic_optimizer(model_parameters)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1353, in _configure_basic_optimizer\u001b[0m\n",
      "\u001b[34moptimizer = FusedAdam(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/adam/fused_adam.py\", line 94, in __init__\u001b[0m\n",
      "\u001b[34mfused_adam_cuda = FusedAdamBuilder().load()\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 531, in load\u001b[0m\n",
      "\u001b[34mreturn self.jit_load(verbose)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 578, in jit_load\u001b[0m\n",
      "\u001b[34mop_module = load(name=self.name,\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1284, in load\u001b[0m\n",
      "\u001b[34mreturn _jit_compile(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1509, in _jit_compile\u001b[0m\n",
      "\u001b[34m_write_ninja_file_and_build_library(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1624, in _write_ninja_file_and_build_library\u001b[0m\n",
      "\u001b[34m_run_ninja_build(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1909, in _run_ninja_build\u001b[0m\n",
      "\u001b[34mraise RuntimeError(message) from e\u001b[0m\n",
      "\u001b[34mRuntimeError\u001b[0m\n",
      "\u001b[34m: Error building extension 'fused_adam'\u001b[0m\n",
      "\u001b[34mLoading extension module fused_adam...Loading extension module fused_adam...Loading extension module fused_adam...Loading extension module fused_adam...Loading extension module fused_adam...Loading extension module fused_adam...\u001b[0m\n",
      "\u001b[34mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 472, in <module>\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 472, in <module>\u001b[0m\n",
      "\u001b[34mFile \"/opt/ml/code/main_tuning.py\", line 472, in <module>\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 472, in <module>\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 472, in <module>\u001b[0m\n",
      "\u001b[34mFile \"/opt/ml/code/main_tuning.py\", line 472, in <module>\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 472, in <module>\u001b[0m\n",
      "\u001b[34mmain()main()\u001b[0m\n",
      "\u001b[34mFile \"/opt/ml/code/main_tuning.py\", line 388, in main\n",
      "          File \"/opt/ml/code/main_tuning.py\", line 388, in main\u001b[0m\n",
      "\u001b[34mmain()main()\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 388, in main\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 388, in main\u001b[0m\n",
      "\u001b[34mmain()train_result = trainer.train(resume_from_checkpoint=checkpoint)    \u001b[0m\n",
      "\u001b[34mtrain_result = trainer.train(resume_from_checkpoint=checkpoint)\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1645, in train\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 388, in main\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1645, in train\n",
      "    train_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
      "      File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1645, in train\u001b[0m\n",
      "\u001b[34mtrain_result = trainer.train(resume_from_checkpoint=checkpoint)\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1645, in train\u001b[0m\n",
      "\u001b[34mmain()\n",
      "    main()  File \"/opt/ml/code/main_tuning.py\", line 388, in main\n",
      "  File \"/opt/ml/code/main_tuning.py\", line 388, in main\n",
      "    train_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1645, in train\u001b[0m\n",
      "\u001b[34mtrain_result = trainer.train(resume_from_checkpoint=checkpoint)\u001b[0m\n",
      "\u001b[34mtrain_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1645, in train\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1645, in train\u001b[0m\n",
      "\u001b[34mreturn inner_training_loop(return inner_training_loop(return inner_training_loop(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1759, in _inner_training_loop\n",
      "      File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1759, in _inner_training_loop\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1759, in _inner_training_loop\u001b[0m\n",
      "\u001b[34mreturn inner_training_loop(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1759, in _inner_training_loop\u001b[0m\n",
      "\u001b[34mreturn inner_training_loop(\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1759, in _inner_training_loop\u001b[0m\n",
      "\u001b[34mmodel, self.optimizer, self.lr_scheduler = self.accelerator.prepare(model, self.optimizer, self.lr_scheduler = self.accelerator.prepare(model, self.optimizer, self.lr_scheduler = self.accelerator.prepare(\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1318, in prepare\u001b[0m\n",
      "\u001b[34mmodel, self.optimizer, self.lr_scheduler = self.accelerator.prepare(  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1318, in prepare\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1318, in prepare\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1318, in prepare\u001b[0m\n",
      "\u001b[34mmodel, self.optimizer, self.lr_scheduler = self.accelerator.prepare(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1318, in prepare\u001b[0m\n",
      "\u001b[34mreturn inner_training_loop(return inner_training_loop(\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1759, in _inner_training_loop\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1759, in _inner_training_loop\n",
      "    result = self._prepare_deepspeed(*args)        \u001b[0m\n",
      "\u001b[34mresult = self._prepare_deepspeed(*args)    result = self._prepare_deepspeed(*args)\u001b[0m\n",
      "\u001b[34mresult = self._prepare_deepspeed(*args)\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1815, in _prepare_deepspeed\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1815, in _prepare_deepspeed\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1815, in _prepare_deepspeed\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1815, in _prepare_deepspeed\u001b[0m\n",
      "\u001b[34mresult = self._prepare_deepspeed(*args)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1815, in _prepare_deepspeed\u001b[0m\n",
      "\u001b[34mengine, optimizer, _, lr_scheduler = deepspeed.initialize(**kwargs)    \n",
      "        engine, optimizer, _, lr_scheduler = deepspeed.initialize(**kwargs)engine, optimizer, _, lr_scheduler = deepspeed.initialize(**kwargs)engine, optimizer, _, lr_scheduler = deepspeed.initialize(**kwargs)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/__init__.py\", line 193, in initialize\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/deepspeed/__init__.py\", line 193, in initialize\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/__init__.py\", line 193, in initialize\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/__init__.py\", line 193, in initialize\u001b[0m\n",
      "\u001b[34mengine = DeepSpeedEngine(args=args,engine = DeepSpeedEngine(args=args,    engine = DeepSpeedEngine(args=args,\u001b[0m\n",
      "\u001b[34mengine = DeepSpeedEngine(args=args,\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 313, in __init__\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 313, in __init__\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 313, in __init__\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 313, in __init__\n",
      "    engine, optimizer, _, lr_scheduler = deepspeed.initialize(**kwargs)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/__init__.py\", line 193, in initialize\u001b[0m\n",
      "\u001b[34mengine = DeepSpeedEngine(args=args,\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 313, in __init__\n",
      "                self._configure_optimizer(optimizer, model_parameters)self._configure_optimizer(optimizer, model_parameters)self._configure_optimizer(optimizer, model_parameters)self._configure_optimizer(optimizer, model_parameters)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1276, in _configure_optimizer\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1276, in _configure_optimizer\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1276, in _configure_optimizer\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1276, in _configure_optimizer\u001b[0m\n",
      "\u001b[34mself._configure_optimizer(optimizer, model_parameters)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1276, in _configure_optimizer\n",
      "        model, self.optimizer, self.lr_scheduler = self.accelerator.prepare(model, self.optimizer, self.lr_scheduler = self.accelerator.prepare(\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1318, in prepare\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1318, in prepare\u001b[0m\n",
      "\u001b[34mbasic_optimizer = self._configure_basic_optimizer(model_parameters)    basic_optimizer = self._configure_basic_optimizer(model_parameters)basic_optimizer = self._configure_basic_optimizer(model_parameters)\u001b[0m\n",
      "\u001b[34mbasic_optimizer = self._configure_basic_optimizer(model_parameters)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1353, in _configure_basic_optimizer\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1353, in _configure_basic_optimizer\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1353, in _configure_basic_optimizer\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1353, in _configure_basic_optimizer\u001b[0m\n",
      "\u001b[34mbasic_optimizer = self._configure_basic_optimizer(model_parameters)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1353, in _configure_basic_optimizer\u001b[0m\n",
      "\u001b[34moptimizer = FusedAdam(\n",
      "    optimizer = FusedAdam(  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/adam/fused_adam.py\", line 94, in __init__\u001b[0m\n",
      "\u001b[34moptimizer = FusedAdam(optimizer = FusedAdam(  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/adam/fused_adam.py\", line 94, in __init__\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/adam/fused_adam.py\", line 94, in __init__\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/adam/fused_adam.py\", line 94, in __init__\u001b[0m\n",
      "\u001b[34moptimizer = FusedAdam(\n",
      "    fused_adam_cuda = FusedAdamBuilder().load()      File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/adam/fused_adam.py\", line 94, in __init__\n",
      "    \u001b[0m\n",
      "\u001b[34mfused_adam_cuda = FusedAdamBuilder().load()    fused_adam_cuda = FusedAdamBuilder().load()\u001b[0m\n",
      "\u001b[34mfused_adam_cuda = FusedAdamBuilder().load()\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 531, in load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 531, in load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 531, in load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 531, in load\u001b[0m\n",
      "\u001b[34mfused_adam_cuda = FusedAdamBuilder().load()        \u001b[0m\n",
      "\u001b[34mresult = self._prepare_deepspeed(*args)result = self._prepare_deepspeed(*args)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 531, in load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1815, in _prepare_deepspeed\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py\", line 1815, in _prepare_deepspeed\n",
      "                return self.jit_load(verbose)return self.jit_load(verbose)return self.jit_load(verbose)return self.jit_load(verbose)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 578, in jit_load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 578, in jit_load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 578, in jit_load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 578, in jit_load\u001b[0m\n",
      "\u001b[34mreturn self.jit_load(verbose)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 578, in jit_load\n",
      "            op_module = load(name=self.name,    op_module = load(name=self.name,op_module = load(name=self.name,\u001b[0m\n",
      "\u001b[34mop_module = load(name=self.name,\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1284, in load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1284, in load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1284, in load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1284, in load\n",
      "    op_module = load(name=self.name,\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1284, in load\u001b[0m\n",
      "\u001b[34mreturn _jit_compile(return _jit_compile(return _jit_compile(return _jit_compile(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1535, in _jit_compile\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1535, in _jit_compile\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1535, in _jit_compile\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1535, in _jit_compile\n",
      "    return _jit_compile(\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1535, in _jit_compile\u001b[0m\n",
      "\u001b[34mengine, optimizer, _, lr_scheduler = deepspeed.initialize(**kwargs)engine, optimizer, _, lr_scheduler = deepspeed.initialize(**kwargs)\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/deepspeed/__init__.py\", line 193, in initialize\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/__init__.py\", line 193, in initialize\u001b[0m\n",
      "\u001b[34mreturn _import_module_from_library(name, build_directory, is_python_module)    return _import_module_from_library(name, build_directory, is_python_module)return _import_module_from_library(name, build_directory, is_python_module)    \u001b[0m\n",
      "\u001b[34mreturn _import_module_from_library(name, build_directory, is_python_module)\u001b[0m\n",
      "\u001b[34mreturn _import_module_from_library(name, build_directory, is_python_module)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1929, in _import_module_from_library\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1929, in _import_module_from_library\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1929, in _import_module_from_library\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1929, in _import_module_from_library\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1929, in _import_module_from_library\u001b[0m\n",
      "\u001b[34mengine = DeepSpeedEngine(args=args,engine = DeepSpeedEngine(args=args,\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 313, in __init__\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 313, in __init__\u001b[0m\n",
      "\u001b[34mself._configure_optimizer(optimizer, model_parameters)self._configure_optimizer(optimizer, model_parameters)\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1276, in _configure_optimizer\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1276, in _configure_optimizer\u001b[0m\n",
      "\u001b[34mmodule = importlib.util.module_from_spec(spec)\n",
      "    module = importlib.util.module_from_spec(spec)  File \"<frozen importlib._bootstrap>\", line 571, in module_from_spec\u001b[0m\n",
      "\u001b[34mFile \"<frozen importlib._bootstrap>\", line 571, in module_from_spec\u001b[0m\n",
      "\u001b[34mmodule = importlib.util.module_from_spec(spec)module = importlib.util.module_from_spec(spec)module = importlib.util.module_from_spec(spec)\n",
      "  File \"<frozen importlib._bootstrap>\", line 571, in module_from_spec\n",
      "  File \"<frozen importlib._bootstrap>\", line 571, in module_from_spec\n",
      "  File \"<frozen importlib._bootstrap>\", line 571, in module_from_spec\u001b[0m\n",
      "\u001b[34mbasic_optimizer = self._configure_basic_optimizer(model_parameters)basic_optimizer = self._configure_basic_optimizer(model_parameters)\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1353, in _configure_basic_optimizer\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py\", line 1353, in _configure_basic_optimizer\u001b[0m\n",
      "\u001b[34mFile \"<frozen importlib._bootstrap_external>\", line 1176, in create_module\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1176, in create_module\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1176, in create_module\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1176, in create_module\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1176, in create_module\u001b[0m\n",
      "\u001b[34mFile \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
      "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
      "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
      "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
      "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\u001b[0m\n",
      "\u001b[34mImportErrorImportErrorImportErrorImportErrorImportError: : : : /root/.cache/torch_extensions/py310_cu118/fused_adam/fused_adam.so: cannot open shared object file: No such file or directory/root/.cache/torch_extensions/py310_cu118/fused_adam/fused_adam.so: cannot open shared object file: No such file or directory: /root/.cache/torch_extensions/py310_cu118/fused_adam/fused_adam.so: cannot open shared object file: No such file or directory/root/.cache/torch_extensions/py310_cu118/fused_adam/fused_adam.so: cannot open shared object file: No such file or directory\u001b[0m\n",
      "\u001b[34m/root/.cache/torch_extensions/py310_cu118/fused_adam/fused_adam.so: cannot open shared object file: No such file or directory\u001b[0m\n",
      "\u001b[34moptimizer = FusedAdam(\n",
      "    optimizer = FusedAdam(\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/adam/fused_adam.py\", line 94, in __init__\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/adam/fused_adam.py\", line 94, in __init__\u001b[0m\n",
      "\u001b[34mfused_adam_cuda = FusedAdamBuilder().load()fused_adam_cuda = FusedAdamBuilder().load()\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 531, in load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 531, in load\u001b[0m\n",
      "\u001b[34mreturn self.jit_load(verbose)return self.jit_load(verbose)\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 578, in jit_load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py\", line 578, in jit_load\u001b[0m\n",
      "\u001b[34mop_module = load(name=self.name,op_module = load(name=self.name,\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1284, in load\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1284, in load\u001b[0m\n",
      "\u001b[34mreturn _jit_compile(return _jit_compile(\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1535, in _jit_compile\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1535, in _jit_compile\u001b[0m\n",
      "\u001b[34mreturn _import_module_from_library(name, build_directory, is_python_module)return _import_module_from_library(name, build_directory, is_python_module)\u001b[0m\n",
      "\u001b[34mFile \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1929, in _import_module_from_library\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/torch/utils/cpp_extension.py\", line 1929, in _import_module_from_library\u001b[0m\n",
      "\u001b[34mmodule = importlib.util.module_from_spec(spec)module = importlib.util.module_from_spec(spec)\u001b[0m\n",
      "\u001b[34mFile \"<frozen importlib._bootstrap>\", line 571, in module_from_spec\n",
      "  File \"<frozen importlib._bootstrap>\", line 571, in module_from_spec\u001b[0m\n",
      "\u001b[34mFile \"<frozen importlib._bootstrap_external>\", line 1176, in create_module\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1176, in create_module\u001b[0m\n",
      "\u001b[34mFile \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
      "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\u001b[0m\n",
      "\u001b[34mImportErrorImportError: : /root/.cache/torch_extensions/py310_cu118/fused_adam/fused_adam.so: cannot open shared object file: No such file or directory/root/.cache/torch_extensions/py310_cu118/fused_adam/fused_adam.so: cannot open shared object file: No such file or directory\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:33:50,372] [INFO] [launch.py:319:sigkill_handler] Killing subprocess 305\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:33:50,547] [INFO] [launch.py:319:sigkill_handler] Killing subprocess 306\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:33:51,120] [INFO] [launch.py:319:sigkill_handler] Killing subprocess 307\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:33:51,121] [INFO] [launch.py:319:sigkill_handler] Killing subprocess 308\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:33:51,122] [INFO] [launch.py:319:sigkill_handler] Killing subprocess 309\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:33:51,122] [INFO] [launch.py:319:sigkill_handler] Killing subprocess 310\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:33:51,123] [INFO] [launch.py:319:sigkill_handler] Killing subprocess 311\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:33:51,124] [INFO] [launch.py:319:sigkill_handler] Killing subprocess 312\u001b[0m\n",
      "\u001b[34m[2024-09-26 10:33:51,124] [ERROR] [launch.py:325:sigkill_handler] ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=7', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/opt/ml/model/adgen-chatglm2-6b-ft', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16'] exits with return code = 1\u001b[0m\n",
      "\u001b[34m2024-09-26 10:33:52,316 sagemaker-training-toolkit INFO     Waiting for the process to finish and give a return code.\u001b[0m\n",
      "\u001b[34m2024-09-26 10:33:52,316 sagemaker-training-toolkit INFO     Done waiting for a return code. Received 0 from exiting process.\u001b[0m\n",
      "\u001b[34m2024-09-26 10:33:52,317 sagemaker-training-toolkit INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2024-09-26 10:34:06 Uploading - Uploading generated training model\n",
      "2024-09-26 10:34:06 Completed - Training job completed\n",
      "Training seconds: 637\n",
      "Billable seconds: 637\n"
     ]
    }
   ],
   "source": [
    "huggingface_estimator.fit(inputs=inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b48c6d6-b465-4cd7-bbc7-3ca3d2951dbb",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 4. 模型部署"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc7b753e-a6b8-4bba-bff7-b094e1fce67c",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 4.1 获取Runtime资源配置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d74b206e-6cd0-4b0c-8a91-0d924699f5c0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sagemaker Execution Role:\n",
      "arn:aws:iam::687912291502:role/service-role/AmazonSageMaker-ExecutionRole-20211013T113123\n",
      "Sagemaker Default Bucket:\n",
      "sagemaker-us-west-2-687912291502\n",
      "Sagemaker Boto Account:\n",
      "687912291502\n",
      "Sagemaker Boto Region:\n",
      "us-west-2\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "from sagemaker import get_execution_role\n",
    "\n",
    "sess                     = sagemaker.Session()\n",
    "role                     = get_execution_role()\n",
    "sagemaker_default_bucket = sess.default_bucket()\n",
    "\n",
    "account                  = sess.boto_session.client(\"sts\").get_caller_identity()[\"Account\"]\n",
    "region                   = sess.boto_session.region_name\n",
    "\n",
    "print(\"Sagemaker Execution Role:\")\n",
    "print(role)\n",
    "print(\"Sagemaker Default Bucket:\")\n",
    "print(sagemaker_default_bucket)\n",
    "print(\"Sagemaker Boto Account:\")\n",
    "print(account)\n",
    "print(\"Sagemaker Boto Region:\")\n",
    "print(region)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "308a1edc-5090-4570-b3a6-0691b6304f50",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 4.2 准备Dummy模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5da1ff4d-cc76-42fc-a711-8f822aed6a75",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dummy\n",
      "upload: ./model.tar.gz to s3://sagemaker-us-west-2-687912291502/chatglm2/assets/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "!touch dummy\n",
    "!tar czvf model.tar.gz dummy\n",
    "assets_dir = 's3://{0}/{1}/assets/'.format(sagemaker_default_bucket, 'chatglm2')\n",
    "model_data = 's3://{0}/{1}/assets/model.tar.gz'.format(sagemaker_default_bucket, 'chatglm2')\n",
    "!aws s3 cp model.tar.gz $assets_dir\n",
    "!rm -f dummy model.tar.gz"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5d844ef-4b13-4ba9-add1-26d0c1913459",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 4.3 配置模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00d2c840-4ed1-4874-84c4-6c0309f3c552",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_name                  = None\n",
    "entry_point                 = 'chatglm2-inference-finetune.py'\n",
    "framework_version           = '2.0'\n",
    "py_version                  = 'py310'\n",
    "base_model_name_or_path     = 's3://{}/llm/models/chatglm2/original-6B/'.format(sagemaker_default_bucket)\n",
    "finetune_model_name_or_path = 's3://{}/llm/models/chatglm2/finetune-ptuning-adgen/adgen-chatglm2-6b-ft/checkpoint-1000/pytorch_model.bin'.format(sagemaker_default_bucket)\n",
    "\n",
    "# 模型环境变量设置\n",
    "model_environment  = {\n",
    "    'SAGEMAKER_MODEL_SERVER_TIMEOUT': '600',\n",
    "    'SAGEMAKER_MODEL_SERVER_WORKERS': '1',\n",
    "    'MODEL_NAME_OR_PATH'            : base_model_name_or_path,\n",
    "    'FINETUNE_MODEL_NAME_OR_PATH'   : finetune_model_name_or_path,\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "076681b6-3064-4dd5-b9ca-83cedab9da2e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.pytorch.model import PyTorchModel\n",
    "\n",
    "model = PyTorchModel(\n",
    "    name              = model_name,\n",
    "    model_data        = model_data,\n",
    "    entry_point       = entry_point,\n",
    "    source_dir        = './code',\n",
    "    role              = role,\n",
    "    framework_version = framework_version, \n",
    "    py_version        = py_version,\n",
    "    env               = model_environment\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13077f97-3549-46bf-b175-aee0aa050c6a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "! chmod +x /usr/local/bin/s5cmd\n",
    "! cp /usr/local/bin/s5cmd code/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae07f38b-b216-49be-ae5b-b666315cfbcf",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## BYOC 方式打包训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "7d7fcf1f-1962-4d69-8c98-9273201e55a8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting Dockerfile\n"
     ]
    }
   ],
   "source": [
    "%%writefile Dockerfile\n",
    "## You should change below region code to the region you used, here sample is use us-west-2\n",
    "#From 763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04 \n",
    "From 763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:2.0.0-gpu-py310-cu118-ubuntu20.04-sagemaker\n",
    "#From pytorch/pytorch:1.5-cuda10.1-cudnn7-runtime\n",
    "\n",
    "ENV LANG=C.UTF-8\n",
    "ENV PYTHONUNBUFFERED=TRUE\n",
    "ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
    "\n",
    "#RUN pip3 uninstall -y deepspeed && pip3 install deepspeed\n",
    "\n",
    "## Make all local GPUs visible\n",
    "ENV NVIDIA_VISIBLE_DEVICES=\"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "3b8ee553",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n",
      "Login Succeeded\n"
     ]
    }
   ],
   "source": [
    "## You should change below region code to the region you used, here sample is use us-west-2\n",
    "!aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-west-2.amazonaws.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3ccc2ba2-6cb2-4922-b71b-4930141fea6c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## define repo name, should contain *sagemaker* in the name\n",
    "repo_name = \"sagemaker-chatglm2-demo\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2af1be4f-a857-480f-8fd1-d7a708957742",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Login Succeeded\n",
      "Process is interrupted.\n"
     ]
    }
   ],
   "source": [
    "%%script env repo_name=$repo_name bash\n",
    "\n",
    "#!/usr/bin/env bash\n",
    "\n",
    "# This script shows how to build the Docker image and push it to ECR to be ready for use\n",
    "# by SageMaker.\n",
    "\n",
    "# The argument to this script is the image name. This will be used as the image on the local\n",
    "# machine and combined with the account and region to form the repository name for ECR.\n",
    "# The name of our algorithm\n",
    "algorithm_name=${repo_name}\n",
    "\n",
    "account=$(aws sts get-caller-identity --query Account --output text)\n",
    "\n",
    "# Get the region defined in the current configuration (default to us-west-2 if none defined)\n",
    "region=$(aws configure get region)\n",
    "region=${region:-us-west-2}\n",
    "\n",
    "fullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest\"\n",
    "\n",
    "# If the repository doesn't exist in ECR, create it.\n",
    "aws ecr describe-repositories --repository-names \"${algorithm_name}\" > /dev/null 2>&1\n",
    "\n",
    "if [ $? -ne 0 ]\n",
    "then\n",
    "    aws ecr create-repository --repository-name \"${algorithm_name}\" > /dev/null\n",
    "fi\n",
    "\n",
    "# Get the login command from ECR and execute it directly\n",
    "aws ecr get-login-password --region ${region}|docker login --username AWS --password-stdin ${fullname}\n",
    "\n",
    "# Build the docker image locally with the image name and then push it to ECR\n",
    "# with the full name.\n",
    "\n",
    "docker build -t ${algorithm_name} .\n",
    "docker tag ${algorithm_name} ${fullname}\n",
    "\n",
    "docker push ${fullname}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9d3a109b-a905-4e6e-b0d8-c6aa720a2206",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./ChatGLM2-6B/ptuning/sm_fulltuning_train.sh\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./ChatGLM2-6B/ptuning/sm_fulltuning_train.sh\n",
    "#!/bin/bash\n",
    "#echo \"Y\"|pip uninstall deepspeed\n",
    "#pip install pydantic==1.9.2\n",
    "#pip3 install --force-reinstall torch  --index-url https://download.pytorch.org/whl/cu118\n",
    "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(dirname $(find / -name libcudart.so 2>/dev/null | head -n 1))\n",
    "#export DS_BUILD_FUSED_ADAM=1 && pip install deepspeed==0.9.5\n",
    "git clone https://github.com/microsoft/DeepSpeed.git\n",
    "cd DeepSpeed\n",
    "DS_BUILD_UTILS=1 DS_BUILD_FUSED_ADAM=1 pip install .\n",
    "cd .. && pip install -r ./requirements.txt\n",
    "LR=1e-4\n",
    "MASTER_PORT=\"23456\"\n",
    "\n",
    "deepspeed --num_gpus=$NUM_GPUS  --master_port $MASTER_PORT main_tuning.py \\\n",
    "    --deepspeed deepspeed.json \\\n",
    "    --do_train \\\n",
    "    --train_file $TRAIN_DATASET \\\n",
    "    --validation_file $TEST_DATASET \\\n",
    "    --prompt_column ${PROMPT_COLUMN} \\\n",
    "    --response_column ${RESPONSE_COLUMN} \\\n",
    "    --overwrite_cache \\\n",
    "    --model_name_or_path ${MODEL_NAME_OR_PATH}   \\\n",
    "    --output_dir ${OUTPUT_DIR} \\\n",
    "    --model_output_s3_path ${MODEL_OUTPUT_S3_PATH} \\\n",
    "    --overwrite_output_dir \\\n",
    "    --max_source_length 64 \\\n",
    "    --max_target_length 64 \\\n",
    "    --per_device_train_batch_size 1 \\\n",
    "    --per_device_eval_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 1 \\\n",
    "    --predict_with_generate \\\n",
    "    --max_steps ${TRAIN_STEPS} \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps ${TRAIN_STEPS} \\\n",
    "    --learning_rate $LR \\\n",
    "    --fp16\n",
    "\n",
    "#./s5cmd sync /tmp/output/ s3://$MODEL_S3_BUCKET/models/chatglm-lora/output/$(date +%Y-%m-%d-%H-%M-%S)/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2d52dee8-ef93-45a9-a906-91a91ba3b7a4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ChatGLM2-6B/ptuning/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile ChatGLM2-6B/ptuning/requirements.txt\n",
    "protobuf\n",
    "#git+https://github.com/huggingface/transformers.git@68d640f7c368bcaaaecfc678f11908ebbd3d6176\n",
    "transformers==4.30.1\n",
    "cpm_kernels\n",
    "#torch>=2.0\n",
    "gradio\n",
    "mdtex2html\n",
    "sentencepiece\n",
    "accelerate==0.23.0\n",
    "sse-starlette\n",
    "streamlit>=1.24.0\n",
    "datasets\n",
    "huggingface\n",
    "jieba\n",
    "rouge_chinese\n",
    "nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "22461ff1-58bc-400a-a6a0-fe16b0848fb8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define Training Job Name\n",
    "import time\n",
    "from sagemaker.huggingface import HuggingFace\n",
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "job_name = f'huggingface-chatglm-finetune-fulltuning-{time.strftime(\"%Y-%m-%d-%H-%M-%S\", time.localtime())}'\n",
    "\n",
    "instance_type  = \"ml.p4d.24xlarge\"\n",
    "#instance_type  = \"ml.g5.48xlarge\"\n",
    "instance_count = 1\n",
    "processes_per_host = 8\n",
    "\n",
    "# 基础模型存放地址\n",
    "model_name_or_path = 's3://{}/llm/models/chatglm2/original-6B/'.format(sagemaker_default_bucket)\n",
    "\n",
    "# 微调模型输出地址\n",
    "#output_dir         = '/opt/ml/model/adgen-chatglm2-6b-ft'\n",
    "output_dir         = '/tmp/model/adgen-chatglm2-6b-ft/'\n",
    "model_s3_path      = 's3://{}/llm/models/chatglm2/finetune-fulltuning-adgen/'.format(sagemaker_default_bucket)\n",
    "\n",
    "# 模型环境变量设置\n",
    "environment = {\n",
    "    'PYTORCH_CUDA_ALLOC_CONF': 'max_split_size_mb:32',\n",
    "    'NUM_GPUS'               : str(processes_per_host),\n",
    "    'TRAIN_DATASET'          : '/opt/ml/input/data/KuaFuTrainData/train.json',\n",
    "    'TEST_DATASET'           : '/opt/ml/input/data/KuaFuTrainData/dev.json',\n",
    "    'PROMPT_COLUMN'          : 'content',\n",
    "    'RESPONSE_COLUMN'        : 'summary',\n",
    "    'MODEL_NAME_OR_PATH'     : model_name_or_path,\n",
    "    'OUTPUT_DIR'             : output_dir,\n",
    "    'MODEL_OUTPUT_S3_PATH'   : model_s3_path,\n",
    "    'TRAIN_STEPS'            : '50'\n",
    "}\n",
    "\n",
    "inputs = {\n",
    "   'KuaFuTrainData': f\"s3://{sagemaker_default_bucket}/llm/datasets/chatglm2/KuaFuTrainData/\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "537494d7-b91c-459b-9c87-3b5494a756e2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'687912291502.dkr.ecr.us-west-2.amazonaws.com/sagemaker-chatglm2-demo:latest'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## The image uri which is build and pushed above\n",
    "image_uri = \"{}.dkr.ecr.{}.amazonaws.com/{}:latest\".format(account, region, repo_name)\n",
    "image_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f75ef7f6-6692-4a0e-a8d7-22c894ef4441",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# create the Estimator\n",
    "huggingface_estimator = Estimator(\n",
    "    entry_point          = 'sm_fulltuning_train.sh',\n",
    "    source_dir           = './ChatGLM2-6B/ptuning',\n",
    "    image_uri            = image_uri,\n",
    "    instance_type        = instance_type,\n",
    "    instance_count       = instance_count,\n",
    "    base_job_name        = job_name,\n",
    "    role                 = role,\n",
    "    environment          = environment\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ef6b2d64-7882-474c-b43c-d9778f237d31",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating training-job with name: huggingface-chatglm-finetune-fulltuning-2024-09-28-13-51-23-173\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-09-28 13:51:25 Starting - Starting the training job\n",
      "2024-09-28 13:51:25 Pending - Training job waiting for capacity......\n",
      "2024-09-28 13:52:18 Pending - Preparing the instances for training..............................\n",
      "2024-09-28 13:57:12 Downloading - Downloading input data...\n",
      "2024-09-28 13:57:37 Downloading - Downloading the training image.....................\n",
      "2024-09-28 14:01:19 Training - Training image download completed. Training in progress......\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2024-09-28 14:02:11,425 sagemaker-training-toolkit INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2024-09-28 14:02:11,538 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-09-28 14:02:11,545 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2024-09-28 14:02:11,547 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2024-09-28 14:02:16,234 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-09-28 14:02:16,378 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-09-28 14:02:16,493 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-09-28 14:02:16,501 sagemaker-training-toolkit INFO     Invoking user script\u001b[0m\n",
      "\u001b[34mTraining Env:\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {\n",
      "        \"KuaFuTrainData\": \"/opt/ml/input/data/KuaFuTrainData\"\n",
      "    },\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"current_instance_group\": \"homogeneousCluster\",\n",
      "    \"current_instance_group_hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "    \"distribution_hosts\": [],\n",
      "    \"distribution_instance_groups\": [],\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {},\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {\n",
      "        \"KuaFuTrainData\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        }\n",
      "    },\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"instance_groups\": [\n",
      "        \"homogeneousCluster\"\n",
      "    ],\n",
      "    \"instance_groups_dict\": {\n",
      "        \"homogeneousCluster\": {\n",
      "            \"instance_group_name\": \"homogeneousCluster\",\n",
      "            \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "            \"hosts\": [\n",
      "                \"algo-1\"\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"is_hetero\": false,\n",
      "    \"is_master\": true,\n",
      "    \"is_modelparallel_enabled\": null,\n",
      "    \"is_smddpmprun_installed\": true,\n",
      "    \"job_name\": \"huggingface-chatglm-finetune-fulltuning-2024-09-28-13-51-23-173\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-us-west-2-687912291502/huggingface-chatglm-finetune-fulltuning-2024-09-28-13-51-23-173/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"sm_fulltuning_train.sh\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 96,\n",
      "    \"num_gpus\": 8,\n",
      "    \"num_neurons\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"sm_fulltuning_train.sh\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mEnvironment variables:\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=sm_fulltuning_train.sh\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={\"KuaFuTrainData\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[\"KuaFuTrainData\"]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_TYPE=ml.p4d.24xlarge\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP=homogeneousCluster\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}}\u001b[0m\n",
      "\u001b[34mSM_DISTRIBUTION_INSTANCE_GROUPS=[]\u001b[0m\n",
      "\u001b[34mSM_IS_HETERO=false\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=sm_fulltuning_train.sh\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=96\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=8\u001b[0m\n",
      "\u001b[34mSM_NUM_NEURONS=0\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-west-2-687912291502/huggingface-chatglm-finetune-fulltuning-2024-09-28-13-51-23-173/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"KuaFuTrainData\":\"/opt/ml/input/data/KuaFuTrainData\"},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.p4d.24xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"KuaFuTrainData\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":true,\"job_name\":\"huggingface-chatglm-finetune-fulltuning-2024-09-28-13-51-23-173\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-west-2-687912291502/huggingface-chatglm-finetune-fulltuning-2024-09-28-13-51-23-173/source/sourcedir.tar.gz\",\"module_name\":\"sm_fulltuning_train.sh\",\"network_interface_name\":\"eth0\",\"num_cpus\":96,\"num_gpus\":8,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"sm_fulltuning_train.sh\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_KUAFUTRAINDATA=/opt/ml/input/data/KuaFuTrainData\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python310.zip:/opt/conda/lib/python3.10:/opt/conda/lib/python3.10/lib-dynload:/opt/conda/lib/python3.10/site-packages\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\u001b[0m\n",
      "\u001b[34m/bin/sh -c \"./sm_fulltuning_train.sh \"\u001b[0m\n",
      "\u001b[34m2024-09-28 14:02:16,529 sagemaker-training-toolkit INFO     Exceptions not imported for SageMaker TF as Tensorflow is not installed.\u001b[0m\n",
      "\u001b[34mCloning into 'DeepSpeed'...\u001b[0m\n",
      "\u001b[34mProcessing /opt/ml/code/DeepSpeed\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: hjson in /opt/conda/lib/python3.10/site-packages (from deepspeed==0.15.2+unknown) (3.1.0)\u001b[0m\n",
      "\u001b[34mCollecting msgpack (from deepspeed==0.15.2+unknown)\u001b[0m\n",
      "\u001b[34mDownloading msgpack-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (378 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 378.0/378.0 kB 9.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: ninja in /opt/conda/lib/python3.10/site-packages (from deepspeed==0.15.2+unknown) (1.11.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from deepspeed==0.15.2+unknown) (1.24.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from deepspeed==0.15.2+unknown) (23.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from deepspeed==0.15.2+unknown) (5.9.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: py-cpuinfo in /opt/conda/lib/python3.10/site-packages (from deepspeed==0.15.2+unknown) (9.0.0)\u001b[0m\n",
      "\u001b[34mCollecting pydantic>=2.0.0 (from deepspeed==0.15.2+unknown)\u001b[0m\n",
      "\u001b[34mDownloading pydantic-2.9.2-py3-none-any.whl (434 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 434.9/434.9 kB 33.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torch in /opt/conda/lib/python3.10/site-packages (from deepspeed==0.15.2+unknown) (2.0.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from deepspeed==0.15.2+unknown) (4.65.0)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-ml-py (from deepspeed==0.15.2+unknown)\u001b[0m\n",
      "\u001b[34mDownloading nvidia_ml_py-12.560.30-py3-none-any.whl (40 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 40.5/40.5 kB 16.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting annotated-types>=0.6.0 (from pydantic>=2.0.0->deepspeed==0.15.2+unknown)\u001b[0m\n",
      "\u001b[34mDownloading annotated_types-0.7.0-py3-none-any.whl (13 kB)\u001b[0m\n",
      "\u001b[34mCollecting pydantic-core==2.23.4 (from pydantic>=2.0.0->deepspeed==0.15.2+unknown)\u001b[0m\n",
      "\u001b[34mDownloading pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 75.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typing-extensions>=4.6.1 in /opt/conda/lib/python3.10/site-packages (from pydantic>=2.0.0->deepspeed==0.15.2+unknown) (4.7.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch->deepspeed==0.15.2+unknown) (3.12.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch->deepspeed==0.15.2+unknown) (1.12)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch->deepspeed==0.15.2+unknown) (3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch->deepspeed==0.15.2+unknown) (3.1.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch->deepspeed==0.15.2+unknown) (2.1.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch->deepspeed==0.15.2+unknown) (1.3.0)\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: deepspeed\u001b[0m\n",
      "\u001b[34mBuilding wheel for deepspeed (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for deepspeed (setup.py): still running...\u001b[0m\n",
      "\u001b[34mBuilding wheel for deepspeed (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for deepspeed: filename=deepspeed-0.15.2+unknown-cp310-cp310-linux_x86_64.whl size=4955265 sha256=94a93c4e372cf8e85cdd1d50ee31adae82bed488400c7d39cddf7d994f0508b8\u001b[0m\n",
      "\u001b[34mStored in directory: /tmp/pip-ephem-wheel-cache-gfckkv74/wheels/07/45/f2/551a5f872960ae8a00b708660edb016f239da41474a044d5b3\u001b[0m\n",
      "\u001b[34mSuccessfully built deepspeed\u001b[0m\n",
      "\u001b[34mInstalling collected packages: nvidia-ml-py, pydantic-core, msgpack, annotated-types, pydantic, deepspeed\u001b[0m\n",
      "\u001b[34mAttempting uninstall: pydantic\u001b[0m\n",
      "\u001b[34mFound existing installation: pydantic 1.10.11\u001b[0m\n",
      "\u001b[34mUninstalling pydantic-1.10.11:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled pydantic-1.10.11\u001b[0m\n",
      "\u001b[34mAttempting uninstall: deepspeed\u001b[0m\n",
      "\u001b[34mFound existing installation: deepspeed 0.6.1+1ea3d4b\u001b[0m\n",
      "\u001b[34mUninstalling deepspeed-0.6.1+1ea3d4b:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled deepspeed-0.6.1+1ea3d4b\u001b[0m\n",
      "\u001b[34mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\u001b[0m\n",
      "\u001b[34mconfection 0.1.0 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mspacy 3.6.0 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mthinc 8.1.10 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mSuccessfully installed annotated-types-0.7.0 deepspeed-0.15.2+unknown msgpack-1.1.0 nvidia-ml-py-12.560.30 pydantic-2.9.2 pydantic-core-2.23.4\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[34m[notice] A new release of pip is available: 23.1.2 -> 24.2\u001b[0m\n",
      "\u001b[34m[notice] To update, run: pip install --upgrade pip\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: protobuf in /opt/conda/lib/python3.10/site-packages (from -r ./requirements.txt (line 1)) (3.20.3)\u001b[0m\n",
      "\u001b[34mCollecting transformers==4.30.1 (from -r ./requirements.txt (line 3))\u001b[0m\n",
      "\u001b[34mDownloading transformers-4.30.1-py3-none-any.whl (7.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.2/7.2 MB 67.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting cpm_kernels (from -r ./requirements.txt (line 4))\u001b[0m\n",
      "\u001b[34mDownloading cpm_kernels-1.0.11-py3-none-any.whl (416 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 416.6/416.6 kB 78.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting gradio (from -r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading gradio-4.44.0-py3-none-any.whl (18.1 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 18.1/18.1 MB 73.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting mdtex2html (from -r ./requirements.txt (line 7))\u001b[0m\n",
      "\u001b[34mDownloading mdtex2html-1.3.0-py3-none-any.whl (13 kB)\u001b[0m\n",
      "\u001b[34mCollecting sentencepiece (from -r ./requirements.txt (line 8))\u001b[0m\n",
      "\u001b[34mDownloading sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 45.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting accelerate==0.23.0 (from -r ./requirements.txt (line 9))\u001b[0m\n",
      "\u001b[34mDownloading accelerate-0.23.0-py3-none-any.whl (258 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 258.1/258.1 kB 21.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting sse-starlette (from -r ./requirements.txt (line 10))\u001b[0m\n",
      "\u001b[34mDownloading sse_starlette-2.1.3-py3-none-any.whl (9.4 kB)\u001b[0m\n",
      "\u001b[34mCollecting streamlit>=1.24.0 (from -r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading streamlit-1.38.0-py2.py3-none-any.whl (8.7 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 8.7/8.7 MB 86.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting datasets (from -r ./requirements.txt (line 12))\u001b[0m\n",
      "\u001b[34mDownloading datasets-3.0.1-py3-none-any.whl (471 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 471.6/471.6 kB 52.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting huggingface (from -r ./requirements.txt (line 13))\u001b[0m\n",
      "\u001b[34mDownloading huggingface-0.0.1-py3-none-any.whl (2.5 kB)\u001b[0m\n",
      "\u001b[34mCollecting jieba (from -r ./requirements.txt (line 14))\u001b[0m\n",
      "\u001b[34mDownloading jieba-0.42.1.tar.gz (19.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 19.2/19.2 MB 75.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting rouge_chinese (from -r ./requirements.txt (line 15))\u001b[0m\n",
      "\u001b[34mDownloading rouge_chinese-1.0.3-py3-none-any.whl (21 kB)\u001b[0m\n",
      "\u001b[34mCollecting nltk (from -r ./requirements.txt (line 16))\u001b[0m\n",
      "\u001b[34mDownloading nltk-3.9.1-py3-none-any.whl (1.5 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB 130.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (3.12.2)\u001b[0m\n",
      "\u001b[34mCollecting huggingface-hub<1.0,>=0.14.1 (from transformers==4.30.1->-r ./requirements.txt (line 3))\u001b[0m\n",
      "\u001b[34mDownloading huggingface_hub-0.25.1-py3-none-any.whl (436 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 436.4/436.4 kB 85.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (1.24.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (23.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (6.0)\u001b[0m\n",
      "\u001b[34mCollecting regex!=2019.12.17 (from transformers==4.30.1->-r ./requirements.txt (line 3))\u001b[0m\n",
      "\u001b[34mDownloading regex-2024.9.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (782 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 782.7/782.7 kB 103.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (2.31.0)\u001b[0m\n",
      "\u001b[34mCollecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers==4.30.1->-r ./requirements.txt (line 3))\u001b[0m\n",
      "\u001b[34mDownloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.8/7.8 MB 109.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting safetensors>=0.3.1 (from transformers==4.30.1->-r ./requirements.txt (line 3))\u001b[0m\n",
      "\u001b[34mDownloading safetensors-0.4.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (435 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 435.0/435.0 kB 89.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.1->-r ./requirements.txt (line 3)) (4.65.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate==0.23.0->-r ./requirements.txt (line 9)) (5.9.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.23.0->-r ./requirements.txt (line 9)) (2.0.0)\u001b[0m\n",
      "\u001b[34mCollecting aiofiles<24.0,>=22.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\u001b[0m\n",
      "\u001b[34mCollecting anyio<5.0,>=3.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading anyio-4.6.0-py3-none-any.whl (89 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 89.6/89.6 kB 33.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting fastapi<1.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading fastapi-0.115.0-py3-none-any.whl (94 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 94.6/94.6 kB 34.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting ffmpy (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading ffmpy-0.4.0-py3-none-any.whl (5.8 kB)\u001b[0m\n",
      "\u001b[34mCollecting gradio-client==1.3.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading gradio_client-1.3.0-py3-none-any.whl (318 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 318.7/318.7 kB 74.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting httpx>=0.24.1 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading httpx-0.27.2-py3-none-any.whl (76 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 76.4/76.4 kB 34.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting importlib-resources<7.0,>=1.3 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading importlib_resources-6.4.5-py3-none-any.whl (36 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jinja2<4.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (3.1.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markupsafe~=2.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (2.1.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: matplotlib~=3.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (3.7.2)\u001b[0m\n",
      "\u001b[34mCollecting orjson~=3.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (141 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 141.9/141.9 kB 39.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pandas<3.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (2.0.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pillow<11.0,>=8.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (10.0.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pydantic>=2.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (2.9.2)\u001b[0m\n",
      "\u001b[34mCollecting pydub (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\u001b[0m\n",
      "\u001b[34mCollecting python-multipart>=0.0.9 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading python_multipart-0.0.11-py3-none-any.whl (22 kB)\u001b[0m\n",
      "\u001b[34mCollecting ruff>=0.2.2 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading ruff-0.6.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.9 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.9/10.9 MB 126.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting semantic-version~=2.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\u001b[0m\n",
      "\u001b[34mCollecting tomlkit==0.12.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading tomlkit-0.12.0-py3-none-any.whl (37 kB)\u001b[0m\n",
      "\u001b[34mCollecting typer<1.0,>=0.12 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading typer-0.12.5-py3-none-any.whl (47 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 47.3/47.3 kB 18.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typing-extensions~=4.0 in /opt/conda/lib/python3.10/site-packages (from gradio->-r ./requirements.txt (line 6)) (4.7.1)\u001b[0m\n",
      "\u001b[34mCollecting urllib3~=2.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading urllib3-2.2.3-py3-none-any.whl (126 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 126.3/126.3 kB 43.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting uvicorn>=0.14.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading uvicorn-0.31.0-py3-none-any.whl (63 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 63.7/63.7 kB 26.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from gradio-client==1.3.0->gradio->-r ./requirements.txt (line 6)) (2023.6.0)\u001b[0m\n",
      "\u001b[34mCollecting websockets<13.0,>=10.0 (from gradio-client==1.3.0->gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 130.2/130.2 kB 44.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting markdown (from mdtex2html->-r ./requirements.txt (line 7))\u001b[0m\n",
      "\u001b[34mDownloading Markdown-3.7-py3-none-any.whl (106 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 106.3/106.3 kB 31.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting latex2mathml (from mdtex2html->-r ./requirements.txt (line 7))\u001b[0m\n",
      "\u001b[34mDownloading latex2mathml-3.77.0-py3-none-any.whl (73 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 73.7/73.7 kB 30.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting starlette (from sse-starlette->-r ./requirements.txt (line 10))\u001b[0m\n",
      "\u001b[34mDownloading starlette-0.39.1-py3-none-any.whl (73 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 73.1/73.1 kB 32.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting altair<6,>=4.0 (from streamlit>=1.24.0->-r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading altair-5.4.1-py3-none-any.whl (658 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 658.1/658.1 kB 114.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting blinker<2,>=1.0.0 (from streamlit>=1.24.0->-r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading blinker-1.8.2-py3-none-any.whl (9.5 kB)\u001b[0m\n",
      "\u001b[34mCollecting cachetools<6,>=4.0 (from streamlit>=1.24.0->-r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading cachetools-5.5.0-py3-none-any.whl (9.5 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: click<9,>=7.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (8.1.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyarrow>=7.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (12.0.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: rich<14,>=10.14.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (13.4.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tenacity<9,>=8.1.0 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (8.2.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: toml<2,>=0.10.1 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (0.10.2)\u001b[0m\n",
      "\u001b[34mCollecting gitpython!=3.1.19,<4,>=3.0.7 (from streamlit>=1.24.0->-r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading GitPython-3.1.43-py3-none-any.whl (207 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 207.3/207.3 kB 63.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting pydeck<1,>=0.8.0b4 (from streamlit>=1.24.0->-r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading pydeck-0.9.1-py2.py3-none-any.whl (6.9 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 6.9/6.9 MB 141.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tornado<7,>=6.0.3 in /opt/conda/lib/python3.10/site-packages (from streamlit>=1.24.0->-r ./requirements.txt (line 11)) (6.3.2)\u001b[0m\n",
      "\u001b[34mCollecting watchdog<5,>=2.1.5 (from streamlit>=1.24.0->-r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading watchdog-4.0.2-py3-none-manylinux2014_x86_64.whl (82 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 82.9/82.9 kB 29.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting pyarrow>=7.0 (from streamlit>=1.24.0->-r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl (39.9 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 39.9/39.9 MB 30.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets->-r ./requirements.txt (line 12)) (0.3.6)\u001b[0m\n",
      "\u001b[34mCollecting requests (from transformers==4.30.1->-r ./requirements.txt (line 3))\u001b[0m\n",
      "\u001b[34mDownloading requests-2.32.3-py3-none-any.whl (64 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 64.9/64.9 kB 25.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting tqdm>=4.27 (from transformers==4.30.1->-r ./requirements.txt (line 3))\u001b[0m\n",
      "\u001b[34mDownloading tqdm-4.66.5-py3-none-any.whl (78 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 78.4/78.4 kB 29.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting xxhash (from datasets->-r ./requirements.txt (line 12))\u001b[0m\n",
      "\u001b[34mDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 194.1/194.1 kB 56.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets->-r ./requirements.txt (line 12)) (0.70.14)\u001b[0m\n",
      "\u001b[34mCollecting aiohttp (from datasets->-r ./requirements.txt (line 12))\u001b[0m\n",
      "\u001b[34mDownloading aiohttp-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.2/1.2 MB 113.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from rouge_chinese->-r ./requirements.txt (line 15)) (1.16.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: joblib in /opt/conda/lib/python3.10/site-packages (from nltk->-r ./requirements.txt (line 16)) (1.3.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jsonschema>=3.0 in /opt/conda/lib/python3.10/site-packages (from altair<6,>=4.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (4.18.3)\u001b[0m\n",
      "\u001b[34mCollecting narwhals>=1.5.2 (from altair<6,>=4.0->streamlit>=1.24.0->-r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading narwhals-1.8.4-py3-none-any.whl (174 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 174.1/174.1 kB 56.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting typing-extensions~=4.0 (from gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading typing_extensions-4.12.2-py3-none-any.whl (37 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: idna>=2.8 in /opt/conda/lib/python3.10/site-packages (from anyio<5.0,>=3.0->gradio->-r ./requirements.txt (line 6)) (3.4)\u001b[0m\n",
      "\u001b[34mCollecting sniffio>=1.1 (from anyio<5.0,>=3.0->gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading sniffio-1.3.1-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[34mCollecting exceptiongroup>=1.0.2 (from anyio<5.0,>=3.0->gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading exceptiongroup-1.2.2-py3-none-any.whl (16 kB)\u001b[0m\n",
      "\u001b[34mCollecting starlette (from sse-starlette->-r ./requirements.txt (line 10))\u001b[0m\n",
      "\u001b[34mDownloading starlette-0.38.6-py3-none-any.whl (71 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 71.5/71.5 kB 30.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting aiohappyeyeballs>=2.3.0 (from aiohttp->datasets->-r ./requirements.txt (line 12))\u001b[0m\n",
      "\u001b[34mDownloading aiohappyeyeballs-2.4.2-py3-none-any.whl (14 kB)\u001b[0m\n",
      "\u001b[34mCollecting aiosignal>=1.1.2 (from aiohttp->datasets->-r ./requirements.txt (line 12))\u001b[0m\n",
      "\u001b[34mDownloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->-r ./requirements.txt (line 12)) (23.1.0)\u001b[0m\n",
      "\u001b[34mCollecting frozenlist>=1.1.1 (from aiohttp->datasets->-r ./requirements.txt (line 12))\u001b[0m\n",
      "\u001b[34mDownloading frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (239 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 239.5/239.5 kB 66.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting multidict<7.0,>=4.5 (from aiohttp->datasets->-r ./requirements.txt (line 12))\u001b[0m\n",
      "\u001b[34mDownloading multidict-6.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (124 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 124.6/124.6 kB 45.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting yarl<2.0,>=1.12.0 (from aiohttp->datasets->-r ./requirements.txt (line 12))\u001b[0m\n",
      "\u001b[34mDownloading yarl-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (447 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 447.9/447.9 kB 88.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting async-timeout<5.0,>=4.0 (from aiohttp->datasets->-r ./requirements.txt (line 12))\u001b[0m\n",
      "\u001b[34mDownloading async_timeout-4.0.3-py3-none-any.whl (5.7 kB)\u001b[0m\n",
      "\u001b[34mCollecting gitdb<5,>=4.0.1 (from gitpython!=3.1.19,<4,>=3.0.7->streamlit>=1.24.0->-r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading gitdb-4.0.11-py3-none-any.whl (62 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.7/62.7 kB 24.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: certifi in /opt/conda/lib/python3.10/site-packages (from httpx>=0.24.1->gradio->-r ./requirements.txt (line 6)) (2023.5.7)\u001b[0m\n",
      "\u001b[34mCollecting httpcore==1.* (from httpx>=0.24.1->gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading httpcore-1.0.5-py3-none-any.whl (77 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 77.9/77.9 kB 32.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx>=0.24.1->gradio->-r ./requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading h11-0.14.0-py3-none-any.whl (58 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 58.3/58.3 kB 22.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (1.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (0.11.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (4.41.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (1.4.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyparsing<3.1,>=2.3.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (3.0.9)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib~=3.0->gradio->-r ./requirements.txt (line 6)) (2.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas<3.0,>=1.0->gradio->-r ./requirements.txt (line 6)) (2023.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas<3.0,>=1.0->gradio->-r ./requirements.txt (line 6)) (2023.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: annotated-types>=0.6.0 in /opt/conda/lib/python3.10/site-packages (from pydantic>=2.0->gradio->-r ./requirements.txt (line 6)) (0.7.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pydantic-core==2.23.4 in /opt/conda/lib/python3.10/site-packages (from pydantic>=2.0->gradio->-r ./requirements.txt (line 6)) (2.23.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.30.1->-r ./requirements.txt (line 3)) (3.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markdown-it-py>=2.2.0 in /opt/conda/lib/python3.10/site-packages (from rich<14,>=10.14.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (3.0.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from rich<14,>=10.14.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (2.15.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.23.0->-r ./requirements.txt (line 9)) (1.12)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.23.0->-r ./requirements.txt (line 9)) (3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: shellingham>=1.3.0 in /opt/conda/lib/python3.10/site-packages (from typer<1.0,>=0.12->gradio->-r ./requirements.txt (line 6)) (1.5.1)\u001b[0m\n",
      "\u001b[34mCollecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=3.1.19,<4,>=3.0.7->streamlit>=1.24.0->-r ./requirements.txt (line 11))\u001b[0m\n",
      "\u001b[34mDownloading smmap-5.0.1-py3-none-any.whl (24 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (2023.6.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: referencing>=0.28.4 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (0.29.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: rpds-py>=0.7.1 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (0.8.10)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: mdurl~=0.1 in /opt/conda/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich<14,>=10.14.0->streamlit>=1.24.0->-r ./requirements.txt (line 11)) (0.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate==0.23.0->-r ./requirements.txt (line 9)) (1.3.0)\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: jieba\u001b[0m\n",
      "\u001b[34mBuilding wheel for jieba (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for jieba (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for jieba: filename=jieba-0.42.1-py3-none-any.whl size=19314458 sha256=914ccec064f1c40050d8106e4b3a9aae7d889b037108e0aff7c3d8fde481411b\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/c9/69/31/d56d90b22a1777b0b231e234b00302a55be255930f8bd92dcd\u001b[0m\n",
      "\u001b[34mSuccessfully built jieba\u001b[0m\n",
      "\u001b[34mInstalling collected packages: tokenizers, sentencepiece, pydub, jieba, huggingface, cpm_kernels, xxhash, websockets, watchdog, urllib3, typing-extensions, tqdm, tomlkit, sniffio, smmap, semantic-version, safetensors, ruff, rouge_chinese, regex, python-multipart, pyarrow, orjson, narwhals, markdown, latex2mathml, importlib-resources, h11, frozenlist, ffmpy, exceptiongroup, cachetools, blinker, async-timeout, aiohappyeyeballs, aiofiles, uvicorn, requests, pydeck, nltk, multidict, mdtex2html, httpcore, gitdb, anyio, aiosignal, yarl, typer, starlette, huggingface-hub, httpx, gitpython, transformers, sse-starlette, gradio-client, fastapi, altair, aiohttp, accelerate, streamlit, gradio, datasets\u001b[0m\n",
      "\u001b[34mAttempting uninstall: urllib3\u001b[0m\n",
      "\u001b[34mFound existing installation: urllib3 1.26.15\u001b[0m\n",
      "\u001b[34mUninstalling urllib3-1.26.15:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled urllib3-1.26.15\u001b[0m\n",
      "\u001b[34mAttempting uninstall: typing-extensions\u001b[0m\n",
      "\u001b[34mFound existing installation: typing_extensions 4.7.1\u001b[0m\n",
      "\u001b[34mUninstalling typing_extensions-4.7.1:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled typing_extensions-4.7.1\u001b[0m\n",
      "\u001b[34mAttempting uninstall: tqdm\u001b[0m\n",
      "\u001b[34mFound existing installation: tqdm 4.65.0\u001b[0m\n",
      "\u001b[34mUninstalling tqdm-4.65.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled tqdm-4.65.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: pyarrow\u001b[0m\n",
      "\u001b[34mFound existing installation: pyarrow 12.0.1\u001b[0m\n",
      "\u001b[34mUninstalling pyarrow-12.0.1:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled pyarrow-12.0.1\u001b[0m\n",
      "\u001b[34mAttempting uninstall: requests\u001b[0m\n",
      "\u001b[34mFound existing installation: requests 2.31.0\u001b[0m\n",
      "\u001b[34mUninstalling requests-2.31.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled requests-2.31.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: typer\u001b[0m\n",
      "\u001b[34mFound existing installation: typer 0.9.0\u001b[0m\n",
      "\u001b[34mUninstalling typer-0.9.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled typer-0.9.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: accelerate\u001b[0m\n",
      "\u001b[34mFound existing installation: accelerate 0.20.3\u001b[0m\n",
      "\u001b[34mUninstalling accelerate-0.20.3:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled accelerate-0.20.3\u001b[0m\n",
      "\u001b[34mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\u001b[0m\n",
      "\u001b[34mbotocore 1.31.2 requires urllib3<1.27,>=1.25.4, but you have urllib3 2.2.3 which is incompatible.\u001b[0m\n",
      "\u001b[34mspacy 3.6.0 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mspacy 3.6.0 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.5 which is incompatible.\u001b[0m\n",
      "\u001b[34mthinc 8.1.10 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\n",
      "\u001b[34mSuccessfully installed accelerate-0.23.0 aiofiles-23.2.1 aiohappyeyeballs-2.4.2 aiohttp-3.10.7 aiosignal-1.3.1 altair-5.4.1 anyio-4.6.0 async-timeout-4.0.3 blinker-1.8.2 cachetools-5.5.0 cpm_kernels-1.0.11 datasets-3.0.1 exceptiongroup-1.2.2 fastapi-0.115.0 ffmpy-0.4.0 frozenlist-1.4.1 gitdb-4.0.11 gitpython-3.1.43 gradio-4.44.0 gradio-client-1.3.0 h11-0.14.0 httpcore-1.0.5 httpx-0.27.2 huggingface-0.0.1 huggingface-hub-0.25.1 importlib-resources-6.4.5 jieba-0.42.1 latex2mathml-3.77.0 markdown-3.7 mdtex2html-1.3.0 multidict-6.1.0 narwhals-1.8.4 nltk-3.9.1 orjson-3.10.7 pyarrow-17.0.0 pydeck-0.9.1 pydub-0.25.1 python-multipart-0.0.11 regex-2024.9.11 requests-2.32.3 rouge_chinese-1.0.3 ruff-0.6.8 safetensors-0.4.5 semantic-version-2.10.0 sentencepiece-0.2.0 smmap-5.0.1 sniffio-1.3.1 sse-starlette-2.1.3 starlette-0.38.6 streamlit-1.38.0 tokenizers-0.13.3 tomlkit-0.12.0 tqdm-4.66.5 transformers-4.30.1 typer-0.12.5 typing-extensions-4.12.2 urllib3-2.2.3 uvicorn-0.31.0 watchdog-4.0.2 websockets-12.0 xxhash-3.5.0 yarl-1.13.1\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[34m[notice] A new release of pip is available: 23.1.2 -> 24.2\u001b[0m\n",
      "\u001b[34m[notice] To update, run: pip install --upgrade pip\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:22,862] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:25,617] [WARNING] [runner.py:213:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only.\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:25,617] [INFO] [runner.py:587:main] cmd = /opt/conda/bin/python3.10 -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMCwgMSwgMiwgMywgNCwgNSwgNiwgN119 --master_addr=127.0.0.1 --master_port=23456 --enable_each_rank_log=None main_tuning.py --deepspeed deepspeed.json --do_train --train_file /opt/ml/input/data/KuaFuTrainData/train.json --validation_file /opt/ml/input/data/KuaFuTrainData/dev.json --prompt_column content --response_column summary --overwrite_cache --model_name_or_path s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/ --output_dir /tmp/model/adgen-chatglm2-6b-ft/ --model_output_s3_path s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/ --overwrite_output_dir --max_source_length 64 --max_target_length 64 --per_device_train_batch_size 1 --per_device_eval_batch_size 4 --gradient_accumulation_steps 1 --predict_with_generate --max_steps 50 --logging_steps 10 --save_steps 50 --learning_rate 1e-4 --fp16\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:30,517] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,181] [INFO] [launch.py:139:main] 0 NCCL_VERSION=2.16.2\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,181] [INFO] [launch.py:139:main] 0 NCCL_SOCKET_IFNAME=eth0\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,181] [INFO] [launch.py:139:main] 0 NCCL_DEBUG=WARN\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,181] [INFO] [launch.py:139:main] 0 NCCL_IB_DISABLE=1\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,181] [INFO] [launch.py:146:main] WORLD INFO DICT: {'localhost': [0, 1, 2, 3, 4, 5, 6, 7]}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,181] [INFO] [launch.py:152:main] nnodes=1, num_local_procs=8, node_rank=0\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,181] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0, 1, 2, 3, 4, 5, 6, 7]})\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,181] [INFO] [launch.py:164:main] dist_world_size=8\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,181] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,182] [INFO] [launch.py:256:main] process 367 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=0', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/tmp/model/adgen-chatglm2-6b-ft/', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,183] [INFO] [launch.py:256:main] process 368 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=1', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/tmp/model/adgen-chatglm2-6b-ft/', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,183] [INFO] [launch.py:256:main] process 369 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=2', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/tmp/model/adgen-chatglm2-6b-ft/', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,184] [INFO] [launch.py:256:main] process 370 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=3', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/tmp/model/adgen-chatglm2-6b-ft/', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,184] [INFO] [launch.py:256:main] process 371 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=4', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/tmp/model/adgen-chatglm2-6b-ft/', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,185] [INFO] [launch.py:256:main] process 372 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=5', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/tmp/model/adgen-chatglm2-6b-ft/', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,185] [INFO] [launch.py:256:main] process 373 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=6', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/tmp/model/adgen-chatglm2-6b-ft/', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:33,185] [INFO] [launch.py:256:main] process 374 spawned with command: ['/opt/conda/bin/python3.10', '-u', 'main_tuning.py', '--local_rank=7', '--deepspeed', 'deepspeed.json', '--do_train', '--train_file', '/opt/ml/input/data/KuaFuTrainData/train.json', '--validation_file', '/opt/ml/input/data/KuaFuTrainData/dev.json', '--prompt_column', 'content', '--response_column', 'summary', '--overwrite_cache', '--model_name_or_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/', '--output_dir', '/tmp/model/adgen-chatglm2-6b-ft/', '--model_output_s3_path', 's3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/', '--overwrite_output_dir', '--max_source_length', '64', '--max_target_length', '64', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '1', '--predict_with_generate', '--max_steps', '50', '--logging_steps', '10', '--save_steps', '50', '--learning_rate', '1e-4', '--fp16']\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:45,122] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:45,170] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:45,184] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:45,236] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:45,237] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:45,247] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:45,251] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:45,256] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:48,803] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:49,140] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:49,166] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:49,167] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:49,167] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:49,279] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:49,324] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:49,327] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:04:49,328] [INFO] [comm.py:652:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m09/28/2024 14:04:50 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/28/2024 14:04:50 - INFO - __main__ - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=None,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=deepspeed.json,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=1,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=0.0001,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=0,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/model/adgen-chatglm2-6b-ft/runs/Sep28_14-04-49_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=linear,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=50,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=3.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_hf,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/model/adgen-chatglm2-6b-ft/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=True,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=4,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=1,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=True,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=[],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/model/adgen-chatglm2-6b-ft/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=50,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m09/28/2024 14:04:50 - WARNING - __main__ - Process rank: 6, device: cuda:6, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/28/2024 14:04:50 - WARNING - __main__ - Process rank: 5, device: cuda:5, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/28/2024 14:04:50 - WARNING - __main__ - Process rank: 7, device: cuda:7, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/28/2024 14:04:50 - WARNING - __main__ - Process rank: 2, device: cuda:2, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/28/2024 14:04:50 - WARNING - __main__ - Process rank: 4, device: cuda:4, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/28/2024 14:04:50 - WARNING - __main__ - Process rank: 1, device: cuda:1, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m09/28/2024 14:04:50 - WARNING - __main__ - Process rank: 3, device: cuda:3, n_gpu: 1distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34mGenerating train split: 0 examples [00:00, ? examples/s]\u001b[0m\n",
      "\u001b[34mGenerating train split: 21913 examples [00:00, 200485.53 examples/s]\u001b[0m\n",
      "\u001b[34mGenerating train split: 42351 examples [00:00, 227978.34 examples/s]\u001b[0m\n",
      "\u001b[34mGenerating validation split: 0 examples [00:00, ? examples/s]\u001b[0m\n",
      "\u001b[34mGenerating validation split: 1000 examples [00:00, 240623.26 examples/s]\u001b[0m\n",
      "\u001b[34mworld_rank===5\u001b[0m\n",
      "\u001b[34mworld_rank===1\u001b[0m\n",
      "\u001b[34mworld_rank===4\u001b[0m\n",
      "\u001b[34mworld_rank===6\u001b[0m\n",
      "\u001b[34mworld_rank===2\u001b[0m\n",
      "\u001b[34mworld_rank===0\u001b[0m\n",
      "\u001b[34mworld_rank===7\u001b[0m\n",
      "\u001b[34mworld_rank===3\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/config.json /tmp/orignal/config.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/modeling_chatglm.py /tmp/orignal/modeling_chatglm.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/quantization.py /tmp/orignal/quantization.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model.bin.index.json /tmp/orignal/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/tokenization_chatglm.py /tmp/orignal/tokenization_chatglm.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/tokenizer_config.json /tmp/orignal/tokenizer_config.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/configuration_chatglm.py /tmp/orignal/configuration_chatglm.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/tokenizer.model /tmp/orignal/tokenizer.model\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00007-of-00007.bin /tmp/orignal/pytorch_model-00007-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00004-of-00007.bin /tmp/orignal/pytorch_model-00004-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00001-of-00007.bin /tmp/orignal/pytorch_model-00001-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00006-of-00007.bin /tmp/orignal/pytorch_model-00006-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00003-of-00007.bin /tmp/orignal/pytorch_model-00003-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00005-of-00007.bin /tmp/orignal/pytorch_model-00005-of-00007.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/pytorch_model-00002-of-00007.bin /tmp/orignal/pytorch_model-00002-of-00007.bin\u001b[0m\n",
      "\u001b[34msync s3 model s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/original-6B/to /tmp/orignal:\u001b[0m\n",
      "\u001b[34m['tokenization_chatglm.py', 'modeling_chatglm.py', 'pytorch_model-00003-of-00007.bin', 'tokenizer_config.json', 'config.json', 'pytorch_model-00002-of-00007.bin', 'pytorch_model-00007-of-00007.bin', 'pytorch_model-00004-of-00007.bin', 'pytorch_model-00005-of-00007.bin', 'pytorch_model-00006-of-00007.bin', 'quantization.py', 'pytorch_model.bin.index.json', 'pytorch_model-00001-of-00007.bin', 'tokenizer.model', 'configuration_chatglm.py']\u001b[0m\n",
      "\u001b[34mNCCL version 2.16.2+cuda11.8\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:667] 2024-09-28 14:05:02,883 >> loading configuration file /tmp/orignal/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:667] 2024-09-28 14:05:02,883 >> loading configuration file /tmp/orignal/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:667] 2024-09-28 14:05:02,889 >> loading configuration file /tmp/orignal/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:667] 2024-09-28 14:05:02,889 >> loading configuration file /tmp/orignal/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:725] 2024-09-28 14:05:02,890 >> Model config ChatGLMConfig {\n",
      "  \"_name_or_path\": \"/tmp/orignal/\",\n",
      "  \"add_bias_linear\": false,\n",
      "  \"add_qkv_bias\": true,\n",
      "  \"apply_query_key_layer_scaling\": true,\n",
      "  \"apply_residual_connection_post_layernorm\": false,\n",
      "  \"architectures\": [\n",
      "    \"ChatGLMModel\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"attention_softmax_in_fp32\": true,\n",
      "  \"auto_map\": {\n",
      "    \"AutoConfig\": \"configuration_chatglm.ChatGLMConfig\",\n",
      "    \"AutoModel\": \"modeling_chatglm.ChatGLMForConditionalGeneration\",\n",
      "    \"AutoModelForCausalLM\": \"modeling_chatglm.ChatGLMForConditionalGeneration\",\n",
      "    \"AutoModelForSeq2SeqLM\": \"modeling_chatglm.ChatGLMForConditionalGeneration\"\n",
      "  },\n",
      "  \"bias_dropout_fusion\": true,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"ffn_hidden_size\": 13696,\n",
      "  \"fp32_residual_connection\": false,\n",
      "  \"hidden_dropout\": 0.0,\n",
      "  \"hidden_size\": 4096,\n",
      "  \"kv_channels\": 128,\n",
      "  \"layernorm_epsilon\": 1e-05,\n",
      "  \"model_type\": \"chatglm\",\n",
      "  \"multi_query_attention\": true,\n",
      "  \"multi_query_group_num\": 2,\n",
      "  \"num_attention_heads\": 32,\n",
      "  \"num_layers\": 28,\n",
      "  \"original_rope\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"padded_vocab_size\": 65024,\n",
      "  \"post_layer_norm\": true,\n",
      "  \"pre_seq_len\": null,\n",
      "  \"prefix_projection\": false,\n",
      "  \"quantization_bit\": 0,\n",
      "  \"rmsnorm\": true,\n",
      "  \"seq_length\": 32768,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"float16\",\n",
      "  \"transformers_version\": \"4.30.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 65024\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:725] 2024-09-28 14:05:02,890 >> Model config ChatGLMConfig {\n",
      "  \"_name_or_path\": \"/tmp/orignal/\",\n",
      "  \"add_bias_linear\": false,\n",
      "  \"add_qkv_bias\": true,\n",
      "  \"apply_query_key_layer_scaling\": true,\n",
      "  \"apply_residual_connection_post_layernorm\": false,\n",
      "  \"architectures\": [\n",
      "    \"ChatGLMModel\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"attention_softmax_in_fp32\": true,\n",
      "  \"auto_map\": {\n",
      "    \"AutoConfig\": \"configuration_chatglm.ChatGLMConfig\",\n",
      "    \"AutoModel\": \"modeling_chatglm.ChatGLMForConditionalGeneration\",\n",
      "    \"AutoModelForCausalLM\": \"modeling_chatglm.ChatGLMForConditionalGeneration\",\n",
      "    \"AutoModelForSeq2SeqLM\": \"modeling_chatglm.ChatGLMForConditionalGeneration\"\n",
      "  },\n",
      "  \"bias_dropout_fusion\": true,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"ffn_hidden_size\": 13696,\n",
      "  \"fp32_residual_connection\": false,\n",
      "  \"hidden_dropout\": 0.0,\n",
      "  \"hidden_size\": 4096,\n",
      "  \"kv_channels\": 128,\n",
      "  \"layernorm_epsilon\": 1e-05,\n",
      "  \"model_type\": \"chatglm\",\n",
      "  \"multi_query_attention\": true,\n",
      "  \"multi_query_group_num\": 2,\n",
      "  \"num_attention_heads\": 32,\n",
      "  \"num_layers\": 28,\n",
      "  \"original_rope\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"padded_vocab_size\": 65024,\n",
      "  \"post_layer_norm\": true,\n",
      "  \"pre_seq_len\": null,\n",
      "  \"prefix_projection\": false,\n",
      "  \"quantization_bit\": 0,\n",
      "  \"rmsnorm\": true,\n",
      "  \"seq_length\": 32768,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"float16\",\n",
      "  \"transformers_version\": \"4.30.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 65024\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-28 14:05:02,893 >> loading file tokenizer.model\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-28 14:05:02,893 >> loading file tokenizer.model\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-28 14:05:02,893 >> loading file added_tokens.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-28 14:05:02,893 >> loading file special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-28 14:05:02,893 >> loading file added_tokens.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-28 14:05:02,893 >> loading file special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-28 14:05:02,893 >> loading file tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1821] 2024-09-28 14:05:02,893 >> loading file tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2575] 2024-09-28 14:05:03,010 >> loading weights file /tmp/orignal/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2575] 2024-09-28 14:05:03,010 >> loading weights file /tmp/orignal/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:577] 2024-09-28 14:05:03,011 >> Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.30.1\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:577] 2024-09-28 14:05:03,011 >> Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.30.1\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:01<00:09,  1.54s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:12,  2.08s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:12,  2.14s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:12,  2.14s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:13,  2.17s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:13,  2.19s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:13,  2.21s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  14%|█▍        | 1/7 [00:02<00:13,  2.22s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:03<00:08,  1.69s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:10,  2.15s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:10,  2.17s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:10,  2.17s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:11,  2.24s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:11,  2.24s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:11,  2.23s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  29%|██▊       | 2/7 [00:04<00:11,  2.25s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:05<00:06,  1.69s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.08s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.12s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.18s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.19s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.19s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.20s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  43%|████▎     | 3/7 [00:06<00:08,  2.20s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:06<00:05,  1.69s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:06,  2.07s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:06,  2.05s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:06,  2.07s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:06,  2.09s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:06,  2.10s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:06,  2.10s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  57%|█████▋    | 4/7 [00:08<00:06,  2.12s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:08<00:03,  1.80s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.06s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.08s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.05s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.08s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.08s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.09s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  71%|███████▏  | 5/7 [00:10<00:04,  2.12s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:10<00:01,  1.87s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:11<00:00,  1.55s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:11<00:00,  1.66s/it]\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3295] 2024-09-28 14:05:14,700 >> All model checkpoint weights were used when initializing ChatGLMForConditionalGeneration.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3295] 2024-09-28 14:05:14,700 >> All model checkpoint weights were used when initializing ChatGLMForConditionalGeneration.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3303] 2024-09-28 14:05:14,700 >> All the weights of ChatGLMForConditionalGeneration were initialized from the model checkpoint at /tmp/orignal/.\u001b[0m\n",
      "\u001b[34mIf your task is similar to the task the model of the checkpoint was trained on, you can already use ChatGLMForConditionalGeneration for predictions without further training.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3303] 2024-09-28 14:05:14,700 >> All the weights of ChatGLMForConditionalGeneration were initialized from the model checkpoint at /tmp/orignal/.\u001b[0m\n",
      "\u001b[34mIf your task is similar to the task the model of the checkpoint was trained on, you can already use ChatGLMForConditionalGeneration for predictions without further training.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2927] 2024-09-28 14:05:14,704 >> Generation config file not found, using a generation config created from the model config.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2927] 2024-09-28 14:05:14,704 >> Generation config file not found, using a generation config created from the model config.\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.10s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.09s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.15s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.12s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.13s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.13s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  86%|████████▌ | 6/7 [00:12<00:02,  2.13s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:13<00:00,  1.82s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:13<00:00,  1.98s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  1.91s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.02s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  1.86s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.02s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  1.94s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.05s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  1.98s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.07s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.04s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.10s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.11s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 7/7 [00:14<00:00,  2.12s/it]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:06<04:12, 163.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:07<02:21, 284.84 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:09<01:46, 370.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:11<01:28, 434.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:12<01:17, 479.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:14<01:11, 511.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:16<01:06, 534.80 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:18<01:02, 551.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:19<00:59, 560.38 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:21<00:56, 569.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:23<00:54, 575.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:24<00:52, 578.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:26<00:50, 580.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:28<00:48, 581.60 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:29<00:46, 583.87 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:31<00:45, 585.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:33<00:43, 584.45 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:35<00:41, 584.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:36<00:39, 586.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:38<00:38, 585.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:40<00:36, 586.61 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:41<00:34, 583.98 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:43<00:33, 581.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:45<00:31, 582.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:47<00:29, 582.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:48<00:28, 582.95 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:50<00:26, 582.20 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:52<00:24, 582.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:53<00:22, 584.44 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:55<00:21, 584.33 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:57<00:19, 587.85 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:59<00:17, 586.25 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [01:00<00:15, 586.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [01:02<00:14, 587.33 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:04<00:12, 589.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:05<00:10, 588.02 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:07<00:09, 589.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:09<00:07, 588.59 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:10<00:05, 587.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:12<00:04, 587.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:14<00:02, 586.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:16<00:00, 586.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:16<00:00, 584.47 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:16<00:00, 552.46 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids\u001b[0m\n",
      "\u001b[34m[64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:577] 2024-09-28 14:06:37,856 >> max_steps is given, it will override any value given in num_train_epochs\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:577] 2024-09-28 14:06:37,856 >> max_steps is given, it will override any value given in num_train_epochs\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   0%|          | 0/42351 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:06:38,038] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed info: version=0.15.2+unknown, git-hash=unknown, git-branch=unknown\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:06:38,038] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 8\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:01<01:12, 569.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:01<01:15, 550.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:01<01:15, 550.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:01<01:17, 536.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:02<01:39, 414.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:02<01:52, 369.15 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   2%|▏         | 1000/42351 [00:02<01:52, 366.30 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:03<01:09, 577.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:03<01:12, 559.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:03<01:12, 558.68 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:03<01:12, 553.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:04<01:20, 501.68 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:04<01:25, 472.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   5%|▍         | 2000/42351 [00:04<01:25, 470.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:05<01:08, 577.54 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:05<01:09, 562.61 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:05<01:10, 560.47 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:05<01:10, 558.66 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:05<01:13, 532.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:06<01:16, 515.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   7%|▋         | 3000/42351 [00:06<01:16, 512.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:06<01:05, 582.60 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:07, 571.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:07, 569.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:07, 569.17 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:09, 553.43 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:10, 541.57 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:   9%|▉         | 4000/42351 [00:07<01:11, 539.76 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:08<01:03, 585.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:08<01:04, 576.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:08<01:05, 574.22 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:08<01:04, 574.95 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:09<01:06, 564.99 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:09<01:07, 557.45 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  12%|█▏        | 5000/42351 [00:09<01:07, 555.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:10<01:02, 586.13 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:10<01:02, 579.28 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:10<01:03, 576.88 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:10<01:02, 578.25 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:10<01:03, 571.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:11<01:04, 566.85 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  14%|█▍        | 6000/42351 [00:11<01:04, 564.90 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:11<01:00, 587.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:00, 581.94 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:01, 579.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:00, 581.15 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:01, 576.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:12<01:01, 573.84 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  17%|█▋        | 7000/42351 [00:13<01:01, 571.68 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:13<00:58, 588.66 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:13<00:58, 583.64 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:13<00:58, 582.98 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:13<00:59, 580.89 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:14<00:59, 580.25 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:14<00:59, 578.60 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  19%|█▉        | 8000/42351 [00:14<00:59, 576.71 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:15<00:56, 586.01 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:15<00:57, 581.89 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:15<00:57, 581.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:15<00:57, 579.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:16<00:57, 579.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:16<00:57, 579.15 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  21%|██▏       | 9000/42351 [00:16<00:57, 577.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:17<00:55, 588.14 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:17<00:55, 584.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:17<00:55, 583.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:17<00:55, 581.22 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:17<00:55, 583.03 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:18<00:55, 582.94 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  24%|██▎       | 10000/42351 [00:18<00:55, 580.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:18<00:53, 589.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 584.88 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 584.63 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 582.21 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 584.62 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 584.91 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  26%|██▌       | 11000/42351 [00:19<00:53, 582.56 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:20<00:51, 587.66 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:20<00:52, 583.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:20<00:52, 583.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:20<00:52, 580.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:21<00:52, 583.45 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:21<00:51, 584.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  28%|██▊       | 12000/42351 [00:21<00:52, 582.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:22<00:49, 587.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:22<00:50, 583.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:22<00:50, 583.38 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:22<00:50, 580.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:22<00:50, 584.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:23<00:50, 584.94 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  31%|███       | 13000/42351 [00:23<00:50, 582.57 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:23<00:48, 586.38 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:48, 582.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:48, 581.98 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:48, 579.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:48, 582.38 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:24<00:48, 582.88 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  33%|███▎      | 14000/42351 [00:25<00:48, 580.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:25<00:46, 585.87 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:25<00:46, 583.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:25<00:46, 582.95 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:25<00:47, 580.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:26<00:46, 582.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:26<00:46, 583.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  35%|███▌      | 15000/42351 [00:26<00:47, 581.33 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:27<00:44, 587.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:27<00:45, 583.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:27<00:45, 583.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:27<00:45, 581.11 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:28<00:45, 583.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:28<00:45, 585.00 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  38%|███▊      | 16000/42351 [00:28<00:45, 582.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:29<00:43, 586.71 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:29<00:43, 582.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:29<00:43, 582.24 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:29<00:43, 579.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:29<00:43, 582.90 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:30<00:43, 584.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  40%|████      | 17000/42351 [00:30<00:43, 581.63 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:30<00:41, 586.98 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:41, 582.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:41, 582.14 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:41, 579.89 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:41, 583.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:41, 584.54 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  43%|████▎     | 18000/42351 [00:31<00:41, 581.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:32<00:39, 588.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:32<00:40, 583.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:32<00:40, 583.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:32<00:40, 581.00 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:33<00:39, 584.83 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:33<00:39, 586.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  45%|████▍     | 19000/42351 [00:33<00:40, 583.00 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:34<00:38, 587.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:34<00:38, 582.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:34<00:38, 582.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:34<00:38, 580.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:34<00:38, 583.63 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:35<00:38, 585.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  47%|████▋     | 20000/42351 [00:35<00:38, 582.01 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:35<00:36, 589.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:36, 584.09 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:36, 583.76 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:36, 581.43 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:36, 585.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:36<00:36, 586.48 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  50%|████▉     | 21000/42351 [00:37<00:36, 583.71 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:37<00:34, 587.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:37<00:34, 581.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:37<00:34, 582.00 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:35, 579.78 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:34, 583.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:34, 584.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  52%|█████▏    | 22000/42351 [00:38<00:34, 582.02 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:39<00:33, 586.30 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:39<00:33, 580.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:39<00:33, 580.91 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:39<00:33, 578.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:40<00:33, 582.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:40<00:33, 583.63 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  54%|█████▍    | 23000/42351 [00:40<00:33, 581.10 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:40<00:31, 586.40 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:41<00:31, 580.85 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:41<00:31, 580.98 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:41<00:31, 578.83 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:41<00:31, 582.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:42<00:31, 583.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  57%|█████▋    | 24000/42351 [00:42<00:31, 581.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:42<00:29, 585.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:29, 580.30 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:29, 580.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:30, 578.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:29, 581.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:29, 583.13 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  59%|█████▉    | 25000/42351 [00:43<00:29, 580.44 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:44<00:27, 585.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:44<00:28, 580.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:44<00:28, 580.80 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:44<00:28, 578.98 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:45<00:28, 582.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:45<00:28, 583.42 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  61%|██████▏   | 26000/42351 [00:45<00:28, 580.91 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:46<00:26, 585.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:46<00:26, 579.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:46<00:26, 579.94 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:46<00:26, 577.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:46<00:26, 581.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:47<00:26, 583.16 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  64%|██████▍   | 27000/42351 [00:47<00:26, 580.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:47<00:24, 585.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:24, 580.16 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:24, 580.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:24, 578.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:24, 582.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:48<00:24, 583.43 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  66%|██████▌   | 28000/42351 [00:49<00:24, 580.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:49<00:22, 586.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:49<00:22, 581.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:22, 582.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:23, 580.09 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:22, 583.60 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:22, 584.89 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  68%|██████▊   | 29000/42351 [00:50<00:22, 582.20 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:51<00:21, 587.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:51<00:21, 582.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:51<00:21, 582.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:51<00:21, 580.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:52<00:21, 584.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:52<00:21, 585.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  71%|███████   | 30000/42351 [00:52<00:21, 583.00 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:52<00:19, 590.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:53<00:19, 585.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:53<00:19, 585.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:53<00:19, 583.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:53<00:19, 586.87 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:53<00:19, 588.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  73%|███████▎  | 31000/42351 [00:54<00:19, 586.02 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:54<00:17, 588.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 583.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 583.94 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 581.44 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 584.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 586.43 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  76%|███████▌  | 32000/42351 [00:55<00:17, 584.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:56<00:15, 588.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:56<00:16, 583.76 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:56<00:16, 584.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:16, 581.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:15, 585.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:15, 586.37 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  78%|███████▊  | 33000/42351 [00:57<00:16, 584.21 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:57<00:14, 589.20 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:58<00:14, 584.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:58<00:14, 584.63 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:58<00:14, 581.99 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:58<00:14, 585.62 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:59<00:14, 586.78 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  80%|████████  | 34000/42351 [00:59<00:14, 584.62 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [00:59<00:12, 590.66 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 585.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 586.33 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 583.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 587.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:00<00:12, 588.34 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  83%|████████▎ | 35000/42351 [01:01<00:12, 586.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:01<00:10, 589.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:01<00:10, 584.68 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:01<00:10, 584.93 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:10, 582.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:10, 585.78 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:10, 587.02 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  85%|████████▌ | 36000/42351 [01:02<00:10, 584.99 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:03<00:09, 590.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:03<00:09, 585.95 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:03<00:09, 586.09 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:03<00:09, 583.61 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:03<00:09, 587.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:04<00:09, 588.43 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  87%|████████▋ | 37000/42351 [01:04<00:09, 586.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:04<00:07, 590.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:05<00:07, 584.99 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:05<00:07, 585.47 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:05<00:07, 583.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:05<00:07, 586.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:05<00:07, 587.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  90%|████████▉ | 38000/42351 [01:06<00:07, 585.40 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:06<00:05, 588.94 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 583.85 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 584.37 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 581.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 585.24 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 586.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  92%|█████████▏| 39000/42351 [01:07<00:05, 584.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:08<00:03, 588.83 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:08<00:04, 583.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:08<00:04, 584.47 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:04, 581.83 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:04, 584.95 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:04, 586.68 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  94%|█████████▍| 40000/42351 [01:09<00:04, 584.47 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:09<00:02, 587.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:10<00:02, 583.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:10<00:02, 583.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:10<00:02, 580.99 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:10<00:02, 584.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:11<00:02, 585.93 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  97%|█████████▋| 41000/42351 [01:11<00:02, 583.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:11<00:00, 588.11 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:12<00:00, 586.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:12<00:00, 587.12 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids\u001b[0m\n",
      "\u001b[34m[64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:12<00:00, 582.99 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:12<00:00, 583.39 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:12<00:00, 580.17 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:12<00:00, 560.56 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:12<00:00, 578.15 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:12<00:00, 581.40 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids [64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:12<00:00, 563.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 580.13 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids [64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:13<00:00, 577.20 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset:  99%|█████████▉| 42000/42351 [01:13<00:00, 549.63 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 562.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:13<00:00, 578.24 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids\u001b[0m\n",
      "\u001b[34m[64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs\u001b[0m\n",
      "\u001b[34m[Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels\u001b[0m\n",
      "\u001b[34m您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:14<00:00, 466.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:14<00:00, 569.17 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids\u001b[0m\n",
      "\u001b[34m[64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs\u001b[0m\n",
      "\u001b[34m[Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:15<00:00, 436.80 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:15<00:00, 563.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:15<00:00, 422.74 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids [64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs [Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mRunning tokenizer on train dataset: 100%|██████████| 42351/42351 [01:15<00:00, 563.17 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids\u001b[0m\n",
      "\u001b[34m[64790, 64792, 790, 30951, 517, 30910, 30939, 30996, 13, 13, 54761, 31211, 31987, 30954, 31785, 32053, 33148, 34753, 31895, 31950, 31007, 31786, 31707, 30954, 31811, 34753, 30910, 30939, 30966, 30969, 30966, 30930, 30970, 30934, 2322, 30967, 30921, 790, 30939, 30939, 30972, 30973, 30930, 30966, 30941, 30939, 30972, 30940, 30966, 30930, 30970, 2547, 30910, 34779, 31838, 30910, 30972, 30972, 30930, 30970, 12970, 790, 30966, 30972, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34minputs\u001b[0m\n",
      "\u001b[34m[Round 1]\u001b[0m\n",
      "\u001b[34m问：需求:完成用户的基础代谢分析报告|数据信息:基础代谢 1393.5kcal/d [1148.3-1403.5], 肌肉质量 44.5kg [34 您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34mlabel_ids\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55353, 33148, 34753, 54541, 31002, 30931, 30994, 30939, 30966, 30969, 30966, 30930, 30970, 1833, 30931, 30994, 31123, 32903, 32584, 34534, 31155, 31811, 34753, 33662, 31902, 54534, 36055, 51794, 41813, 33917, 33000, 35943, 31123, 54964, 31738, 53006, 33945, 54542, 35145, 41639, 31155, 31811, 34753, 55087, 40714, 31123, 31902, 35943, 51021, 54567, 37559, 31123, 32583, 35434, 31002, 30931, 30994, 32382, 54618, 56111, 34446, 1833, 30931, 2]\u001b[0m\n",
      "\u001b[34mlabels\u001b[0m\n",
      "\u001b[34m您的基础代谢为<b>1393.5</b>，处于合理范围内。基础代谢是指身体在安静状态下所需的最低能量消耗，它对于体型维持和身体健康非常重要。基础代谢率越高，身体消耗的能量就越多，从而有助于<b>减少体脂含量</b\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:06,558] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:06,610] [INFO] [logging.py:96:log_dist] [Rank 0] Using DeepSpeed Optimizer param name adam as basic optimizer\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:06,610] [INFO] [logging.py:96:log_dist] [Rank 0] Removing param_group that has no 'params' in the basic Optimizer\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:06,617] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:06,617] [INFO] [utils.py:59:is_zero_supported_optimizer] Checking ZeRO support for optimizer=FusedAdam type=<class 'deepspeed.ops.adam.fused_adam.FusedAdam'>\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:06,617] [INFO] [logging.py:96:log_dist] [Rank 0] Creating torch.float16 ZeRO stage 2 optimizer\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:06,617] [INFO] [stage_1_and_2.py:149:__init__] Reduce bucket size 500000000\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:06,617] [INFO] [stage_1_and_2.py:150:__init__] Allgather bucket size 500000000\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:06,617] [INFO] [stage_1_and_2.py:151:__init__] CPU Offload: False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:06,617] [INFO] [stage_1_and_2.py:152:__init__] Round robin gradient partitioning: False\u001b[0m\n",
      "\u001b[34m09/28/2024 14:08:22 - WARNING - transformers_modules.modeling_chatglm - `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m09/28/2024 14:08:22 - WARNING - transformers_modules.modeling_chatglm - `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m09/28/2024 14:08:22 - WARNING - transformers_modules.modeling_chatglm - `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m09/28/2024 14:08:23 - WARNING - transformers_modules.modeling_chatglm - `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m09/28/2024 14:08:24 - WARNING - transformers_modules.modeling_chatglm - `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m09/28/2024 14:08:24 - WARNING - transformers_modules.modeling_chatglm - `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m09/28/2024 14:08:24 - WARNING - transformers_modules.modeling_chatglm - `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,225] [INFO] [utils.py:781:see_memory_usage] Before initializing optimizer states\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,226] [INFO] [utils.py:782:see_memory_usage] MA 14.54 GB         Max_MA 15.99 GB         CA 16.0 GB         Max_CA 16 GB\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,226] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 37.51 GB, percent = 3.3%\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,407] [INFO] [utils.py:781:see_memory_usage] After initializing optimizer states\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,408] [INFO] [utils.py:782:see_memory_usage] MA 14.54 GB         Max_MA 17.45 GB         CA 18.9 GB         Max_CA 19 GB\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,408] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 37.69 GB, percent = 3.4%\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,408] [INFO] [stage_1_and_2.py:544:__init__] optimizer state initialized\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,583] [INFO] [utils.py:781:see_memory_usage] After initializing ZeRO optimizer\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,584] [INFO] [utils.py:782:see_memory_usage] MA 14.54 GB         Max_MA 14.54 GB         CA 18.9 GB         Max_CA 19 GB\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,584] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory:  used = 37.72 GB, percent = 3.4%\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,585] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Final Optimizer = DeepSpeedZeroOptimizer\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,585] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed using configured LR scheduler = WarmupLR\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,585] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed LR Scheduler = <deepspeed.runtime.lr_schedules.WarmupLR object at 0x7f87672b4fd0>\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,585] [INFO] [logging.py:96:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0001], mom=[[0.9, 0.999]]\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,586] [INFO] [config.py:999:print] DeepSpeedEngine configuration:\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   activation_checkpointing_config  {\n",
      "    \"partition_activations\": false, \n",
      "    \"contiguous_memory_optimization\": false, \n",
      "    \"cpu_checkpointing\": false, \n",
      "    \"number_checkpoints\": null, \n",
      "    \"synchronize_checkpoint_boundary\": false, \n",
      "    \"profile\": false\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True, 'use_gds': False}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   amp_enabled .................. False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   amp_params ................... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   autotuning_config ............ {\n",
      "    \"enabled\": false, \n",
      "    \"start_step\": null, \n",
      "    \"end_step\": null, \n",
      "    \"metric_path\": null, \n",
      "    \"arg_mappings\": null, \n",
      "    \"metric\": \"throughput\", \n",
      "    \"model_info\": null, \n",
      "    \"results_dir\": \"autotuning_results\", \n",
      "    \"exps_dir\": \"autotuning_exps\", \n",
      "    \"overwrite\": true, \n",
      "    \"fast\": true, \n",
      "    \"start_profile_step\": 3, \n",
      "    \"end_profile_step\": 5, \n",
      "    \"tuner_type\": \"gridsearch\", \n",
      "    \"tuner_early_stopping\": 5, \n",
      "    \"tuner_num_trials\": 50, \n",
      "    \"model_info_path\": null, \n",
      "    \"mp_size\": 1, \n",
      "    \"max_train_batch_size\": null, \n",
      "    \"min_train_batch_size\": 1, \n",
      "    \"max_train_micro_batch_size_per_gpu\": 1.024000e+03, \n",
      "    \"min_train_micro_batch_size_per_gpu\": 1, \n",
      "    \"num_tuning_micro_batch_sizes\": 3\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   bfloat16_enabled ............. False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   bfloat16_immediate_grad_update  False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   checkpoint_parallel_write_pipeline  False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   checkpoint_tag_validation_enabled  True\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   checkpoint_tag_validation_fail  False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x7f87673e8e80>\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   communication_data_type ...... None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   curriculum_enabled_legacy .... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   curriculum_params_legacy ..... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   data_efficiency_enabled ...... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   dataloader_drop_last ......... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   disable_allgather ............ False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   dump_state ................... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   dynamic_loss_scale_args ...... {'init_scale': 65536, 'scale_window': 1000, 'delayed_shift': 2, 'consecutive_hysteresis': False, 'min_scale': 1}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   eigenvalue_enabled ........... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   eigenvalue_gas_boundary_resolution  1\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   eigenvalue_layer_name ........ bert.encoder.layer\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   eigenvalue_layer_num ......... 0\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   eigenvalue_max_iter .......... 100\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   eigenvalue_stability ......... 1e-06\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   eigenvalue_tol ............... 0.01\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   eigenvalue_verbose ........... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   elasticity_enabled ........... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   flops_profiler_config ........ {\n",
      "    \"enabled\": false, \n",
      "    \"recompute_fwd_factor\": 0.0, \n",
      "    \"profile_step\": 1, \n",
      "    \"module_depth\": -1, \n",
      "    \"top_modules\": 1, \n",
      "    \"detailed\": true, \n",
      "    \"output_file\": null\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,587] [INFO] [config.py:1003:print]   fp16_auto_cast ............... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   fp16_enabled ................. True\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   fp16_master_weights_and_gradients  False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   global_rank .................. 0\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   grad_accum_dtype ............. None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   gradient_accumulation_steps .. 1\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   gradient_clipping ............ 0.0\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   gradient_predivide_factor .... 1.0\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   graph_harvesting ............. False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   hybrid_engine ................ enabled=False max_out_tokens=512 inference_tp_size=1 release_inference_cache=False pin_parameters=True tp_gather_partition_size=8\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   initial_dynamic_scale ........ 65536\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   load_universal_checkpoint .... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   loss_scale ................... 0\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   memory_breakdown ............. False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   mics_hierarchial_params_gather  False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   mics_shard_size .............. -1\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   monitor_config ............... tensorboard=TensorBoardConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') comet=CometConfig(enabled=False, samples_log_interval=100, project=None, workspace=None, api_key=None, experiment_name=None, experiment_key=None, online=None, mode=None) wandb=WandbConfig(enabled=False, group=None, team=None, project='deepspeed') csv_monitor=CSVConfig(enabled=False, output_path='', job_name='DeepSpeedJobName')\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   nebula_config ................ {\n",
      "    \"enabled\": false, \n",
      "    \"persistent_storage_path\": null, \n",
      "    \"persistent_time_interval\": 100, \n",
      "    \"num_of_version_in_retention\": 2, \n",
      "    \"enable_nebula_load\": true, \n",
      "    \"load_path\": null\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   optimizer_legacy_fusion ...... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   optimizer_name ............... adam\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   optimizer_params ............. {'lr': 0.0001, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0.0}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0, 'pipe_partitioned': True, 'grad_partitioned': True}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   pld_enabled .................. False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   pld_params ................... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   prescale_gradients ........... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   scheduler_name ............... WarmupLR\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   scheduler_params ............. {'warmup_min_lr': 0, 'warmup_max_lr': 0.0001, 'warmup_num_steps': 0}\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   seq_parallel_communication_data_type  torch.float32\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   sparse_attention ............. None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   sparse_gradients_enabled ..... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   steps_per_print .............. inf\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   timers_config ................ enabled=True synchronized=True\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   train_batch_size ............. 8\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   train_micro_batch_size_per_gpu  1\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   use_data_before_expert_parallel_  False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   use_node_local_storage ....... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   wall_clock_breakdown ......... False\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   weight_quantization_config ... None\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   world_size ................... 8\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   zero_allow_untested_optimizer  True\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   zero_config .................. stage=2 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 use_multi_rank_bucket_allreduce=True allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False use_all_reduce_for_fetch_params=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False zero_hpz_partition_size=1 zero_quantized_weights=False zero_quantized_nontrainable_weights=False zero_quantized_gradients=False mics_shard_size=-1 mics_hierarchical_params_gather=False memory_efficient_linear=True pipeline_loading_checkpoint=False override_module_apply=True\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   zero_enabled ................. True\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   zero_force_ds_cpu_optimizer .. True\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,588] [INFO] [config.py:1003:print]   zero_optimization_stage ...... 2\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:24,589] [INFO] [config.py:989:print_user_config]   json = {\n",
      "    \"train_micro_batch_size_per_gpu\": 1, \n",
      "    \"zero_allow_untested_optimizer\": true, \n",
      "    \"fp16\": {\n",
      "        \"enabled\": true, \n",
      "        \"loss_scale\": 0, \n",
      "        \"initial_scale_power\": 16, \n",
      "        \"loss_scale_window\": 1000, \n",
      "        \"hysteresis\": 2, \n",
      "        \"min_loss_scale\": 1\n",
      "    }, \n",
      "    \"optimizer\": {\n",
      "        \"type\": \"Adam\", \n",
      "        \"params\": {\n",
      "            \"lr\": 0.0001, \n",
      "            \"betas\": [0.9, 0.999], \n",
      "            \"eps\": 1e-08, \n",
      "            \"weight_decay\": 0.0\n",
      "        }\n",
      "    }, \n",
      "    \"scheduler\": {\n",
      "        \"type\": \"WarmupLR\", \n",
      "        \"params\": {\n",
      "            \"warmup_min_lr\": 0, \n",
      "            \"warmup_max_lr\": 0.0001, \n",
      "            \"warmup_num_steps\": 0\n",
      "        }\n",
      "    }, \n",
      "    \"zero_optimization\": {\n",
      "        \"stage\": 2, \n",
      "        \"allgather_partitions\": true, \n",
      "        \"allgather_bucket_size\": 5.000000e+08, \n",
      "        \"overlap_comm\": false, \n",
      "        \"reduce_scatter\": true, \n",
      "        \"reduce_bucket_size\": 5.000000e+08, \n",
      "        \"contiguous_gradients\": true\n",
      "    }, \n",
      "    \"gradient_accumulation_steps\": 1, \n",
      "    \"steps_per_print\": inf, \n",
      "    \"bf16\": {\n",
      "        \"enabled\": false\n",
      "    }\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1786] 2024-09-28 14:08:24,589 >> ***** Running training *****\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1787] 2024-09-28 14:08:24,589 >>   Num examples = 42,351\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1788] 2024-09-28 14:08:24,589 >>   Num Epochs = 1\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1789] 2024-09-28 14:08:24,589 >>   Instantaneous batch size per device = 1\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1790] 2024-09-28 14:08:24,589 >>   Total train batch size (w. parallel, distributed & accumulation) = 8\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1791] 2024-09-28 14:08:24,589 >>   Gradient Accumulation steps = 1\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1792] 2024-09-28 14:08:24,589 >>   Total optimization steps = 50\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1786] 2024-09-28 14:08:24,589 >> ***** Running training *****\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1787] 2024-09-28 14:08:24,589 >>   Num examples = 42,351\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1788] 2024-09-28 14:08:24,589 >>   Num Epochs = 1\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1789] 2024-09-28 14:08:24,589 >>   Instantaneous batch size per device = 1\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1790] 2024-09-28 14:08:24,589 >>   Total train batch size (w. parallel, distributed & accumulation) = 8\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1791] 2024-09-28 14:08:24,589 >>   Gradient Accumulation steps = 1\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1792] 2024-09-28 14:08:24,589 >>   Total optimization steps = 50\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1793] 2024-09-28 14:08:24,590 >>   Number of trainable parameters = 6,243,584,000\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1793] 2024-09-28 14:08:24,590 >>   Number of trainable parameters = 6,243,584,000\u001b[0m\n",
      "\u001b[34m0%|          | 0/50 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m09/28/2024 14:08:24 - WARNING - transformers_modules.modeling_chatglm - `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:25,645] [INFO] [loss_scaler.py:190:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 65536, but hysteresis is 2. Reducing hysteresis to 1\u001b[0m\n",
      "\u001b[34m2%|▏         | 1/50 [00:01<00:51,  1.06s/it]\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:25,955] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 65536, reducing to 32768\u001b[0m\n",
      "\u001b[34m4%|▍         | 2/50 [00:01<00:29,  1.62it/s]\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:26,257] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 32768, reducing to 16384\u001b[0m\n",
      "\u001b[34m6%|▌         | 3/50 [00:01<00:22,  2.11it/s]\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:26,559] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 16384, reducing to 8192\u001b[0m\n",
      "\u001b[34m8%|▊         | 4/50 [00:01<00:18,  2.47it/s]\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:08:26,861] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 8192, reducing to 4096\u001b[0m\n",
      "\u001b[34m10%|█         | 5/50 [00:02<00:16,  2.72it/s]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 6/50 [00:03<00:22,  1.99it/s]\u001b[0m\n",
      "\u001b[34m14%|█▍        | 7/50 [00:04<00:28,  1.52it/s]\u001b[0m\n",
      "\u001b[34m16%|█▌        | 8/50 [00:04<00:31,  1.34it/s]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 9/50 [00:06<00:36,  1.12it/s]\u001b[0m\n",
      "\u001b[34m20%|██        | 10/50 [00:07<00:37,  1.08it/s]\u001b[0m\n",
      "\u001b[34m{'loss': 2.8061, 'learning_rate': 0.0001, 'epoch': 0.0}\u001b[0m\n",
      "\u001b[34m20%|██        | 10/50 [00:07<00:37,  1.08it/s]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 11/50 [00:08<00:40,  1.04s/it]\u001b[0m\n",
      "\u001b[34m24%|██▍       | 12/50 [00:09<00:40,  1.07s/it]\u001b[0m\n",
      "\u001b[34m26%|██▌       | 13/50 [00:10<00:41,  1.13s/it]\u001b[0m\n",
      "\u001b[34m28%|██▊       | 14/50 [00:12<00:41,  1.16s/it]\u001b[0m\n",
      "\u001b[34m30%|███       | 15/50 [00:13<00:44,  1.26s/it]\u001b[0m\n",
      "\u001b[34m32%|███▏      | 16/50 [00:14<00:42,  1.26s/it]\u001b[0m\n",
      "\u001b[34m34%|███▍      | 17/50 [00:16<00:41,  1.26s/it]\u001b[0m\n",
      "\u001b[34m36%|███▌      | 18/50 [00:17<00:40,  1.28s/it]\u001b[0m\n",
      "\u001b[34m38%|███▊      | 19/50 [00:18<00:39,  1.28s/it]\u001b[0m\n",
      "\u001b[34m40%|████      | 20/50 [00:20<00:39,  1.31s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8322, 'learning_rate': 0.0001, 'epoch': 0.0}\u001b[0m\n",
      "\u001b[34m40%|████      | 20/50 [00:20<00:39,  1.31s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 21/50 [00:21<00:38,  1.31s/it]\u001b[0m\n",
      "\u001b[34m44%|████▍     | 22/50 [00:22<00:37,  1.34s/it]\u001b[0m\n",
      "\u001b[34m46%|████▌     | 23/50 [00:24<00:36,  1.35s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 24/50 [00:25<00:36,  1.40s/it]\u001b[0m\n",
      "\u001b[34m50%|█████     | 25/50 [00:27<00:34,  1.39s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 26/50 [00:28<00:34,  1.45s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▍    | 27/50 [00:30<00:34,  1.48s/it]\u001b[0m\n",
      "\u001b[34m56%|█████▌    | 28/50 [00:31<00:32,  1.49s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 29/50 [00:33<00:31,  1.51s/it]\u001b[0m\n",
      "\u001b[34m60%|██████    | 30/50 [00:34<00:30,  1.53s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.5387, 'learning_rate': 0.0001, 'epoch': 0.01}\u001b[0m\n",
      "\u001b[34m60%|██████    | 30/50 [00:34<00:30,  1.53s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 31/50 [00:36<00:30,  1.58s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 32/50 [00:38<00:28,  1.60s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 33/50 [00:40<00:28,  1.68s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 34/50 [00:41<00:27,  1.73s/it]\u001b[0m\n",
      "\u001b[34m70%|███████   | 35/50 [00:43<00:26,  1.76s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 36/50 [00:45<00:25,  1.79s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 37/50 [00:47<00:24,  1.85s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 38/50 [00:49<00:22,  1.90s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 39/50 [00:51<00:21,  1.92s/it]\u001b[0m\n",
      "\u001b[34m80%|████████  | 40/50 [00:53<00:20,  2.01s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.2698, 'learning_rate': 0.0001, 'epoch': 0.01}\u001b[0m\n",
      "\u001b[34m80%|████████  | 40/50 [00:53<00:20,  2.01s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 41/50 [00:55<00:18,  2.04s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▍ | 42/50 [00:57<00:16,  2.02s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 43/50 [00:59<00:14,  2.03s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 44/50 [01:01<00:12,  2.04s/it]\u001b[0m\n",
      "\u001b[34m90%|█████████ | 45/50 [01:04<00:10,  2.04s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 46/50 [01:06<00:08,  2.08s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 47/50 [01:08<00:06,  2.14s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▌| 48/50 [01:10<00:04,  2.14s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 49/50 [01:12<00:02,  2.13s/it]\u001b[0m\n",
      "\u001b[34m100%|██████████| 50/50 [01:15<00:00,  2.17s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1364, 'learning_rate': 0.0001, 'epoch': 0.01}\u001b[0m\n",
      "\u001b[34m100%|██████████| 50/50 [01:15<00:00,  2.17s/it]\u001b[0m\n",
      "\u001b[34mSaving the whole model\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:458] 2024-09-28 14:09:39,616 >> Configuration saved in /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:458] 2024-09-28 14:09:39,616 >> Configuration saved in /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:364] 2024-09-28 14:09:39,617 >> Configuration saved in /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/generation_config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:364] 2024-09-28 14:09:39,617 >> Configuration saved in /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/generation_config.json\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:1853] 2024-09-28 14:10:01,001 >> Model weights saved in /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/pytorch_model.bin\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:1853] 2024-09-28 14:10:01,001 >> Model weights saved in /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/pytorch_model.bin\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2194] 2024-09-28 14:10:01,002 >> tokenizer config file saved in /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2194] 2024-09-28 14:10:01,002 >> tokenizer config file saved in /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2201] 2024-09-28 14:10:01,002 >> Special tokens file saved in /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2201] 2024-09-28 14:10:01,002 >> Special tokens file saved in /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:10:01,004] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step50 is about to be saved!\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:10:01,009] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/mp_rank_00_model_states.pt\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:10:01,009] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/mp_rank_00_model_states.pt...\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:10:19,541] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/mp_rank_00_model_states.pt.\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:10:19,543] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_0_mp_rank_00_optim_states.pt...\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:10:39,778] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_0_mp_rank_00_optim_states.pt.\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:10:39,778] [INFO] [engine.py:3536:_save_zero_checkpoint] zero checkpoint saved /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_0_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:10:39,778] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step50 is ready now!\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:2053] 2024-09-28 14:10:41,127 >> \u001b[0m\n",
      "\u001b[34mTraining completed. Do not forget to share your model on huggingface.co/models =)\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:2053] 2024-09-28 14:10:41,127 >> \u001b[0m\n",
      "\u001b[34mTraining completed. Do not forget to share your model on huggingface.co/models =)\u001b[0m\n",
      "\u001b[34m{'train_runtime': 136.5372, 'train_samples_per_second': 2.93, 'train_steps_per_second': 0.366, 'train_loss': 1.7166397094726562, 'epoch': 0.01}\u001b[0m\n",
      "\u001b[34m100%|██████████| 50/50 [02:16<00:00,  2.17s/it]\u001b[0m\n",
      "\u001b[34m------saving model!-----\u001b[0m\n",
      "\u001b[34m------saving model!-----\u001b[0m\n",
      "\u001b[34m------saving model!-----\u001b[0m\n",
      "\u001b[34m------saving model!-----\u001b[0m\n",
      "\u001b[34m------saving model!-----------saving model!-----------saving model!-----\u001b[0m\n",
      "\u001b[34m100%|██████████| 50/50 [02:16<00:00,  2.73s/it]\u001b[0m\n",
      "\u001b[34m***** train metrics *****\u001b[0m\n",
      "\u001b[34mepoch                    =       0.01\n",
      "  train_loss               =     1.7166\u001b[0m\n",
      "\u001b[34mtrain_runtime            = 0:02:16.53\n",
      "  train_samples            =      42351\n",
      "  train_samples_per_second =       2.93\u001b[0m\n",
      "\u001b[34mtrain_steps_per_second   =      0.366\u001b[0m\n",
      "\u001b[34m------saving model!-----\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2194] 2024-09-28 14:10:41,129 >> tokenizer config file saved in /tmp/model/adgen-chatglm2-6b-ft/tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2194] 2024-09-28 14:10:41,129 >> tokenizer config file saved in /tmp/model/adgen-chatglm2-6b-ft/tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2201] 2024-09-28 14:10:41,129 >> Special tokens file saved in /tmp/model/adgen-chatglm2-6b-ft/special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2201] 2024-09-28 14:10:41,129 >> Special tokens file saved in /tmp/model/adgen-chatglm2-6b-ft/special_tokens_map.json\u001b[0m\n",
      "\u001b[34msave_model_dir : /tmp/model/adgen-chatglm2-6b-ft/\u001b[0m\n",
      "\u001b[34m------model is saved!-----\u001b[0m\n",
      "\u001b[34mmodel sync to : s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/\u001b[0m\n",
      "\u001b[34mworld_rank===1\u001b[0m\n",
      "\u001b[34msave_model_dir : /tmp/model/adgen-chatglm2-6b-ft/\u001b[0m\n",
      "\u001b[34m------model is saved!-----\u001b[0m\n",
      "\u001b[34mmodel sync to : s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/\u001b[0m\n",
      "\u001b[34mworld_rank===3\u001b[0m\n",
      "\u001b[34msave_model_dir : /tmp/model/adgen-chatglm2-6b-ft/\u001b[0m\n",
      "\u001b[34m------model is saved!-----\u001b[0m\n",
      "\u001b[34mmodel sync to : s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/\u001b[0m\n",
      "\u001b[34mworld_rank===7\u001b[0m\n",
      "\u001b[34msave_model_dir : /tmp/model/adgen-chatglm2-6b-ft/\u001b[0m\n",
      "\u001b[34m------model is saved!-----\u001b[0m\n",
      "\u001b[34mmodel sync to : s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/\u001b[0m\n",
      "\u001b[34mworld_rank===5\u001b[0m\n",
      "\u001b[34msave_model_dir : /tmp/model/adgen-chatglm2-6b-ft/\u001b[0m\n",
      "\u001b[34m------model is saved!-----\u001b[0m\n",
      "\u001b[34mmodel sync to : s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/\u001b[0m\n",
      "\u001b[34mworld_rank===2\u001b[0m\n",
      "\u001b[34msave_model_dir : /tmp/model/adgen-chatglm2-6b-ft/\u001b[0m\n",
      "\u001b[34m------model is saved!-----\u001b[0m\n",
      "\u001b[34mmodel sync to : s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/\u001b[0m\n",
      "\u001b[34mworld_rank===6\u001b[0m\n",
      "\u001b[34msave_model_dir : /tmp/model/adgen-chatglm2-6b-ft/\u001b[0m\n",
      "\u001b[34m------model is saved!-----\u001b[0m\n",
      "\u001b[34mmodel sync to : s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/\u001b[0m\n",
      "\u001b[34mworld_rank===4\u001b[0m\n",
      "\u001b[34mSaving the whole model\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:458] 2024-09-28 14:10:41,133 >> Configuration saved in /tmp/model/adgen-chatglm2-6b-ft/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:458] 2024-09-28 14:10:41,133 >> Configuration saved in /tmp/model/adgen-chatglm2-6b-ft/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:364] 2024-09-28 14:10:41,134 >> Configuration saved in /tmp/model/adgen-chatglm2-6b-ft/generation_config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:364] 2024-09-28 14:10:41,134 >> Configuration saved in /tmp/model/adgen-chatglm2-6b-ft/generation_config.json\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:1853] 2024-09-28 14:11:00,824 >> Model weights saved in /tmp/model/adgen-chatglm2-6b-ft/pytorch_model.bin\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:1853] 2024-09-28 14:11:00,824 >> Model weights saved in /tmp/model/adgen-chatglm2-6b-ft/pytorch_model.bin\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2194] 2024-09-28 14:11:00,825 >> tokenizer config file saved in /tmp/model/adgen-chatglm2-6b-ft/tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2194] 2024-09-28 14:11:00,825 >> tokenizer config file saved in /tmp/model/adgen-chatglm2-6b-ft/tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2201] 2024-09-28 14:11:00,825 >> Special tokens file saved in /tmp/model/adgen-chatglm2-6b-ft/special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:2201] 2024-09-28 14:11:00,825 >> Special tokens file saved in /tmp/model/adgen-chatglm2-6b-ft/special_tokens_map.json\u001b[0m\n",
      "\u001b[34msave_model_dir : /tmp/model/adgen-chatglm2-6b-ft/\u001b[0m\n",
      "\u001b[34m------model is saved!-----\u001b[0m\n",
      "\u001b[34mmodel sync to : s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/\u001b[0m\n",
      "\u001b[34mworld_rank===0\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/generation_config.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/generation_config.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/latest s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/latest\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/rng_state_3.pth s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/rng_state_3.pth\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/rng_state_2.pth s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/rng_state_2.pth\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/rng_state_7.pth s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/rng_state_7.pth\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/config.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/config.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/special_tokens_map.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/special_tokens_map.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/generation_config.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/generation_config.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/special_tokens_map.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/special_tokens_map.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/rng_state_5.pth s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/rng_state_5.pth\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/training_args.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/training_args.bin\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/tokenizer_config.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/tokenizer_config.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/rng_state_0.pth s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/rng_state_0.pth\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/trainer_state.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/trainer_state.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/trainer_state.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/trainer_state.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/zero_to_fp32.py s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/zero_to_fp32.py\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/rng_state_1.pth s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/rng_state_1.pth\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/config.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/config.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/tokenizer_config.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/tokenizer_config.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/rng_state_6.pth s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/rng_state_6.pth\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/training_args.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/training_args.bin\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/all_results.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/all_results.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/rng_state_4.pth s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/rng_state_4.pth\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/train_results.json s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/train_results.json\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/tokenizer.model s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/tokenizer.model\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/tokenizer.model s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/tokenizer.model\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_4_mp_rank_00_optim_states.pt s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/global_step50/zero_pp_rank_4_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_6_mp_rank_00_optim_states.pt s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/global_step50/zero_pp_rank_6_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_2_mp_rank_00_optim_states.pt s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/global_step50/zero_pp_rank_2_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_5_mp_rank_00_optim_states.pt s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/global_step50/zero_pp_rank_5_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_3_mp_rank_00_optim_states.pt s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/global_step50/zero_pp_rank_3_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_7_mp_rank_00_optim_states.pt s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/global_step50/zero_pp_rank_7_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_1_mp_rank_00_optim_states.pt s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/global_step50/zero_pp_rank_1_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/zero_pp_rank_0_mp_rank_00_optim_states.pt s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/global_step50/zero_pp_rank_0_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/pytorch_model.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/pytorch_model.bin\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/global_step50/mp_rank_00_model_states.pt s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/global_step50/mp_rank_00_model_states.pt\u001b[0m\n",
      "\u001b[34mcp /tmp/model/adgen-chatglm2-6b-ft/checkpoint-50/pytorch_model.bin s3://sagemaker-us-west-2-687912291502/llm/models/chatglm2/finetune-fulltuning-adgen/checkpoint-50/pytorch_model.bin\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:11:37,620] [INFO] [launch.py:351:main] Process 367 exits successfully.\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:11:38,621] [INFO] [launch.py:351:main] Process 370 exits successfully.\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:11:38,622] [INFO] [launch.py:351:main] Process 372 exits successfully.\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:11:38,622] [INFO] [launch.py:351:main] Process 368 exits successfully.\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:11:38,622] [INFO] [launch.py:351:main] Process 374 exits successfully.\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:11:38,623] [INFO] [launch.py:351:main] Process 373 exits successfully.\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:11:38,623] [INFO] [launch.py:351:main] Process 369 exits successfully.\u001b[0m\n",
      "\u001b[34m[2024-09-28 14:11:38,623] [INFO] [launch.py:351:main] Process 371 exits successfully.\u001b[0m\n",
      "\u001b[34m2024-09-28 14:11:40,916 sagemaker-training-toolkit INFO     Waiting for the process to finish and give a return code.\u001b[0m\n",
      "\u001b[34m2024-09-28 14:11:40,916 sagemaker-training-toolkit INFO     Done waiting for a return code. Received 0 from exiting process.\u001b[0m\n",
      "\u001b[34m2024-09-28 14:11:40,916 sagemaker-training-toolkit INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2024-09-28 14:11:54 Uploading - Uploading generated training model\n",
      "2024-09-28 14:11:54 Completed - Training job completed\n",
      "Training seconds: 883\n",
      "Billable seconds: 883\n"
     ]
    }
   ],
   "source": [
    "huggingface_estimator.fit(inputs=inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ebf89bb-5d55-4871-b339-506742d4c4a1",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 4.4 部署微调模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92557cf7-950a-46ca-b58c-2e239bb02207",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.serializers import JSONSerializer\n",
    "from sagemaker.deserializers import JSONDeserializer\n",
    "\n",
    "endpoint_name         = None\n",
    "instance_type         = 'ml.g4dn.2xlarge'\n",
    "instance_count        = 1\n",
    "\n",
    "predictor = model.deploy(\n",
    "    endpoint_name          = endpoint_name,\n",
    "    instance_type          = instance_type, \n",
    "    initial_instance_count = instance_count,\n",
    "    serializer             = JSONSerializer(),\n",
    "    deserializer           = JSONDeserializer()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7e3c7af-c214-4db1-b2f7-0973f1f5eaa8",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 4.5 测试微调模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4edb8c37-13c3-4277-986e-aff82c85e39f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Wait model loading\n",
    "\n",
    "import time\n",
    "\n",
    "time.sleep(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b202bd61-2565-4ff8-b79a-811742391218",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = {\n",
    "    \"ask\": \"需求:完成用户的高低肩分析报告|数据信息:性别女 身高174cm 年龄23;体型:标准;高低肩数值: 1.2 可能存在左高\"\n",
    "\n",
    "}\n",
    "\n",
    "response = predictor.predict(inputs)\n",
    "print(response[\"answer\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92e25039-c0b4-4768-a057-5022da9139fd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = {\n",
    "    \"ask\": \"告诉我，你是谁？您能干什么？\"\n",
    "\n",
    "}\n",
    "\n",
    "response = predictor.predict(inputs)\n",
    "print(response[\"answer\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efad53ba-9eb3-47d9-8b86-80f44d5096b3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = {\n",
    "    \"ask\": \"对于ChatGLM的数据微调工作，你有什么想说的吗？\"\n",
    "\n",
    "}\n",
    "\n",
    "response = predictor.predict(inputs)\n",
    "print(response[\"answer\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a06d160e-5333-4843-a177-521bdbcf3971",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = {\n",
    "    \"ask\": \"需求:完成用户的体型分析报告|数据信息:性别男 身高175cm 年龄35;体型:标准;身体成分数据:体重 71.9kg [60.6-82], 肌肉质量 58.7kg [48.7-59.5], 体脂肪 10.3kg [8.1-16.2], 体脂率 14.3% [10-20], BMI 23.5 [18.5-24], 腰臀比 0.82 [0.8-0.9], 内脏脂肪等级 3 [0.9-10]\"\n",
    "}\n",
    "\n",
    "response = predictor.predict(inputs)\n",
    "print(response[\"answer\"])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "024da335",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 4.6 清除资源"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a914fa4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "predictor.delete_endpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "919e9907-b52f-4e31-837b-5b471a317614",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mERROR: Could not find a version that satisfies the requirement huggingface==0.99999 (from versions: 0.0.1)\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31mERROR: No matching distribution found for huggingface==0.99999\u001b[0m\u001b[31m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install huggingface==0.99999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6362da1d-ddda-4da1-98b8-c33b5e429ab5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p310",
   "language": "python",
   "name": "conda_pytorch_p310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}