streaming_beta_notebook_chatglm3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c261e5f4-17a8-40da-beb9-599f1717e0fe",
   "metadata": {},
   "source": [
    "### 1. 安装HuggingFace 并下载模型到本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02785614-9268-41c8-85a5-d579490edbbf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install huggingface-hub -Uqq\n",
    "!pip install -U sagemaker\n",
    "!pip install -U boto\n",
    "!pip install -U botocore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d846ed0-24cb-45d3-be44-42a28bf5ce72",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#%pip install sagemaker pip --upgrade  --quiet\n",
    "# Note the following may error depending on which awscli is installed in your jupyter kernel, \n",
    "# but that is ok \n",
    "#%pip install ./code/botocore-1.29.157-py3-none-any.whl ./code/boto3-1.26.157-py3-none-any.whl --force"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f6daddee-833f-4755-9541-b5e1ea675024",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!rm -rf ./LLM_chatglm3_stream_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9e6bd7ee-16a3-4f5a-8857-8bbba83eb9e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_model_path = Path(\"./LLM_chatglm3_stream_model\")\n",
    "local_model_path.mkdir(exist_ok=True)\n",
    "model_name = \"THUDM/chatglm3-6b\"\n",
    "commit_hash = \"fc3235f807ef5527af598c05f04f2ffd17f48bab\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "94e8abc5-a58e-40e2-b1e6-fbf48307c716",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70518fe69e0d4b1daf7ca3c61b0907f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 18 files:   0%|          | 0/18 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6513794e45cf4f358bbc66f55b40210f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00001-of-00007.bin:   0%|          | 0.00/1.83G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1f50e2b1dc44a7489afbdbb7094729f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)8c05f04f2ffd17f48bab/modeling_chatglm.py:   0%|          | 0.00/55.6k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44cbffe3aef74d0fa0f5c529b6b8ecf6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)7af598c05f04f2ffd17f48bab/.gitattributes:   0%|          | 0.00/1.52k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "982217e8d95646e580cafa1ffe5f5850",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)5527af598c05f04f2ffd17f48bab/config.json:   0%|          | 0.00/1.32k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de17889aefa64120a44b7cf21346b905",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00002-of-00007.bin:   0%|          | 0.00/1.97G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49f3fa898e9c4f58aae68698ecc5c4dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)04f2ffd17f48bab/configuration_chatglm.py:   0%|          | 0.00/2.33k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eae697429a974e82946379c4f110037e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)27af598c05f04f2ffd17f48bab/MODEL_LICENSE:   0%|          | 0.00/4.13k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7dc0321e85ef47cc8d4e08a7d6300782",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)ef5527af598c05f04f2ffd17f48bab/README.md:   0%|          | 0.00/5.57k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58a67663d5a3441394d7eeca5a6d2aa7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00004-of-00007.bin:   0%|          | 0.00/1.82G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c3f41abbfc34d4398a498e34e5e5576",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00003-of-00007.bin:   0%|          | 0.00/1.93G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22102d203b534c67a2f56d34c4d2d9af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00005-of-00007.bin:   0%|          | 0.00/1.97G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10ef65ddbf224e589876860339894e6d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00006-of-00007.bin:   0%|          | 0.00/1.93G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f77c96e4d5274a6b94b6a3461ef38f46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)ffd17f48bab/pytorch_model.bin.index.json:   0%|          | 0.00/20.4k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "212790d0de794891800f307573c048dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00007-of-00007.bin:   0%|          | 0.00/1.05G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1760be4e80df45d6a8344d5a6969a8d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)af598c05f04f2ffd17f48bab/quantization.py:   0%|          | 0.00/14.7k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "371a72c728454bfdb22f2798f847e777",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)f04f2ffd17f48bab/tokenization_chatglm.py:   0%|          | 0.00/11.3k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6047f46c44c0459995bc8ea398f46ca2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.model:   0%|          | 0.00/1.02M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26475b8e678f4f32b982ea964c0e133f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)05f04f2ffd17f48bab/tokenizer_config.json:   0%|          | 0.00/244 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snapshot_download(repo_id=model_name, cache_dir=local_model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d666c79-b039-4258-ac3b-46b19e63c3b8",
   "metadata": {},
   "source": [
    "### 2. 把模型拷贝到S3为后续部署做准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e9431deb-6359-442d-847b-1563f8dd3854",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n"
     ]
    }
   ],
   "source": [
    "import sagemaker\n",
    "from sagemaker import image_uris\n",
    "import boto3\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "\n",
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "bucket = sess.default_bucket()  # bucket to house artifacts\n",
    "\n",
    "region = sess._region_name\n",
    "account_id = sess.account_id()\n",
    "\n",
    "s3_client = boto3.client(\"s3\")\n",
    "sm_client = boto3.client(\"sagemaker\")\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "40dd8f16-ae7c-48bf-8e52-1a15425fa74d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3_code_prefix: LLM-RAG/workshop/LLM_chatglm3_stream_deploy_code\n",
      "model_snapshot_path: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab\n"
     ]
    }
   ],
   "source": [
    "s3_model_prefix = \"LLM-RAG/workshop/LLM_chatglm3_stream_model\"  # folder where model checkpoint will go\n",
    "model_snapshot_path = list(local_model_path.glob(\"**/snapshots/*\"))[0]\n",
    "s3_code_prefix = \"LLM-RAG/workshop/LLM_chatglm3_stream_deploy_code\"\n",
    "print(f\"s3_code_prefix: {s3_code_prefix}\")\n",
    "print(f\"model_snapshot_path: {model_snapshot_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "067292c9-c066-4649-a61f-b460a24da584",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/.gitattributes\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/MODEL_LICENSE\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/README.md\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/configuration_chatglm.py\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/config.json\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/modeling_chatglm.py\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model.bin.index.json\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/quantization.py\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/tokenization_chatglm.py\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00006-of-00007.bin\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00001-of-00007.bin\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00003-of-00007.bin\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00005-of-00007.bin\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/tokenizer_config.json\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00004-of-00007.bin\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00007-of-00007.bin\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/tokenizer.model\n",
      "delete: s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00002-of-00007.bin\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/.gitattributes to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/.gitattributes\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/config.json to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/config.json\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/README.md to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/README.md\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/modeling_chatglm.py to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/modeling_chatglm.py\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/MODEL_LICENSE to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/MODEL_LICENSE\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/configuration_chatglm.py to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/configuration_chatglm.py\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/pytorch_model-00004-of-00007.bin to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00004-of-00007.bin\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/pytorch_model-00001-of-00007.bin to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00001-of-00007.bin\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/pytorch_model.bin.index.json to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model.bin.index.json\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/pytorch_model-00003-of-00007.bin to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00003-of-00007.bin\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/quantization.py to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/quantization.py\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/tokenization_chatglm.py to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/tokenization_chatglm.py\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/tokenizer.model to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/tokenizer.model\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/tokenizer_config.json to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/tokenizer_config.json\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/pytorch_model-00002-of-00007.bin to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00002-of-00007.bin\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/pytorch_model-00005-of-00007.bin to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00005-of-00007.bin\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/pytorch_model-00007-of-00007.bin to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00007-of-00007.bin\n",
      "upload: LLM_chatglm3_stream_model/models--THUDM--chatglm3-6b/snapshots/fc3235f807ef5527af598c05f04f2ffd17f48bab/pytorch_model-00006-of-00007.bin to s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/pytorch_model-00006-of-00007.bin\n"
     ]
    }
   ],
   "source": [
    "!aws s3 rm --recursive s3://{bucket}/{s3_model_prefix}\n",
    "!aws s3 cp --recursive {model_snapshot_path} s3://{bucket}/{s3_model_prefix}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "696b70c3-90f1-4175-95bf-568bafbcd383",
   "metadata": {},
   "source": [
    "### 3. 模型部署准备（entrypoint脚本，容器镜像，服务配置）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "68d60730-528b-4707-9189-6bf6f5cad754",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.23.0-deepspeed0.9.5-cu118\n"
     ]
    }
   ],
   "source": [
    "inference_image_uri = (\n",
    "    f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.23.0-deepspeed0.9.5-cu118\"\n",
    ")\n",
    "\n",
    "#中国区需要替换为下面的image_uri\n",
    "# inference_image_uri = (\n",
    "#     f\"727897471807.dkr.ecr.{region}.amazonaws.com.cn/djl-inference:0.21.0-deepspeed0.8.3-cu117\"\n",
    "# )\n",
    "\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "8d771bdb-11d2-45d2-9bef-face29221838",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mkdir -p LLM_chatglm3_stream_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "e5348ecb-43df-4094-97d8-a6723004862a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_chatglm3_stream_deploy_code/model.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_chatglm3_stream_deploy_code/model.py\n",
    "from djl_python import Input, Output\n",
    "import torch\n",
    "import logging\n",
    "import math\n",
    "import os\n",
    "\n",
    "from transformers import pipeline, AutoModel, AutoTokenizer\n",
    "model = None\n",
    "tokenizer = None\n",
    "STOP_flag = \"[DONE]\"\n",
    "\n",
    "DEVICE = \"cuda\"\n",
    "DEVICE_ID = \"0\"\n",
    "CUDA_DEVICE = f\"{DEVICE}:{DEVICE_ID}\" if DEVICE_ID else DEVICE\n",
    "def torch_gc():\n",
    "    if torch.cuda.is_available():\n",
    "        with torch.cuda.device(CUDA_DEVICE):\n",
    "            torch.cuda.empty_cache()\n",
    "            torch.cuda.ipc_collect()\n",
    "            \n",
    "def load_model(properties):\n",
    "    global tokenizer,model\n",
    "    tensor_parallel = properties[\"tensor_parallel_degree\"]\n",
    "    model_location = properties['model_dir']\n",
    "    if \"model_id\" in properties:\n",
    "        model_location = properties['model_id']\n",
    "    logging.info(f\"Loading model in {model_location}\")\n",
    "    \n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_location, trust_remote_code=True)\n",
    "   \n",
    "    model = AutoModel.from_pretrained(model_location, trust_remote_code=True).half().cuda()\n",
    "    \n",
    "    model.requires_grad_(False)\n",
    "    model.eval()\n",
    "    \n",
    "    return model, tokenizer\n",
    "\n",
    "\n",
    "\n",
    "def stream_items(prompt, history, max_length, top_p, temperature):\n",
    "    global model, tokenizer\n",
    "    size = 0\n",
    "    response = \"\"\n",
    "    for response, history in model.stream_chat(tokenizer, prompt, history=history, max_length=max_length, top_p=top_p,\n",
    "                                               temperature=temperature):\n",
    "        this_response = response[size:]\n",
    "        history = [list(h) for h in history]\n",
    "        size = len(response)\n",
    "        stream_buffer = { \"outputs\":this_response}\n",
    "        yield stream_buffer\n",
    "    ## stop\n",
    "    # yield {\"query\": prompt, \"outputs\": STOP_flag, \"response\": response, \"history\": history, \"finished\": True}\n",
    "    \n",
    "\n",
    "\n",
    "def handle(inputs: Input):\n",
    "    global model, tokenizer\n",
    "    if not model:\n",
    "        model, tokenizer = load_model(inputs.get_properties())\n",
    "\n",
    "    if inputs.is_empty():\n",
    "        return None\n",
    "    data = inputs.get_as_json()\n",
    "    \n",
    "    input_sentences = data[\"inputs\"]\n",
    "    params = data[\"parameters\"]\n",
    "    history = data[\"history\"]\n",
    "    print(f'input prompt:{input_sentences}')    \n",
    "    outputs = Output()\n",
    "    outputs.add_property(\"content-type\", \"application/jsonlines\")\n",
    "    outputs.add_stream_content(stream_items(input_sentences,history=history,**params))\n",
    "    return outputs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "60a9a063-43ed-4bf2-9f51-200bca51d477",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "option.s3url ==> s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/\n"
     ]
    }
   ],
   "source": [
    "print(f\"option.s3url ==> s3://{bucket}/{s3_model_prefix}/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a06d1e60-3914-4059-a08f-05ac26761165",
   "metadata": {},
   "source": [
    "#### Note: option.s3url 需要按照自己的账号进行修改, 可以拷贝上一个cell的输出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "8996fe44-8e70-468b-abc1-38187cb33f4f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_chatglm3_stream_deploy_code/serving.properties\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_chatglm3_stream_deploy_code/serving.properties\n",
    "engine=Python\n",
    "option.tensor_parallel_degree=1\n",
    "option.enable_streaming=True\n",
    "option.predict_timeout=240\n",
    "option.s3url=s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d67241ec-7b58-47d4-afd9-1d1745ddbbcd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-10-30 00:51:55    1.5 KiB .gitattributes\n",
      "2023-10-30 00:51:55    4.0 KiB MODEL_LICENSE\n",
      "2023-10-30 00:51:55    5.4 KiB README.md\n",
      "2023-10-30 00:51:55    1.3 KiB config.json\n",
      "2023-10-30 00:51:55    2.3 KiB configuration_chatglm.py\n",
      "2023-10-30 00:51:55   54.3 KiB modeling_chatglm.py\n",
      "2023-10-30 00:51:55    1.7 GiB pytorch_model-00001-of-00007.bin\n",
      "2023-10-30 00:51:55    1.8 GiB pytorch_model-00002-of-00007.bin\n",
      "2023-10-30 00:51:55    1.8 GiB pytorch_model-00003-of-00007.bin\n",
      "2023-10-30 00:51:55    1.7 GiB pytorch_model-00004-of-00007.bin\n",
      "2023-10-30 00:51:55    1.8 GiB pytorch_model-00005-of-00007.bin\n",
      "2023-10-30 00:53:08    1.8 GiB pytorch_model-00006-of-00007.bin\n",
      "2023-10-30 00:53:08 1004.0 MiB pytorch_model-00007-of-00007.bin\n",
      "2023-10-30 00:53:12   20.0 KiB pytorch_model.bin.index.json\n",
      "2023-10-30 00:53:13   14.3 KiB quantization.py\n",
      "2023-10-30 00:53:13   11.0 KiB tokenization_chatglm.py\n",
      "2023-10-30 00:53:13  994.5 KiB tokenizer.model\n",
      "2023-10-30 00:53:13  244 Bytes tokenizer_config.json\n"
     ]
    }
   ],
   "source": [
    "!aws s3 ls s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_model/ --human"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feef22a2-27b9-4018-a46b-6a99b532512f",
   "metadata": {},
   "source": [
    "#### 注意: 必须把transformers升级到4.27.1以上，否则会出现 [Issue344](https://github.com/THUDM/ChatGLM-6B/issues/344)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7b7e76c6-6dbc-47fc-9f47-4765c526ab76",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_chatglm3_stream_deploy_code/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_chatglm3_stream_deploy_code/requirements.txt\n",
    "transformers==4.30.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "0ae6734a-aacd-410d-818d-0a962697c3c4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LLM_chatglm3_stream_deploy_code/\n",
      "LLM_chatglm3_stream_deploy_code/model.py\n",
      "LLM_chatglm3_stream_deploy_code/serving.properties\n",
      "LLM_chatglm3_stream_deploy_code/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "!rm model.tar.gz\n",
    "!cd LLM_chatglm3_stream_deploy_code && rm -rf \".ipynb_checkpoints\"\n",
    "!tar czvf model.tar.gz LLM_chatglm3_stream_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "0f77dc76-6d8c-4665-ba88-f03e887c136c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S3 Code or Model tar ball uploaded to --- > s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_chatglm3_stream_deploy_code/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "s3_code_artifact = sess.upload_data(\"model.tar.gz\", bucket, s3_code_prefix)\n",
    "print(f\"S3 Code or Model tar ball uploaded to --- > {s3_code_artifact}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5853daa-b8a3-4485-8c0a-64bf83e93a18",
   "metadata": {},
   "source": [
    "### 4. 创建模型 & 创建endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "ef974ca1-9638-45a8-9145-ea9d03b2b072",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chatglm-stream-2023-10-30-02-34-02-040\n",
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.23.0-deepspeed0.9.5-cu118\n",
      "Created Model: arn:aws:sagemaker:us-west-2:687912291502:model/chatglm-stream-2023-10-30-02-34-02-040\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.utils import name_from_base\n",
    "import boto3\n",
    "\n",
    "model_name = name_from_base(f\"chatglm-stream\") # Append a timestamp to the provided string\n",
    "print(model_name)\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")\n",
    "\n",
    "create_model_response = sm_client.create_model(\n",
    "    ModelName=model_name,\n",
    "    ExecutionRoleArn=role,\n",
    "    PrimaryContainer={\n",
    "        \"Image\": inference_image_uri,\n",
    "        \"ModelDataUrl\": s3_code_artifact,\n",
    "        'Environment': {\n",
    "            'SERVING_OPTS': '-Dai.djl.logging.level=DEBUG'\n",
    "        },\n",
    "    },\n",
    "    \n",
    ")\n",
    "model_arn = create_model_response[\"ModelArn\"]\n",
    "\n",
    "print(f\"Created Model: {model_arn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "233bb3a4-d737-41ad-8fcc-7082c6278e8c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'EndpointConfigArn': 'arn:aws:sagemaker:us-west-2:687912291502:endpoint-config/chatglm-stream-2023-10-30-02-34-02-040-config',\n",
       " 'ResponseMetadata': {'RequestId': '5c42c51f-cfa7-4e52-a9eb-17221ca70f8f',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '5c42c51f-cfa7-4e52-a9eb-17221ca70f8f',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '126',\n",
       "   'date': 'Mon, 30 Oct 2023 02:34:02 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "endpoint_config_name = f\"{model_name}-config\"\n",
    "endpoint_name = f\"{model_name}-endpoint\"\n",
    "\n",
    "#Note: ml.g4dn.2xlarge 也可以选择\n",
    "endpoint_config_response = sm_client.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"VariantName\": \"variant1\",\n",
    "            \"ModelName\": model_name,\n",
    "            \"InstanceType\": \"ml.g4dn.xlarge\",\n",
    "            \"InitialInstanceCount\": 1,\n",
    "            # \"VolumeSizeInGB\" : 400,\n",
    "            # \"ModelDataDownloadTimeoutInSeconds\": 2400,\n",
    "            \"ContainerStartupHealthCheckTimeoutInSeconds\": 10*60,\n",
    "        },\n",
    "    ],\n",
    ")\n",
    "endpoint_config_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "734a39b0-473e-4421-94c8-74d2b4105038",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created Endpoint: arn:aws:sagemaker:us-west-2:687912291502:endpoint/chatglm-stream-2023-10-30-02-34-02-040-endpoint\n"
     ]
    }
   ],
   "source": [
    "create_endpoint_response = sm_client.create_endpoint(\n",
    "    EndpointName=f\"{endpoint_name}\", EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "print(f\"Created Endpoint: {create_endpoint_response['EndpointArn']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1262e826-a810-401d-a5a9-f62febb24e5f",
   "metadata": {},
   "source": [
    "#### 持续检测模型部署进度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "08969928-6b9e-4d9c-a033-a31f5f77bdfb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: InService\n",
      "Arn: arn:aws:sagemaker:us-west-2:687912291502:endpoint/chatglm-stream-2023-10-30-02-34-02-040-endpoint\n",
      "Status: InService\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "status = resp[\"EndpointStatus\"]\n",
    "print(\"Status: \" + status)\n",
    "\n",
    "while status == \"Creating\":\n",
    "    time.sleep(60)\n",
    "    resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "    status = resp[\"EndpointStatus\"]\n",
    "    print(\"Status: \" + status)\n",
    "\n",
    "print(\"Arn: \" + resp[\"EndpointArn\"])\n",
    "print(\"Status: \" + status)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d985b427-3959-46f7-9a50-5a2b45e2d513",
   "metadata": {},
   "source": [
    "### 5. 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e56bfdaa-3469-4784-aa8a-e32177cde3f2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.92 ms, sys: 0 ns, total: 2.92 ms\n",
      "Wall time: 2.4 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import json\n",
    "import boto3\n",
    "\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")\n",
    "\n",
    "parameters = {\n",
    "  \"max_length\": 2048,\n",
    "  \"temperature\": 0.01,\n",
    "  \"top_p\":0.8\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "3a2b6b7b-2e54-4f83-9258-cfbb30cfc829",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import io\n",
    "\n",
    "endpoint_name=\"chatglm-stream-2023-10-30-02-34-02-040-endpoint\"\n",
    "class StreamScanner:\n",
    "    \"\"\"\n",
    "    A helper class for parsing the InvokeEndpointWithResponseStream event stream. \n",
    "    \n",
    "    The output of the model will be in the following format:\n",
    "    ```\n",
    "    b'{\"outputs\": [\" a\"]}\\n'\n",
    "    b'{\"outputs\": [\" challenging\"]}\\n'\n",
    "    b'{\"outputs\": [\" problem\"]}\\n'\n",
    "    ...\n",
    "    ```\n",
    "    \n",
    "    While usually each PayloadPart event from the event stream will contain a byte array \n",
    "    with a full json, this is not guaranteed and some of the json objects may be split across\n",
    "    PayloadPart events. For example:\n",
    "    ```\n",
    "    {'PayloadPart': {'Bytes': b'{\"outputs\": '}}\n",
    "    {'PayloadPart': {'Bytes': b'[\" problem\"]}\\n'}}\n",
    "    ```\n",
    "    \n",
    "    This class accounts for this by concatenating bytes written via the 'write' function\n",
    "    and then exposing a method which will return lines (ending with a '\\n' character) within\n",
    "    the buffer via the 'readlines' function. It maintains the position of the last read \n",
    "    position to ensure that previous bytes are not exposed again. \n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.buff = io.BytesIO()\n",
    "        self.read_pos = 0\n",
    "        \n",
    "    def write(self, content):\n",
    "        self.buff.seek(0, io.SEEK_END)\n",
    "        self.buff.write(content)\n",
    "        \n",
    "    def readlines(self):\n",
    "        self.buff.seek(self.read_pos)\n",
    "        for line in self.buff.readlines():\n",
    "            if line[-1] != b'\\n':\n",
    "                self.read_pos += len(line)\n",
    "                yield line[:-1]\n",
    "                \n",
    "    def reset(self):\n",
    "        self.read_pos = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "e2c13afb-2c4a-41da-a58e-4d52ea13a6d9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': '很久'}}\n",
      "{'outputs': {'outputs': '很久'}}\n",
      "{'outputs': {'outputs': '以前'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '有个'}}\n",
      "{'outputs': {'outputs': '小朋友'}}\n",
      "{'outputs': {'outputs': '叫'}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他'}}\n",
      "{'outputs': {'outputs': '有一'}}\n",
      "{'outputs': {'outputs': '只'}}\n",
      "{'outputs': {'outputs': '可爱的'}}\n",
      "{'outputs': {'outputs': '宠物'}}\n",
      "{'outputs': {'outputs': '——'}}\n",
      "{'outputs': {'outputs': '一只'}}\n",
      "{'outputs': {'outputs': '叫做'}}\n",
      "{'outputs': {'outputs': '“'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '”'}}\n",
      "{'outputs': {'outputs': '的小'}}\n",
      "{'outputs': {'outputs': '狗'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '和小'}}\n",
      "{'outputs': {'outputs': '白'}}\n",
      "{'outputs': {'outputs': '一起'}}\n",
      "{'outputs': {'outputs': '生活'}}\n",
      "{'outputs': {'outputs': '得'}}\n",
      "{'outputs': {'outputs': '非常'}}\n",
      "{'outputs': {'outputs': '快乐'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '每天'}}\n",
      "{'outputs': {'outputs': '他们在'}}\n",
      "{'outputs': {'outputs': '公园'}}\n",
      "{'outputs': {'outputs': '里'}}\n",
      "{'outputs': {'outputs': '玩耍'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '总是'}}\n",
      "{'outputs': {'outputs': '跟着'}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '成为了'}}\n",
      "{'outputs': {'outputs': '他'}}\n",
      "{'outputs': {'outputs': '最好的'}}\n",
      "{'outputs': {'outputs': '朋友'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': '\\n\\n然而'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '有一天'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '发现'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '不见'}}\n",
      "{'outputs': {'outputs': '了'}}\n",
      "{'outputs': {'outputs': '!'}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '非常'}}\n",
      "{'outputs': {'outputs': '着急'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他'}}\n",
      "{'outputs': {'outputs': '开始'}}\n",
      "{'outputs': {'outputs': '在'}}\n",
      "{'outputs': {'outputs': '公园'}}\n",
      "{'outputs': {'outputs': '里'}}\n",
      "{'outputs': {'outputs': '四处'}}\n",
      "{'outputs': {'outputs': '寻找'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '他'}}\n",
      "{'outputs': {'outputs': '问'}}\n",
      "{'outputs': {'outputs': '了'}}\n",
      "{'outputs': {'outputs': '所有'}}\n",
      "{'outputs': {'outputs': '的小朋友'}}\n",
      "{'outputs': {'outputs': '和'}}\n",
      "{'outputs': {'outputs': '工作人员'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '但是'}}\n",
      "{'outputs': {'outputs': '没有人'}}\n",
      "{'outputs': {'outputs': '见过'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '开始'}}\n",
      "{'outputs': {'outputs': '担心'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他'}}\n",
      "{'outputs': {'outputs': '觉得'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '可能'}}\n",
      "{'outputs': {'outputs': '被'}}\n",
      "{'outputs': {'outputs': '别人'}}\n",
      "{'outputs': {'outputs': '抓'}}\n",
      "{'outputs': {'outputs': '走了'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': '\\n\\n于是'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '决定'}}\n",
      "{'outputs': {'outputs': '要'}}\n",
      "{'outputs': {'outputs': '自己'}}\n",
      "{'outputs': {'outputs': '找回'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '他和'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '一起'}}\n",
      "{'outputs': {'outputs': '生活'}}\n",
      "{'outputs': {'outputs': '了'}}\n",
      "{'outputs': {'outputs': '那么'}}\n",
      "{'outputs': {'outputs': '长时间'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '之间'}}\n",
      "{'outputs': {'outputs': '有着'}}\n",
      "{'outputs': {'outputs': '非常'}}\n",
      "{'outputs': {'outputs': '深厚的'}}\n",
      "{'outputs': {'outputs': '感情'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '决定'}}\n",
      "{'outputs': {'outputs': '要'}}\n",
      "{'outputs': {'outputs': '踏上'}}\n",
      "{'outputs': {'outputs': '冒险'}}\n",
      "{'outputs': {'outputs': '之旅'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '寻找'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': '\\n\\n小明'}}\n",
      "{'outputs': {'outputs': '带着'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '走'}}\n",
      "{'outputs': {'outputs': '遍'}}\n",
      "{'outputs': {'outputs': '了'}}\n",
      "{'outputs': {'outputs': '整个'}}\n",
      "{'outputs': {'outputs': '城市'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '来到了'}}\n",
      "{'outputs': {'outputs': '动物园'}}\n",
      "{'outputs': {'outputs': '、'}}\n",
      "{'outputs': {'outputs': '狗'}}\n",
      "{'outputs': {'outputs': '舍'}}\n",
      "{'outputs': {'outputs': '、'}}\n",
      "{'outputs': {'outputs': '公园'}}\n",
      "{'outputs': {'outputs': '等等'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '还'}}\n",
      "{'outputs': {'outputs': '去'}}\n",
      "{'outputs': {'outputs': '向'}}\n",
      "{'outputs': {'outputs': '了'}}\n",
      "{'outputs': {'outputs': '警察'}}\n",
      "{'outputs': {'outputs': '局'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '希望'}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '能够'}}\n",
      "{'outputs': {'outputs': '提供'}}\n",
      "{'outputs': {'outputs': '一些'}}\n",
      "{'outputs': {'outputs': '帮助'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '但是'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '并没有'}}\n",
      "{'outputs': {'outputs': '找到'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': '\\n\\n小明'}}\n",
      "{'outputs': {'outputs': '开始'}}\n",
      "{'outputs': {'outputs': '感到'}}\n",
      "{'outputs': {'outputs': '沮丧'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '但是他'}}\n",
      "{'outputs': {'outputs': '并没有'}}\n",
      "{'outputs': {'outputs': '放弃'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '他'}}\n",
      "{'outputs': {'outputs': '决定'}}\n",
      "{'outputs': {'outputs': '要'}}\n",
      "{'outputs': {'outputs': '带着'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '去'}}\n",
      "{'outputs': {'outputs': '寻找'}}\n",
      "{'outputs': {'outputs': '更多的'}}\n",
      "{'outputs': {'outputs': '线索'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '来到了'}}\n",
      "{'outputs': {'outputs': '一些'}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '之前'}}\n",
      "{'outputs': {'outputs': '从未'}}\n",
      "{'outputs': {'outputs': '去'}}\n",
      "{'outputs': {'outputs': '过的'}}\n",
      "{'outputs': {'outputs': '地区'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '看到了'}}\n",
      "{'outputs': {'outputs': '很多'}}\n",
      "{'outputs': {'outputs': '美景'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '也'}}\n",
      "{'outputs': {'outputs': '遇到了'}}\n",
      "{'outputs': {'outputs': '很多'}}\n",
      "{'outputs': {'outputs': '有趣'}}\n",
      "{'outputs': {'outputs': '的事情'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': '\\n\\n在一次'}}\n",
      "{'outputs': {'outputs': '旅'}}\n",
      "{'outputs': {'outputs': '途中'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '和小'}}\n",
      "{'outputs': {'outputs': '白'}}\n",
      "{'outputs': {'outputs': '发现'}}\n",
      "{'outputs': {'outputs': '了一个'}}\n",
      "{'outputs': {'outputs': '神秘的'}}\n",
      "{'outputs': {'outputs': '洞'}}\n",
      "{'outputs': {'outputs': '穴'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '决定'}}\n",
      "{'outputs': {'outputs': '进去'}}\n",
      "{'outputs': {'outputs': '看看'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '看看'}}\n",
      "{'outputs': {'outputs': '里面'}}\n",
      "{'outputs': {'outputs': '是否有'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '走了'}}\n",
      "{'outputs': {'outputs': '很'}}\n",
      "{'outputs': {'outputs': '远'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '终于'}}\n",
      "{'outputs': {'outputs': '来到了'}}\n",
      "{'outputs': {'outputs': '洞'}}\n",
      "{'outputs': {'outputs': '穴'}}\n",
      "{'outputs': {'outputs': '的'}}\n",
      "{'outputs': {'outputs': '深处'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '在那里'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '发现'}}\n",
      "{'outputs': {'outputs': '了一'}}\n",
      "{'outputs': {'outputs': '只'}}\n",
      "{'outputs': {'outputs': '小'}}\n",
      "{'outputs': {'outputs': '狗'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '它'}}\n",
      "{'outputs': {'outputs': '看起来'}}\n",
      "{'outputs': {'outputs': '非常'}}\n",
      "{'outputs': {'outputs': '像'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '!'}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': '\\n\\n小明'}}\n",
      "{'outputs': {'outputs': '和小'}}\n",
      "{'outputs': {'outputs': '白'}}\n",
      "{'outputs': {'outputs': '非常'}}\n",
      "{'outputs': {'outputs': '高兴'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '终于'}}\n",
      "{'outputs': {'outputs': '找到了'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '!'}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '把'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '带'}}\n",
      "{'outputs': {'outputs': '回家'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '给'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '洗澡'}}\n",
      "{'outputs': {'outputs': '、'}}\n",
      "{'outputs': {'outputs': '梳理'}}\n",
      "{'outputs': {'outputs': '毛'}}\n",
      "{'outputs': {'outputs': '发'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '还'}}\n",
      "{'outputs': {'outputs': '给'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '准备了'}}\n",
      "{'outputs': {'outputs': '美味的'}}\n",
      "{'outputs': {'outputs': '食物'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '和小'}}\n",
      "{'outputs': {'outputs': '白'}}\n",
      "{'outputs': {'outputs': '非常'}}\n",
      "{'outputs': {'outputs': '感激'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '决定'}}\n",
      "{'outputs': {'outputs': '要'}}\n",
      "{'outputs': {'outputs': '永远'}}\n",
      "{'outputs': {'outputs': '做好'}}\n",
      "{'outputs': {'outputs': '朋友'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': ''}}\n",
      "{'outputs': {'outputs': '\\n\\n从此'}}\n",
      "{'outputs': {'outputs': '以后'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '和小'}}\n",
      "{'outputs': {'outputs': '白'}}\n",
      "{'outputs': {'outputs': '一起'}}\n",
      "{'outputs': {'outputs': '生活'}}\n",
      "{'outputs': {'outputs': '得'}}\n",
      "{'outputs': {'outputs': '更加'}}\n",
      "{'outputs': {'outputs': '快乐'}}\n",
      "{'outputs': {'outputs': '了'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '每天'}}\n",
      "{'outputs': {'outputs': '都在'}}\n",
      "{'outputs': {'outputs': '公园'}}\n",
      "{'outputs': {'outputs': '里'}}\n",
      "{'outputs': {'outputs': '玩耍'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '享受'}}\n",
      "{'outputs': {'outputs': '着'}}\n",
      "{'outputs': {'outputs': '彼此的'}}\n",
      "{'outputs': {'outputs': '陪伴'}}\n",
      "{'outputs': {'outputs': '。'}}\n",
      "{'outputs': {'outputs': '小白'}}\n",
      "{'outputs': {'outputs': '也'}}\n",
      "{'outputs': {'outputs': '成为了'}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '最好的'}}\n",
      "{'outputs': {'outputs': '朋友'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '他们'}}\n",
      "{'outputs': {'outputs': '一起'}}\n",
      "{'outputs': {'outputs': '经历了'}}\n",
      "{'outputs': {'outputs': '许多'}}\n",
      "{'outputs': {'outputs': '冒险'}}\n",
      "{'outputs': {'outputs': ','}}\n",
      "{'outputs': {'outputs': '让'}}\n",
      "{'outputs': {'outputs': '小明'}}\n",
      "{'outputs': {'outputs': '的生活'}}\n",
      "{'outputs': {'outputs': '变得更加'}}\n",
      "{'outputs': {'outputs': '丰富多彩'}}\n",
      "{'outputs': {'outputs': '。'}}\n"
     ]
    }
   ],
   "source": [
    "prompts1 = \"\"\"帮我写一个童话故事，400字左右，主题是丢失的宠物冒险： 描述一个小朋友的宠物走失了，主人为了找回宠物踏上了冒险之旅，他们一起经历了什么？\"\"\"\n",
    "# prompts1 = \"\"\"write a 500 words story about scifiction\"\"\"\n",
    "response_model = smr_client.invoke_endpoint_with_response_stream(\n",
    "            EndpointName=endpoint_name,\n",
    "            Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": prompts1,\n",
    "                \"parameters\": parameters,\n",
    "                \"history\" : []\n",
    "            }\n",
    "            ),\n",
    "            ContentType=\"application/json\",\n",
    "        )\n",
    "\n",
    "event_stream = response_model['Body']\n",
    "scanner = StreamScanner()\n",
    "for event in event_stream:\n",
    "    #print(event)\n",
    "    scanner.write(event['PayloadPart']['Bytes'])\n",
    "    for line in scanner.readlines():\n",
    "        try:\n",
    "            resp = json.loads(line)\n",
    "            print(resp)\n",
    "            # print(resp.get(\"outputs\")['outputs'], end='')\n",
    "        except Exception as e:\n",
    "            # print(line)\n",
    "            continue"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21c8b703-e312-4964-8be9-a754468e07cd",
   "metadata": {},
   "source": [
    "#### 清除模型Endpoint和config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f70d116f-4fb1-4f04-8732-3d6e4fb520de",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint --endpoint-name  {endpoint_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "184e4d1d-3d62-43df-9b17-5d64ece928bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint-config --endpoint-config-name {endpoint_config_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b7ae59f-caae-4719-b84f-dbc9ac36990f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-model --model-name  {model_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "395747d2-7d5d-41e7-b23b-a922f9f411b1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "conda_pytorch_p310",
   "language": "python",
   "name": "conda_pytorch_p310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}