sagemaker-inference-server-batch-chatglm2.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1ae65fec",
   "metadata": {},
   "source": [
    "### 1. 安装HuggingFace 并下载模型到本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2122fbd3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install huggingface-hub -Uqq\n",
    "!pip install -U sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfad28cc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_model_path = Path(\"./LLM_chatglm2_model\")\n",
    "local_model_path.mkdir(exist_ok=True)\n",
    "model_name = \"THUDM/chatglm2-6b\"\n",
    "commit_hash = \"b259b27320263629b0afccef134c54028233673d\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e36cc29",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "snapshot_download(repo_id=model_name, revision=commit_hash, cache_dir=local_model_path)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e2407531",
   "metadata": {},
   "source": [
    "### 2. 把模型拷贝到S3为后续部署做准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92eb50c8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from sagemaker.model import Model\n",
    "from sagemaker import serializers, deserializers\n",
    "from sagemaker import image_uris\n",
    "import boto3\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "\n",
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "bucket = sess.default_bucket()  # bucket to house artifacts\n",
    "\n",
    "region = sess._region_name\n",
    "account_id = sess.account_id()\n",
    "\n",
    "s3_client = boto3.client(\"s3\")\n",
    "sm_client = boto3.client(\"sagemaker\")\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "810c5e42",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import jinja2\n",
    "from pathlib import Path\n",
    "jinja_env = jinja2.Environment() # jinja environment to generate model configuration templates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "356ef70d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3_model_prefix = \"LLM-RAG/workshop/LLM_chatglm2_model\"  # folder where model checkpoint will go\n",
    "model_snapshot_path = list(local_model_path.glob(\"**/snapshots/*\"))[0]\n",
    "s3_code_prefix = \"LLM-RAG/workshop/LLM_chatglm2_sb_deploy_code\"\n",
    "print(f\"s3_code_prefix: {s3_code_prefix}\")\n",
    "print(f\"model_snapshot_path: {model_snapshot_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26771102",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws s3 cp --recursive {model_snapshot_path} s3://{bucket}/{s3_model_prefix}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cde11a06",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3_model_location = f\"s3://{bucket}/{s3_model_prefix}/\"\n",
    "print(\"s3_model_location => {}\".format(s3_model_location))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "a674dd94",
   "metadata": {},
   "source": [
    "### 3. 模型部署准备（entrypoint脚本，容器镜像，服务配置）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38930529",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# lookup the inference image uri based on our current region\n",
    "inference_image_uri = (\n",
    "    f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.23.0-deepspeed0.9.5-cu118\"\n",
    ")\n",
    "\n",
    "#中国区需要替换为下面的image_uri\n",
    "# inference_image_uri = (\n",
    "#     f\"727897471807.dkr.ecr.{region}.amazonaws.com.cn/djl-inference:0.21.0-deepspeed0.8.3-cu117\"\n",
    "# )\n",
    "\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ba81931",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mkdir -p LLM_chatglm2_sb_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a502a4cf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile LLM_chatglm2_sb_deploy_code/serving.template\n",
    "engine=DeepSpeed\n",
    "option.entryPoint=model.py\n",
    "option.tensor_parallel_degree=1\n",
    "option.s3url={{ s3url }}\n",
    "batch_size=4\n",
    "max_batch_delay=1000\n",
    "#option.task=text-generation\n",
    "#option.device_map=auto\n",
    "#option.load_in_8bit=TRUE #lyb: quantize directly?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea4d11fc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# we plug in the appropriate model location into our `serving.properties` file based on the region in which this notebook is running\n",
    "template = jinja_env.from_string(Path(\"LLM_chatglm2_sb_deploy_code/serving.template\").open().read())\n",
    "Path(\"LLM_chatglm2_sb_deploy_code/serving.properties\").open(\"w\").write(template.render(s3url=s3_model_location))\n",
    "!pygmentize LLM_chatglm2_sb_deploy_code/serving.properties | cat -n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ffbdf9f-4faa-48bb-b223-e19be3da0bcd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile LLM_chatglm2_sb_deploy_code/model.py\n",
    "from djl_python import Input, Output\n",
    "import os\n",
    "import logging\n",
    "import torch\n",
    "import deepspeed\n",
    "import transformers\n",
    "from transformers import pipeline, AutoTokenizer, AutoModel\n",
    "import re\n",
    "\n",
    "model = None\n",
    "#here, we need to set the global variable batch_size according to the batch_size in the serving.properties file.\n",
    "batch_size = 4\n",
    "\n",
    "def process_response(response):\n",
    "    response = response.strip()\n",
    "    response = response.replace(\"[[训练时间]]\", \"2023年\")\n",
    "    punkts = [\n",
    "        [\",\", \"，\"],\n",
    "        [\"!\", \"！\"],\n",
    "        [\":\", \"：\"],\n",
    "        [\";\", \"；\"],\n",
    "        [\"\\?\", \"？\"],\n",
    "    ]\n",
    "    for item in punkts:\n",
    "        response = re.sub(r\"([\\u4e00-\\u9fff])%s\" % item[0], r\"\\1%s\" % item[1], response)\n",
    "        response = re.sub(r\"%s([\\u4e00-\\u9fff])\" % item[0], r\"%s\\1\" % item[1], response)\n",
    "    return response\n",
    "\n",
    "def load_model(properties):\n",
    "\n",
    "    model_location = properties['model_dir']\n",
    "    tensor_parallel = properties[\"tensor_parallel_degree\"]\n",
    "    \n",
    "    if \"model_id\" in properties:\n",
    "        model_location = properties['model_id']\n",
    "    logging.info(f\"Loading model in {model_location}\")\n",
    "    \n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_location, trust_remote_code=True)\n",
    "\n",
    "    model = AutoModel.from_pretrained(model_location, low_cpu_mem_usage=True, torch_dtype=torch.float16, trust_remote_code=True)\n",
    "    print(\"----------model dtype is {0}---------\".format(model.dtype))\n",
    "    \n",
    "    model = deepspeed.init_inference(\n",
    "        model,\n",
    "        mp_size=tensor_parallel,\n",
    "        dtype=torch.half,\n",
    "        replace_method=\"auto\",\n",
    "        replace_with_kernel_inject=True,\n",
    "    )\n",
    "    \n",
    "#     local_rank = int(os.getenv(\"LOCAL_RANK\", \"0\"))\n",
    "#     generator = pipeline(task=\"text-generation\", model=model, tokenizer=tokenizer, use_cache=True, device=local_rank)\n",
    "    \n",
    "    model.requires_grad_(False)\n",
    "    model.eval()\n",
    "    \n",
    "    return model, tokenizer\n",
    "\n",
    "def handle(inputs: Input) -> None:\n",
    "\n",
    "    global model, tokenizer\n",
    "\n",
    "    try:\n",
    "        if not model:\n",
    "            model,tokenizer = load_model(inputs.get_properties())\n",
    "\n",
    "        if inputs.is_empty():\n",
    "            # Model server makes an empty call to warmup the model on startup\n",
    "            return None\n",
    "        \n",
    "        if inputs.is_batch():\n",
    "            #the demo code is just suitable for single sample per client request\n",
    "            bs = inputs.get_batch_size()\n",
    "            logging.info(f\"Dynamic batching size: {bs}.\")\n",
    "            batch = inputs.get_batches()\n",
    "            \n",
    "            tmp_inputs = []\n",
    "            for _, item in enumerate(batch):\n",
    "                tmp_item = item.get_as_json()\n",
    "                tmp_inputs.append(tmp_item.get(\"inputs\"))\n",
    "            \n",
    "            #For server side batch, we just use the custom generation parameters for single Sagemaker Endpoint.\n",
    "            inputs = tokenizer(tmp_inputs, return_tensors=\"pt\")\n",
    "            inputs = inputs.to('cuda')\n",
    "\n",
    "            gen_kwargs = {\"max_length\": 512, \"do_sample\": True,\n",
    "                      \"temperature\": 1.0}\n",
    "            results = model.generate(**inputs, **gen_kwargs)\n",
    "            \n",
    "            outputs = Output()\n",
    "            for i in range(len(results)):\n",
    "                exist_len = len(inputs[\"input_ids\"][i])\n",
    "                tem_result = results[i][exist_len:]\n",
    "                decode_result = tokenizer.decode(tem_result)\n",
    "                outputs.add(decode_result, key=\"generate_text\", batch_index=i)\n",
    "\n",
    "            return outputs\n",
    "        else:\n",
    "            inputs = inputs.get_as_json()\n",
    "            if not inputs.get(\"inputs\"):\n",
    "                return Output().add_as_json({\"code\":-1,\"msg\":\"input field can't be null\"})\n",
    "\n",
    "            #input data\n",
    "            data = inputs.get(\"inputs\")\n",
    "            params = inputs.get(\"parameters\",{})\n",
    "\n",
    "            #for pure client side batch\n",
    "            if type(data) == str:\n",
    "                bs = 1\n",
    "            elif type(data) == list:\n",
    "                if len(data) > batch_size:\n",
    "                    bs = batch_size\n",
    "                else:\n",
    "                    bs = len(data)\n",
    "            else:\n",
    "                return Output().add_as_json({\"code\":-1,\"msg\": \"input has wrong type\"})\n",
    "                \n",
    "            print(\"client side batch size is \", bs)\n",
    "            #predictor\n",
    "#             result = predictor(data, batch_size = bs, **params)\n",
    "#             result = model.chat(tokenizer, data, **params)\n",
    "            inputs = tokenizer(data, return_tensors=\"pt\")\n",
    "            inputs = inputs.to('cuda')\n",
    "\n",
    "            result = model.generate(**inputs, **params)\n",
    "            result = result.tolist()[0][len(inputs[\"input_ids\"][0]):]\n",
    "            result = tokenizer.decode(result)\n",
    "            result = process_response(result)\n",
    "            #return\n",
    "            return Output().add({\"code\":0,\"msg\":\"ok\",\"data\":result})\n",
    "    except Exception as e:\n",
    "        return Output().add_as_json({\"code\":-1,\"msg\":e})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "762f512f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile LLM_chatglm2_sb_deploy_code/requirements.txt\n",
    "transformers==4.28.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fb8c94f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!rm model.tar.gz\n",
    "!cd LLM_chatglm2_sb_deploy_code && rm -rf \".ipynb_checkpoints\"\n",
    "!tar czvf model.tar.gz LLM_chatglm2_sb_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af7bcb54",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3_code_artifact = sess.upload_data(\"model.tar.gz\", bucket, s3_code_prefix)\n",
    "print(f\"S3 Code or Model tar ball uploaded to --- > {s3_code_artifact}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "733244da",
   "metadata": {},
   "source": [
    "### 4. 创建模型 & 创建endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20917372",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.utils import name_from_base\n",
    "import boto3\n",
    "\n",
    "model_name = \"chatglm2-sb-2023-07-26-02-32-59-520\"#name_from_base(f\"chatglm2-sb\") #Note: Need to specify model_name\n",
    "print(model_name)\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")\n",
    "\n",
    "create_model_response = sm_client.create_model(\n",
    "    ModelName=model_name,\n",
    "    ExecutionRoleArn=role,\n",
    "    PrimaryContainer={\n",
    "        \"Image\": inference_image_uri,\n",
    "        \"ModelDataUrl\": s3_code_artifact\n",
    "    },\n",
    "    \n",
    ")\n",
    "model_arn = create_model_response[\"ModelArn\"]\n",
    "\n",
    "print(f\"Created Model: {model_arn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca111589",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "endpoint_config_name = f\"{model_name}-config\"\n",
    "endpoint_name = f\"{model_name}-endpoint\"\n",
    "\n",
    "#Note: ml.g4dn.2xlarge 也可以选择\n",
    "endpoint_config_response = sm_client.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"VariantName\": \"variant1\",\n",
    "            \"ModelName\": model_name,\n",
    "            \"InstanceType\": \"ml.g4dn.2xlarge\",\n",
    "            \"InitialInstanceCount\": 2,\n",
    "            # \"VolumeSizeInGB\" : 400,\n",
    "            # \"ModelDataDownloadTimeoutInSeconds\": 2400,\n",
    "            \"ContainerStartupHealthCheckTimeoutInSeconds\": 15*60,\n",
    "        },\n",
    "    ],\n",
    ")\n",
    "endpoint_config_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5eb29d3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "create_endpoint_response = sm_client.create_endpoint(\n",
    "    EndpointName=f\"{endpoint_name}\", EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "print(f\"Created Endpoint: {create_endpoint_response['EndpointArn']}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "38af5da7",
   "metadata": {},
   "source": [
    "#### 持续检测模型部署进度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c38cde1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "status = resp[\"EndpointStatus\"]\n",
    "print(\"Status: \" + status)\n",
    "\n",
    "while status == \"Creating\":\n",
    "    time.sleep(60)\n",
    "    resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "    status = resp[\"EndpointStatus\"]\n",
    "    print(\"Status: \" + status)\n",
    "\n",
    "print(\"Arn: \" + resp[\"EndpointArn\"])\n",
    "print(\"Status: \" + status)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "a0b9f012",
   "metadata": {},
   "source": [
    "### 5. 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57ffeae5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "import json\n",
    "import boto3\n",
    "\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")\n",
    "\n",
    "parameters = {\n",
    "  \"max_length\": 2048,\n",
    "  \"temperature\": 0.01,\n",
    "  \"num_beams\": 1, # >1可能会报错，\"probability tensor contains either `inf`, `nan` or element < 0\"； 即使remove_invalid_values=True也不能解决\n",
    "  \"do_sample\": False,\n",
    "  \"top_p\": 0.7,\n",
    "  \"logits_processor\" : None,\n",
    "  # \"remove_invalid_values\" : True\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b4c37df",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "prompts1 = \"\"\"AWS Clean Rooms 的FAQ文档有提到 Q: 是否发起者和数据贡献者都会被收费？A: 是单方收费，只有查询的接收方会收费。\n",
    "请问AWS Clean Rooms是多方都会收费吗？\n",
    "\"\"\"\n",
    "response_model = smr_client.invoke_endpoint(\n",
    "            EndpointName=endpoint_name,\n",
    "            Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": prompts1,\n",
    "                \"parameters\": parameters,\n",
    "                \"history\" : []\n",
    "            }\n",
    "            ),\n",
    "            ContentType=\"application/json\",\n",
    "        )\n",
    "\n",
    "response_model['Body'].read().decode('utf8')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "536da638",
   "metadata": {},
   "source": [
    "#### 清除模型Endpoint和config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af96d0c0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint --endpoint-name chatglm2-sb-2023-07-26-02-32-59-520-endpoint\n",
    "!aws sagemaker delete-endpoint-config --endpoint-config-name chatglm2-sb-2023-07-26-02-32-59-520-config\n",
    "!aws sagemaker delete-model --model-name chatglm2-sb-2023-07-26-02-32-59-520"
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.m5.large",
  "kernelspec": {
   "display_name": "Python 3 (Data Science 3.0)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-west-2:236514542706:image/sagemaker-data-science-310-v1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}