langchain-sagemaker-endpoint-pdf-summary.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c1069cd2",
   "metadata": {},
   "source": [
    "# PDF Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bb587555",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install --upgrade sagemaker --quiet\n",
    "!pip install langchain --quiet\n",
    "\n",
    "#!pip install faiss-cpu --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "11a17c39",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import sagemaker, boto3, json\n",
    "from sagemaker.session import Session\n",
    "from sagemaker.model import Model\n",
    "from sagemaker import image_uris, model_uris, script_uris, hyperparameters\n",
    "from sagemaker.predictor import Predictor\n",
    "from sagemaker.utils import name_from_base\n",
    "from typing import Any, Dict, List, Optional\n",
    "from langchain.embeddings import SagemakerEndpointEmbeddings\n",
    "from langchain.llms.sagemaker_endpoint import ContentHandlerBase\n",
    "\n",
    "sagemaker_session = Session()\n",
    "aws_role = sagemaker_session.get_caller_identity_arn()\n",
    "\n",
    "sess = sagemaker.Session()\n",
    "model_version = \"*\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c956fa37",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3, json\n",
    "from sagemaker.session import Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a6cce8ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "sagemaker_session = Session()\n",
    "aws_role = sagemaker_session.get_caller_identity_arn()\n",
    "aws_region = boto3.Session().region_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d8a67163",
   "metadata": {},
   "outputs": [],
   "source": [
    "endpoint_name = 'jumpstart-dft-hf-llm-falcon-7b-instruct-bf16'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "710649d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain import SagemakerEndpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c6e1ee34",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.llms.sagemaker_endpoint import LLMContentHandler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c192ca03",
   "metadata": {},
   "outputs": [],
   "source": [
    "aws_region = boto3.Session().region_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1932cb05",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ContentHandler(LLMContentHandler):\n",
    "    content_type = \"application/json\"\n",
    "    accepts = \"application/json\"\n",
    "\n",
    "    def transform_input(self, prompt: str, model_kwargs: dict) -> bytes:\n",
    "        input_str = json.dumps({'inputs': prompt, 'parameters': model_kwargs})\n",
    "        # input_str = json.dumps({'inputs': prompt, **model_kwargs})\n",
    "        return input_str.encode('utf-8')\n",
    "      \n",
    "    def transform_output(self, output: bytes) -> str:\n",
    "        response_json = json.loads(output.read().decode(\"utf-8\"))\n",
    "        return response_json[0][\"generated_text\"]\n",
    "\n",
    "content_handler = ContentHandler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9e1c0fec",
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "    \"max_new_tokens\": 300,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f4c4846c",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = SagemakerEndpoint(\n",
    "    endpoint_name = endpoint_name, \n",
    "    region_name = aws_region, \n",
    "    model_kwargs = parameters,\n",
    "    content_handler = content_handler\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f35dbb2",
   "metadata": {},
   "source": [
    "## PDF Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "930655e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install PyPDF2 --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7a51b21f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF2\n",
    "from io import BytesIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9c38371e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = sagemaker.Session()\n",
    "s3_bucket = sess.default_bucket()\n",
    "s3_prefix = 'docs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9b46cf35",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !aws s3 cp --recursive contents `s3://{s3_bucket}/{s3_prefix}/`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cee4363e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#s3_file_name = 'sample-blog.pdf'\n",
    "s3_file_name = '2016-3series.pdf'\n",
    "#s3_file_name = 'gen-ai-aws.pdf'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b6dd1db7",
   "metadata": {},
   "outputs": [],
   "source": [
    "s3r = boto3.resource(\"s3\")\n",
    "doc = s3r.Object(s3_bucket, s3_prefix+'/'+s3_file_name)\n",
    "        \n",
    "contents = doc.get()['Body'].read()\n",
    "reader = PyPDF2.PdfReader(BytesIO(contents))\n",
    "        \n",
    "raw_text = []\n",
    "for page in reader.pages:\n",
    "    raw_text.append(page.extract_text())\n",
    "contents = '\\n'.join(raw_text)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "81df33ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_contents = str(contents).replace(\"\\n\",\" \") \n",
    "#new_contents = str(contents[:8000]).replace(\"\\n\",\" \") "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a72207fb",
   "metadata": {},
   "source": [
    "### RecursiveCharacterTextSplitter를 이용해 chunk로 텍스트를 분리합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f85cf8a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000,chunk_overlap=0)\n",
    "texts = text_splitter.split_text(new_contents) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b1f1a665",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "444"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "49491adf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Owner's Manual for Vehicle The Ultimate Driving Machine® THE BMW 3 SERIES SEDAN. OWNER'S MANUAL. Contents A-Z Online Edition for Part no. 01 40 2 960 440 - II/15  3 Series Owner's Manual for Vehicle Thank you for choosing a BMW. The more familiar you are with your vehicle, the better control you will have on the road. We therefore strongly suggest: Read this Owner's Manual before starting off in your new BMW. Also use the Integrated Owner's Manual in your vehicle. It con‐ tains important information on vehicle operation that will help you make full use of the technical features available in your BMW. The manual also contains information designed to en‐ hance operating reliability and road safety, and to contribute to maintaining the value of your BMW. Any updates made after the editorial deadline for the printed or Integrated Owner's Manual are found in the appendix of the printed Quick Reference for the vehicle. Supplementary information can be found in the additional bro‐ chures in\n"
     ]
    }
   ],
   "source": [
    "print(texts[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25ac3422",
   "metadata": {},
   "source": [
    "(주의) page가 여러개일 경우에 refine이나 map_reduce를 쓰는데, pages수가 너무 많으면 1) 입력 Token수가 1024개를 넘을 수 있고, 2) 브라우저 시간제한(30초)내에 처리가 어려울 수 있습니다. 편의상 여기서는 3000자로 요약을 수행합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3e26e95c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.docstore.document import Document\n",
    "docs = [\n",
    "    Document(\n",
    "        page_content=t\n",
    "    ) for t in texts[:3]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "156b23ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27b874ad",
   "metadata": {},
   "source": [
    "### Summerrize type: stuff, refine, map_reduce\n",
    "\n",
    "- stuff puts all the chunks into one prompt. Thus, this would hit the maximum limit of tokens.\n",
    "- map_reduce summarizes each chunk, combines the summary, and summarizes the combined summary. If the combined summary is too large, it would raise error. (채팅에서 쓰기에 너무 많은 시간이 소요가되며, map_reduce를 쓰려면 transfomer를 설치하여야 하는데, 노트북에서는 잘 동작하지만 container에서는 write가 안되어서 사용할 수 없습니다)\n",
    "- refine summarizes the first chunk, and then summarizes the second chunk with the first summary. The same process repeats until all chunks are summarized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "306687ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install transformers --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3afb1e7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "None of PyTorch, TensorFlow >= 2.0, or Flax have been found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The Owner's Manual for the 3 Series Sedan provides essential information on vehicle operation, maintenance, and safety features. It is important to familiarize yourself with the manual to ensure optimal control and value of your BMW. The manual is available in many countries as an app and online.\n"
     ]
    }
   ],
   "source": [
    "from langchain.chains.summarize import load_summarize_chain\n",
    "chain = load_summarize_chain(llm, chain_type=\"map_reduce\")\n",
    "summary = chain.run(docs)\n",
    "print(summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "dcdb1266",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(summary.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9cf4abe8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from langchain.chains.summarize import load_summarize_chain\n",
    "chain = load_summarize_chain(llm, chain_type=\"refine\")\n",
    "summary = chain.run(docs)\n",
    "print(summary)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e22d380",
   "metadata": {},
   "source": [
    "### Prompt로 load_summarize_chain을 하는 방법\n",
    "- Prompt에 docs가 한꺼번에 들어가므로 chain_type은 stuff가 되어야 하고, LLM의 token 숫자가 보다 docs의 단어수가 많은 경우에는 실패합니다. 따라서 아래처럼 docs의 일부만 요약(Summary)할 수 있습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e306881c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.docstore.document import Document\n",
    "docs = [\n",
    "    Document(\n",
    "        page_content=t\n",
    "    ) for t in texts[:3]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6afe6c07",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.prompts import PromptTemplate\n",
    "\n",
    "prompt_template = \"\"\"Write a concise summary of the following:\n",
    "\n",
    "\n",
    "{text}\n",
    "\n",
    "\n",
    "CONCISE SUMMARY \"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "22945264",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "The Owner's Manual for the 3 Series is a comprehensive guide to the vehicle's features and operation. It contains important information on vehicle operation that will help you make full use of the technical features available in your BMW. The manual also contains information designed to enhance operating reliability and road safety, and to contribute to maintaining the value of your BMW. Any updates made after the editorial deadline for the printed or Integrated Owner's Manual are found in the appendix of the printed Quick Reference for the vehicle. Supplementary information can be found in the additional brochures in the onboard literature. We wish you a safe and enjoyable ride.\n",
      "\n",
      "BMW AG\n",
      "The Owner's Manual is available in many countries as an app. Additional information on the Internet: www.bmw.com/bmw_drivers_guide Online Edition for Part no. 01 40 2 960 440 - II/15 © 2015 Bayerische Motoren Werke Aktiengesellschaft Munich, Germany Reprinting, including excerpts, only with the written consent of Bayerische Motoren Werke Aktiengesellschaft Munich, Germany. US English II/15, 03 15 490 Printed on environmentally friendly paper, bleached without chlorine, suitable for recycling. Online Edition for Part no. 01 40 2 960 440 - II/15 Addendum ADDENDUM TO OWNER'S MANUAL We wanted to provide you with some updates  and\n"
     ]
    }
   ],
   "source": [
    "PROMPT = PromptTemplate(template=prompt_template, input_variables=[\"text\"])\n",
    "chain = load_summarize_chain(llm, chain_type=\"stuff\", prompt=PROMPT)\n",
    "summary = chain.run(docs)\n",
    "print(summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "56f95fe4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "207"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(summary.split())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}