BVR_Summarization-Gemma.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8c564fb8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Gemma's activation function should be approximate GeLU and not exact GeLU.\n",
      "Changing the activation function to `gelu_pytorch_tanh`.if you want to use the legacy `gelu`, edit the `model.config` to set `hidden_activation=gelu`   instead of `hidden_act`. See https://github.com/huggingface/transformers/pull/29402 for more details.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0cbca949270d41c8808682ff852fe126",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
    "import os\n",
    "from accelerate import PartialState\n",
    "\n",
    "\n",
    "# if torch.cuda.is_available():\n",
    "#     torch.set_default_device(\"cuda\")\n",
    "# else:\n",
    "#     torch.set_default_device(\"cpu\")\n",
    "\n",
    "os.environ['HF_TOKEN'] = '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'\n",
    "os.environ['HF_HOME'] = '/data_vault/hexai/huggingface/hub/'\n",
    "\n",
    "model_type = 'gemma-2b-it' # orca13b\n",
    "model_id = \"google/gemma-2b-it\"\n",
    "\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_8bit=True,\n",
    "    bnb_8bit_quant_type=\"nf8\",\n",
    "    bnb_8bit_compute_dtype=torch.bfloat16\n",
    ")\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id, token=os.environ['HF_TOKEN'], cache_dir=os.environ['HF_HOME'], use_fast=False)\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_id, quantization_config=bnb_config, device_map=\"auto\", token=os.environ['HF_TOKEN'], cache_dir=os.environ['HF_HOME']\n",
    ")\n",
    "\n",
    "# model = AutoModelForCausalLM.from_pretrained(f\"nlp/model/{model_type}\", device_map=\"cuda:1\", torch_dtype=torch.float16)\n",
    "# tokenizer = AutoTokenizer.from_pretrained(model_id, token=os.environ['HF_TOKEN'], cache_dir=os.environ['HF_HOME'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ad7df812",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data = \"/data_vault/hexai/Biolaysum/biolaysumm2024_data/eLife_val.jsonl\"\n",
    "elife_train = pd.read_json(path_or_buf=data, lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2edc8bb4-fee7-484e-b7ed-c013b669da5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "from langchain import HuggingFacePipeline, PromptTemplate, LLMChain\n",
    "\n",
    "text_generation_pipeline = pipeline(\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    task=\"text-generation\",\n",
    "    eos_token_id=tokenizer.eos_token_id,\n",
    "    pad_token_id=tokenizer.pad_token_id,\n",
    "    batch_size=4,\n",
    "    max_new_tokens=1000,\n",
    "    temperature = 0.3,\n",
    "    do_sample=True,\n",
    ")\n",
    "llm = HuggingFacePipeline(pipeline=text_generation_pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ab94653a-db9f-4aa0-91ae-b41a0736c881",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tiktoken\n",
    "def num_tokens_from_string(string: str, encoding_name: str) -> int:\n",
    "    \"\"\"Returns the number of tokens in a text string.\"\"\"\n",
    "    encoding = tiktoken.get_encoding(encoding_name)\n",
    "    num_tokens = len(encoding.encode(string))\n",
    "    return num_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "55b93376-d6d6-4d70-9212-3143fcc02842",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import MapReduceDocumentsChain, LLMChain, ReduceDocumentsChain, StuffDocumentsChain\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain.prompts import PromptTemplate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8c6d6597",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import JSONLoader\n",
    "\n",
    "def metadata_func(record: dict, metadata: dict) -> dict:\n",
    "\n",
    "    metadata[\"lay_summary\"] = record.get(\"lay_summary\")\n",
    "\n",
    "    return metadata\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a6b21306-f0e6-4a41-a998-2d391d61d5cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_json():\n",
    "    # Load the pdf file\n",
    "    loader = JSONLoader(\n",
    "        file_path=\"/data_vault/hexai/Biolaysum/biolaysumm2024_data/eLife_val.jsonl\",\n",
    "        jq_schema='.',\n",
    "        content_key=\"article\",\n",
    "        metadata_func=metadata_func,\n",
    "        json_lines=True\n",
    "    )\n",
    "\n",
    "    documents = loader.load()\n",
    "\n",
    "    token_count = num_tokens_from_string(str(documents), \"cl100k_base\")\n",
    "    print(f'JSON Token Count: {token_count}')\n",
    "    return documents, token_count\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9265bc25-2491-4399-b75d-a9160997c7c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JSON Token Count: 3475695\n"
     ]
    }
   ],
   "source": [
    "docs, counts = load_json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "df54aac4-d40b-4c5e-abad-dae7223aba22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38.251953125"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "19585/512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3da35c41-9d07-4993-bbc4-79e7243e6149",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=5000, chunk_overlap=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "00babcbd-3598-4ba8-afeb-6c2879536397",
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = text_splitter.create_documents([docs[40].page_content])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9e3914c5-3696-48be-b002-992e335cb7bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(splits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ccbcfb9b-1e9a-4033-9920-aaea33c9f888",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1131"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(splits[0].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a854ee70-4250-4f16-b7f7-e09e232f7f79",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import T5Tokenizer, T5EncoderModel\n",
    "import torch\n",
    "\n",
    "\n",
    "extractor = pipeline(\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    task=\"feature-extraction\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a1cf3116-9823-4c1d-ad7b-1ecb52d9d134",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_embeddings = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "dedc7453-b46c-4394-9815-0b475b2ccf52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting summarization: 2024-04-21 21:40:05.660516\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████████████████████████████████████████████████████████████████▋                    | 10/13 [03:22<00:57, 19.33s/it]--- Logging error ---\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/logging/__init__.py\", line 1160, in emit\n",
      "    msg = self.format(record)\n",
      "          ^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/logging/__init__.py\", line 999, in format\n",
      "    return fmt.format(record)\n",
      "           ^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/logging/__init__.py\", line 703, in format\n",
      "    record.message = record.getMessage()\n",
      "                     ^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/logging/__init__.py\", line 392, in getMessage\n",
      "    msg = msg % self.args\n",
      "          ~~~~^~~~~~~~~~~\n",
      "TypeError: not all arguments converted during string formatting\n",
      "Call stack:\n",
      "  File \"<frozen runpy>\", line 198, in _run_module_as_main\n",
      "  File \"<frozen runpy>\", line 88, in _run_code\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/ipykernel_launcher.py\", line 18, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/traitlets/config/application.py\", line 1075, in launch_instance\n",
      "    app.start()\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/ipykernel/kernelapp.py\", line 739, in start\n",
      "    self.io_loop.start()\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/tornado/platform/asyncio.py\", line 205, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/asyncio/base_events.py\", line 639, in run_forever\n",
      "    self._run_once()\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/asyncio/base_events.py\", line 1985, in _run_once\n",
      "    handle._run()\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/asyncio/events.py\", line 88, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 545, in dispatch_queue\n",
      "    await self.process_one()\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 534, in process_one\n",
      "    await dispatch(*args)\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 437, in dispatch_shell\n",
      "    await result\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/ipykernel/ipkernel.py\", line 362, in execute_request\n",
      "    await super().execute_request(stream, ident, parent)\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 778, in execute_request\n",
      "    reply_content = await reply_content\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/ipykernel/ipkernel.py\", line 449, in do_execute\n",
      "    res = shell.run_cell(\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/ipykernel/zmqshell.py\", line 549, in run_cell\n",
      "    return super().run_cell(*args, **kwargs)\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3075, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3130, in _run_cell\n",
      "    result = runner(coro)\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/IPython/core/async_helpers.py\", line 129, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3334, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3517, in run_ast_nodes\n",
      "    if await self.run_code(code, result, async_=asy):\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3577, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"/tmp/ipykernel_470488/441188894.py\", line 7, in <module>\n",
      "    embedd = extractor(doc.page_content)\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/transformers/pipelines/feature_extraction.py\", line 86, in __call__\n",
      "    return super().__call__(*args, **kwargs)\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/transformers/pipelines/base.py\", line 1167, in __call__\n",
      "    logger.warning_once(\n",
      "  File \"/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/transformers/utils/logging.py\", line 329, in warning_once\n",
      "    self.warning(*args, **kwargs)\n",
      "Message: 'You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset'\n",
      "Arguments: (<class 'UserWarning'>,)\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 13/13 [04:24<00:00, 20.37s/it]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from datetime import datetime\n",
    "print(\"Starting summarization:\",datetime.now()) \n",
    "\n",
    "for doc in tqdm(splits):\n",
    "    embedd = extractor(doc.page_content)\n",
    "    mean_embedd = np.array(embedd).mean(axis=1).squeeze(axis=0)\n",
    "    document_embeddings.append(mean_embedd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3b0f374a-24d8-4589-b39f-1643cbe864cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "nump_embedd = np.array(document_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3dab88a7-ac41-4eaa-9e2a-972af48d2c61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13, 256000)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nump_embedd.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "438b49fe-f656-4bfa-aa90-a822f4e8b11c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de4a2365-2ca7-4b49-8e79-fa433a6bdc05",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c3a35164-8c64-4467-b290-f21cdb865bab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming 'embeddings' is a list or array of 1536-dimensional embeddings\n",
    "\n",
    "# Choose the number of clusters, this can be adjusted based on the book's content.\n",
    "# I played around and found ~10 was the best.\n",
    "# Usually if you have 10 passages from a book you can tell what it's about\n",
    "num_clusters = 10\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "# Perform K-means clustering\n",
    "kmeans = KMeans(n_clusters=num_clusters, random_state=42).fit(nump_embedd)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "eb6566e3-f1da-4500-a109-111e39e3bda1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the closest embeddings to the centroids\n",
    "\n",
    "# Create an empty list that will hold your closest points\n",
    "closest_indices = []\n",
    "\n",
    "# Loop through the number of clusters you have\n",
    "for i in range(num_clusters):\n",
    "    \n",
    "    # Get the list of distances from that particular cluster center\n",
    "    distances = np.linalg.norm(nump_embedd - kmeans.cluster_centers_[i], axis=1)\n",
    "    \n",
    "    # Find the list position of the closest one (using argmin to find the smallest distance)\n",
    "    closest_index = np.argmin(distances)\n",
    "    \n",
    "    # Append that position to your closest indices list\n",
    "    closest_indices.append(closest_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "810f96a9-4730-42ad-b14d-993174809312",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4, 6, 8, 9, 10, 11, 12]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_indices = sorted(closest_indices)\n",
    "selected_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c87c6a11-5a1e-4848-b825-710b2eed8317",
   "metadata": {},
   "outputs": [],
   "source": [
    "map_prompt  = \"\"\"Given the text enclosed in triple backticks (```) provide a condensed summary in layterms:\n",
    "\n",
    "```{text}```\n",
    "\n",
    "SUMMARY:\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1ade3164-b4a9-4912-a62b-8313e870c0ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "map_prompt_template = PromptTemplate(template=map_prompt, input_variables=[\"text\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8d1e67c3-fcb8-4c42-b8af-6ea10955c840",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import load_summarize_chain\n",
    "map_chain = load_summarize_chain(llm=llm,\n",
    "                                 chain_type=\"stuff\",\n",
    "                                 prompt=map_prompt_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a290b20e-5ee1-4090-a323-612aa8b67d15",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_docs = [splits[doc] for doc in selected_indices]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a906529e-82cb-4212-ae1a-dd51c4347adf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def post_process_summaries(summary):\n",
    "    return summary.split(\"SUMMARY:\\n\")[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "0f23f542-c752-4491-966f-d71677f7db1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ngl18/anaconda3/envs/lora/lib/python3.12/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `run` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use invoke instead.\n",
      "  warn_deprecated(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary #0 (chunk #1) - Preview: The monkey with the long-standing primary visual cortex lesion had residual visual function, which was determined to be mediated by the LGN, pulvinar, SC, and extrastriate visual cortical networks. The results suggest that the structural and function\n",
      "\n",
      "Summary #1 (chunk #2) - Preview: The LGN is often reduced in size, due to retrograde degeneration ( Miki et al . , 2005; Bridge et al . , 2011 ) when humans with lesions of V1 have hemianopia. A similar result has also been found in the adult marmoset ( Atapour et al . , 2017 ) . Wh\n",
      "\n",
      "Summary #2 (chunk #3) - Preview: The cortical changes in monkey S were investigated by acquiring post mortem T2-weighted images. Measurement of cortical thickness in V1 in both monkeys indicated that monkey S has substantially thinner cortex around the lesion. By contrast, extrastri\n",
      "\n",
      "Summary #3 (chunk #4) - Preview: The timeseries analysis of the BOLD signal in the LGN and the pulvinar of monkey S revealed that there was no spatially extensive region of activation in area V5/MT to either checkerboard or moving dots stimuli. However, there were a number of region\n",
      "\n",
      "Summary #4 (chunk #6) - Preview: The LGN in monkey S appears to be intact, despite a large cortical lesion. This supports the requirement for an intact LGN to support remaining visual function after V1 lesions. The microstructure of the tracts between the LGN and hMT+ appears to be \n",
      "\n",
      "Summary #5 (chunk #8) - Preview: The structural connectivity between the subcortical regions and area V5/MT was weak , but the microstructure was intact. Thus, unlike adult-acquired lesions, there appeared to be a maintenance of structural integrity of the visual system when V1 is d\n",
      "\n",
      "Summary #6 (chunk #9) - Preview: The seven monkeys undergoing functional MRI scans were sedated with a mixture of ketamine ( 7.5 mg/kg ), xylazine ( 0.125 mg/kg ) and medetomidine ( 0.1 mg/kg ). They were intubated, and the MRI scans were performed under general anesthesia. The stru\n",
      "\n",
      "Summary #7 (chunk #10) - Preview: The study was conducted to investigate the effects of isoflurane on myelin-weighted images in monkeys. The results showed that lower average levels of isoflurane were associated with significant visual stimulation responses, but the pattern was varia\n",
      "\n",
      "Summary #8 (chunk #11) - Preview: Monkey S and four control animals were presented with the checkerboard and moving dots stimuli. Functional MRI data were acquired from the checkerboard and motion stimuli runs. Pre-processing and statistical analysis were performed using tools from t\n",
      "\n",
      "Summary #9 (chunk #12) - Preview: The study used diffusion tractography to explore the brain of monkeys and compared the results with tracers. The results showed that a threshold of 10% most reliably reflect the anatomy when compared with tracers.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make an empty list to hold your summaries\n",
    "summary_list = []\n",
    "\n",
    "# Loop through a range of the lenght of your selected docs\n",
    "for i, doc in enumerate(selected_docs):\n",
    "    \n",
    "    # Go get a summary of the chunk\n",
    "    chunk_summary = map_chain.run([doc])\n",
    "    \n",
    "    # Append that summary to your list\n",
    "    summary_list.append(chunk_summary)\n",
    "    \n",
    "    print (f\"Summary #{i} (chunk #{selected_indices[i]}) - Preview: {post_process_summaries(chunk_summary)[:250]}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "df8423ae-049d-4421-82f5-e016c319154d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (1070 > 1024). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your total summary has 1070 tokens\n"
     ]
    }
   ],
   "source": [
    "from langchain.schema import Document\n",
    "\n",
    "summaries = \"\\n\".join([post_process_summaries(summ) for summ in summary_list])\n",
    "\n",
    "# Convert it back to a document\n",
    "summaries = Document(page_content=summaries)\n",
    "\n",
    "print (f\"Your total summary has {llm.get_num_tokens(summaries.page_content)} tokens\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "302f7238-fae5-478f-ba04-e7d1b74ceb41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='The monkey with the long-standing primary visual cortex lesion had residual visual function, which was determined to be mediated by the LGN, pulvinar, SC, and extrastriate visual cortical networks. The results suggest that the structural and functional networks underlying residual visual function are largely intact in this monkey.\\nThe LGN is often reduced in size, due to retrograde degeneration ( Miki et al . , 2005; Bridge et al . , 2011 ) when humans with lesions of V1 have hemianopia. A similar result has also been found in the adult marmoset ( Atapour et al . , 2017 ) . When we investigated the structure of the LGN in humans with lesions of V1, we found that the LGN was completely absent in one case, and that the LGN was significantly reduced in size in another case. These results suggest that the LGN is a critical structure for visual processing in humans with lesions of V1.\\nThe cortical changes in monkey S were investigated by acquiring post mortem T2-weighted images. Measurement of cortical thickness in V1 in both monkeys indicated that monkey S has substantially thinner cortex around the lesion. By contrast, extrastriate visual area V5/MT appeared to have a thickness of about 2 mm , which is comparable to the control monkey. These results suggest that information from the retina can reach the brain.\\nThe timeseries analysis of the BOLD signal in the LGN and the pulvinar of monkey S revealed that there was no spatially extensive region of activation in area V5/MT to either checkerboard or moving dots stimuli. However, there were a number of regions within the sulcus showing BOLD activation to the moving dot stimulus, including in area MST on the anterior bank and area FST at the bottom of the sulcus. These results suggest that the activation in area V5/MT in monkey S may be related to the visual motion complex hMT+.\\nThe LGN in monkey S appears to be intact, despite a large cortical lesion. This supports the requirement for an intact LGN to support remaining visual function after V1 lesions. The microstructure of the tracts between the LGN and hMT+\\xa0appears to be close to control values, suggesting that the reduction in size could reflect a reduction in feedback connections into the LGN rather than a change in feedforward connectivity into cortex.\\nThe structural connectivity between the subcortical regions and area V5/MT was weak , but the microstructure was intact. Thus, unlike adult-acquired lesions, there appeared to be a maintenance of structural integrity of the visual system when V1 is damaged neonatally. This may explain the increased residual function both , in monkey S and in children with early damage to the visual cortex.\\nThe seven monkeys undergoing functional MRI scans were sedated with a mixture of ketamine ( 7.5 mg/kg ), xylazine ( 0.125 mg/kg ) and medetomidine ( 0.1 mg/kg ). They were intubated, and the MRI scans were performed under general anesthesia. The structural myelin data of the four control monkeys have previously been reported elsewhere ( Large et al . , 2016 ) , and so have the DWI data from four of the six controls ( Rafal et al . , 2015 ) . The monkeys were social housed together in same sex groups of between 2 and 6 animals and housing and husbandry were in compliance with the ARRIVE guidelines of the European Directive ( 2010/63/EU ) for the care and use of laboratory animals. All animal procedures were carried out in accordance with Home Office ( UK ) Regulations and European Union guidelines ( EU directive 86/609/EEC; EU Directive 2010/63/EU ).\\nThe study was conducted to investigate the effects of isoflurane on myelin-weighted images in monkeys. The results showed that lower average levels of isoflurane were associated with significant visual stimulation responses, but the pattern was variable.\\nMonkey S and four control animals were presented with the checkerboard and moving dots stimuli. Functional MRI data were acquired from the checkerboard and motion stimuli runs. Pre-processing and statistical analysis were performed using tools from the FSL toolbox. Non-brain tissue was excluded from analysis using BET, motion correction was performed using MCFLIRT, and spatial smoothing was applied using a full-width half-height Gaussian kernel of 3 mm and high pass temporal filtering. Functional images were registered to high-resolution structural scans using FLIRT. A general linear model was used to contrast the presentation of the checkerboard or moving dots against the mid-grey or stationary dot background and data from the two stimulus runs were combined using a fixed effects analysis. Probabilistic tractography was performed using ProbtrackX2 from the FSL FDT toolbox. We traced two unilateral pathways in each hemisphere: pulvinar to V5/MT and LGN to V5/MT.\\nThe study used diffusion tractography to explore the brain of monkeys and compared the results with tracers. The results showed that a threshold of 10% most reliably reflect the anatomy when compared with tracers.')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ff3c21dd-09e5-424f-86b2-13fb649a2308",
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_prompt = \"\"\"\n",
    "Given the following text write a condensed and precise paragraph summarizing the relevant points. Remove irrelevant information. \n",
    "\n",
    "```{text}```\n",
    "\n",
    "SUMMARY:\n",
    "\"\"\"\n",
    "combine_prompt_template = PromptTemplate(template=combine_prompt, input_variables=[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "f612c0a4-5eef-43a5-8dda-02a169e2035a",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm2 = HuggingFacePipeline(pipeline=text_generation_pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e4ae9bbb-da44-47fb-8e44-e7a94ec74678",
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_chain = load_summarize_chain(llm=llm2,\n",
    "                             chain_type=\"stuff\",\n",
    "                             prompt=combine_prompt_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "350ef490-4a63-4aff-bd7d-1863d1ba7026",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = reduce_chain.run([summaries])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "f1291879-c030-4964-bcb8-87c959974d1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = post_process_summaries(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "9b327374-e7fa-4fc9-87c7-bd76100dc6f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The monkey with the long-standing primary visual cortex lesion has residual visual function, which is mediated by the LGN, pulvinar, SC, and extrastriate visual cortical networks. The results suggest that the structural and functional networks underlying residual visual function are largely intact in this monkey.\n"
     ]
    }
   ],
   "source": [
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "7364e8d7-c525-48f5-a082-0ff5bd20a6f7",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax. Perhaps you forgot a comma? (589089834.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  Cell \u001b[0;32mIn[41], line 1\u001b[0;36m\u001b[0m\n\u001b[0;31m    print(\"Ending summarization:\" datetime.now())\u001b[0m\n\u001b[0m          ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax. Perhaps you forgot a comma?\n"
     ]
    }
   ],
   "source": [
    "print(\"Ending summarization:\" datetime.now()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d7312f4-dea4-4df6-9b4a-63285f65888a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}