Summarize theme groups.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "6db61bae-9e89-46a9-a399-9e9560a5ee54",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.prompts.chat import (\n",
    "    ChatPromptTemplate,\n",
    "    HumanMessagePromptTemplate,\n",
    "    SystemMessagePromptTemplate,\n",
    ")\n",
    "from langchain.schema import HumanMessage, SystemMessage\n",
    "from langchain_openai import ChatOpenAI\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "from graphdatascience import GraphDataScience\n",
    "from getpass import getpass\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0710100-48c2-47e5-ae69-28e1d3e0f6af",
   "metadata": {},
   "source": [
    "# Set up connection information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c09556f0-e034-43cc-814c-3602ec4a3657",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " ········\n"
     ]
    }
   ],
   "source": [
    "openai_api_key = getpass()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "fe6e4864-2c4a-48fb-b14a-618bb913bfe4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " ········\n"
     ]
    }
   ],
   "source": [
    "neo4j_password = getpass()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "d31b1d39-5dac-4b3e-80b5-dac623f151dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to write data to connection ResolvedIPv4Address(('34.28.32.244', 7687)) (ResolvedIPv4Address(('34.28.32.244', 7687)))\n",
      "Failed to write data to connection IPv4Address(('2fe3bf28.databases.neo4j.io', 7687)) (ResolvedIPv4Address(('34.28.32.244', 7687)))\n"
     ]
    }
   ],
   "source": [
    "neo4j_uri = \"neo4j+s://2fe3bf28.databases.neo4j.io\"\n",
    "neo4j_user = \"neo4j\"\n",
    "gds = GraphDataScience(neo4j_uri, auth=(neo4j_user, neo4j_password))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c11b5e97-99ff-4e7e-84bc-c9b5ef6143d5",
   "metadata": {},
   "source": [
    "# Get ThemeGroup nodes that need a summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "5272cd56-4cce-4ac5-a13d-4f0a6afb4285",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_info = gds.run_cypher(\"\"\"\n",
    "MATCH (g:ThemeGroup)<-[:IN_GROUP]-()<-[:HAS_THEME]-(m)\n",
    "WHERE g.summary IS null\n",
    "WITH g, collect(distinct m{.title, .overview}) AS movieData\n",
    "RETURN g.id AS id, g.descriptions as themes, apoc.coll.randomItems(movieData, $resultLimit) AS movie_data\"\"\",\n",
    "              {\"resultLimit\": 20})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "f32334ae-c476-445c-bb29-6c43113c2166",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>themes</th>\n",
       "      <th>movie_data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4166</td>\n",
       "      <td>[Annihilator, Exterminate, annihilation, exter...</td>\n",
       "      <td>[{'title': 'KUEBIPUTO', 'overview': 'A punk sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4176</td>\n",
       "      <td>[bar, bars, box, boxes, boxing]</td>\n",
       "      <td>[{'title': 'Jamel Herring vs. Nick Molina', 'o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4179</td>\n",
       "      <td>[Captures, captivating, captive, captivity, ca...</td>\n",
       "      <td>[{'title': 'Red Fox', 'overview': 'In a baron ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4180</td>\n",
       "      <td>[Cheerleader, Cheerleading, cheer, cheerful]</td>\n",
       "      <td>[{'title': 'Holy Rosita', 'overview': 'Rosita ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>269</td>\n",
       "      <td>[Christmas celebration, Christmas celebrations]</td>\n",
       "      <td>[{'title': 'Aunt Virginia', 'overview': 'Aunt ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2910</th>\n",
       "      <td>7085</td>\n",
       "      <td>[cruising]</td>\n",
       "      <td>[{'title': 'How cruise ships became a catastro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2911</th>\n",
       "      <td>7086</td>\n",
       "      <td>[countess]</td>\n",
       "      <td>[{'title': 'Le Nozze Di Figaro - Opera Ballet ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2912</th>\n",
       "      <td>7087</td>\n",
       "      <td>[Ariana Grande]</td>\n",
       "      <td>[{'title': 'yes, and?', 'overview': 'music vid...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2913</th>\n",
       "      <td>7088</td>\n",
       "      <td>[wheat]</td>\n",
       "      <td>[{'title': 'The Songs of Red Tresses: Woman, l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2914</th>\n",
       "      <td>7089</td>\n",
       "      <td>[hyperpop]</td>\n",
       "      <td>[{'title': 'Midnight Rising', 'overview': 'An ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2915 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                             themes  \\\n",
       "0     4166  [Annihilator, Exterminate, annihilation, exter...   \n",
       "1     4176                    [bar, bars, box, boxes, boxing]   \n",
       "2     4179  [Captures, captivating, captive, captivity, ca...   \n",
       "3     4180       [Cheerleader, Cheerleading, cheer, cheerful]   \n",
       "4      269    [Christmas celebration, Christmas celebrations]   \n",
       "...    ...                                                ...   \n",
       "2910  7085                                         [cruising]   \n",
       "2911  7086                                         [countess]   \n",
       "2912  7087                                    [Ariana Grande]   \n",
       "2913  7088                                            [wheat]   \n",
       "2914  7089                                         [hyperpop]   \n",
       "\n",
       "                                             movie_data  \n",
       "0     [{'title': 'KUEBIPUTO', 'overview': 'A punk sc...  \n",
       "1     [{'title': 'Jamel Herring vs. Nick Molina', 'o...  \n",
       "2     [{'title': 'Red Fox', 'overview': 'In a baron ...  \n",
       "3     [{'title': 'Holy Rosita', 'overview': 'Rosita ...  \n",
       "4     [{'title': 'Aunt Virginia', 'overview': 'Aunt ...  \n",
       "...                                                 ...  \n",
       "2910  [{'title': 'How cruise ships became a catastro...  \n",
       "2911  [{'title': 'Le Nozze Di Figaro - Opera Ballet ...  \n",
       "2912  [{'title': 'yes, and?', 'overview': 'music vid...  \n",
       "2913  [{'title': 'The Songs of Red Tresses: Woman, l...  \n",
       "2914  [{'title': 'Midnight Rising', 'overview': 'An ...  \n",
       "\n",
       "[2915 rows x 3 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_info"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a27b22e6-1a3d-4427-a348-7a450ace7f3c",
   "metadata": {},
   "source": [
    "# Set up smmary chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "b9686097-5a19-4158-adfc-8404eba78fa7",
   "metadata": {},
   "outputs": [],
   "source": [
    "chat = ChatOpenAI(temperature=0, model=\"gpt-3.5-turbo\", openai_api_key=openai_api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "1a9bd99c-e165-421b-b684-076026bf746b",
   "metadata": {},
   "outputs": [],
   "source": [
    "system_message = SystemMessage(\n",
    "    content=\"\"\"You are a movie expert. \n",
    "    You will be given a list of themes and a list of movies that contain those themes.\n",
    "    Write two or three sentences that describe the what the themes have in common.\n",
    "    Use the example movie information to guide your description of the themes but do not include the titles of any movies in your sentences.\"\"\")\n",
    "final_prompt = ChatPromptTemplate.from_messages(\n",
    "    [system_message,\n",
    "     (\"human\", \"\"\"themes: {themes}\n",
    "     movie information: {movie_data}\"\"\")])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39838e2a-5532-4b83-b17b-0d41294add97",
   "metadata": {},
   "source": [
    "## Test the chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "e8214c52-1acd-4ad2-b302-f8821f316f83",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Guest', 'Visitors', 'guests', 'visitor']"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_info.iloc[10,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "bc1a62e1-0433-45fc-b394-46d790b817d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "chain = final_prompt | chat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "8167f4f4-fefd-4b98-9322-f98e15061aa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = chain.invoke({\"movie_data\": movie_info.iloc[10,2], \"themes\": movie_info.iloc[10,1]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "eb727997-ef90-4570-b25f-5cb9b6e56b35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"The themes of 'Guest' and 'Visitor' both involve individuals who are not permanent residents or members of a particular place or group. These themes often introduce new characters or elements into a story, creating opportunities for unexpected interactions, revelations, or conflicts. Whether it's a special guest at a ministerial service or an unexpected visitor in someone's dreams, these themes can add intrigue and depth to a narrative.\""
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "a938845d-8c75-4b25-b942-96e110315a9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = OpenAIEmbeddings(model=\"text-embedding-3-small\", openai_api_key=openai_api_key)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a2322f1-ebac-4f70-af08-777b70e6248d",
   "metadata": {},
   "source": [
    "# Send summaries with their embeddings to Neo4j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7102fbf0-ba01-4267-b1d9-368ba337bbf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "gds.run_cypher(\"\"\"CREATE VECTOR INDEX theme_group_summary_vectors IF NOT EXISTS \n",
    "                  FOR (g:ThemeGroup)\n",
    "                  ON (g.summaryEmbedding)\n",
    "                  OPTIONS {indexConfig: \n",
    "                      {`vector.dimensions`: 1536,\n",
    "                       `vector.similarity_function`: 'cosine'\n",
    "                       }}\n",
    "                       \"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "d6b5b446-dcdc-47e7-a506-7bc0fe4b284b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_theme_group_description(start_index, end_index):\n",
    "    movie_slice = movie_info.iloc[start_index:end_index].copy()\n",
    "    movie_slice['response'] = chain.batch(movie_slice[['movie_data', 'themes']].to_dict(\"records\"))\n",
    "    movie_slice['summary'] = movie_slice['response'].map(lambda x: x.content)\n",
    "    movie_slice['embedding'] = embeddings.embed_documents(movie_slice['summary'].tolist())\n",
    "    gds.run_cypher(\"\"\"\n",
    "        UNWIND $data AS row\n",
    "        MATCH (g:ThemeGroup {id:row['id']})\n",
    "        SET g.summary = row['summary']\n",
    "        WITH g, row\n",
    "        CALL db.create.setNodeVectorProperty(g, 'summaryEmbedding', row['embedding']) \"\"\", \n",
    "                   {\"data\": movie_slice[[\"id\", \"summary\", \"embedding\"]].to_dict(\"records\")})\n",
    "    return movie_slice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "5134df63-40f1-465f-867a-18570cc9e231",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 150\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 350\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 550\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 650\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 850\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 900\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1050\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1150\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1350\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1550\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1650\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1850\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1900\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 1950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2050\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2150\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2350\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2550\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2650\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2850\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 2900\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    }
   ],
   "source": [
    "for i in range(0, int(movie_info.shape[0]/10) + 1):\n",
    "    get_theme_group_description(i*10,(i+1)*10)\n",
    "    time.sleep(3)\n",
    "    if (i + 1) % 5 == 0:\n",
    "        print(f\"Finished row {(i+1)*10}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd7e0702-a070-4c67-8350-66a9b77b889e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chatbot",
   "language": "python",
   "name": "chatbot"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}