Clean up themes and get embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "793bace0-0b27-4a1a-9054-8b2d35e954d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/nathansmith/anaconda3/envs/chatbot/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from langchain_openai import OpenAIEmbeddings\n",
    "from graphdatascience import GraphDataScience\n",
    "from getpass import getpass\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efd1d707-83c2-4227-8615-432f74c350bd",
   "metadata": {},
   "source": [
    "# Set up connection information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d119f881-3c7f-450a-ad53-b40363dd26b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " ········\n"
     ]
    }
   ],
   "source": [
    "openai_api_key = getpass()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8c2e5ff4-20fb-4c9b-92ab-7e778f3db6d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " ········\n"
     ]
    }
   ],
   "source": [
    "neo4j_password = getpass()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8e68f7ef-9ccc-4ccf-829a-1bd33654dfba",
   "metadata": {},
   "outputs": [],
   "source": [
    "neo4j_uri = \"neo4j+s://2fe3bf28.databases.neo4j.io\"\n",
    "neo4j_user = \"neo4j\"\n",
    "gds = GraphDataScience(neo4j_uri, auth=(neo4j_user, neo4j_password))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b897d10-f75d-450f-a0f5-0916a98b4243",
   "metadata": {},
   "source": [
    "# Clean up themes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15e19138-642b-4fd5-b29f-a5861e8a8f01",
   "metadata": {},
   "source": [
    "Remove themes without English characters. Films with non-English titles sometimes caused the LLM to give themes in the language of the title even if the overview was in English."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "eaa631e9-6f32-4a78-bb83-ef9783c0e89f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme) WHERE t.description =~ '^[^a-zA-Z0-9]*$'\n",
    "DETACH DELETE t\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6086349-c2a3-4e17-b176-2afc8a2381a2",
   "metadata": {},
   "source": [
    "If the content is too explicit for the LLM to work with, drop it from our dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "4dedfeb0-2007-43c4-9f9e-0ba2ba554450",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme)<-[:HAS_THEME]-(m)\n",
    "WHERE t.description STARTS WITH \"I will not\"\n",
    "OR t.description STARTS WITH \"I am sorry\"\n",
    "OR t.description STARTS WITH \"I apologize\"\n",
    "OR t.description STARTS WITH \"I will not generate a summary\"\n",
    "OR t.description STARTS WITH \"I cannot provide\"\n",
    "OR t.description STARTS WITH \"I will have to politely decline\"\n",
    "DETACH DELETE m\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "1bb84192-9800-4bec-beaa-62c07e20562a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme) WHERE NOT EXISTS {()-[:HAS_THEME]->(t)} DETACH DELETE t\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf15f188-d36c-48ff-a23e-e6420dba3012",
   "metadata": {},
   "source": [
    "Sometimes LLM gave us a heading followed by a colon and a list. Drop the heading and just keep the list element.\n",
    "\n",
    "If the first list item already exists as a separate node, merge this node with the existing node."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b47dd744-d743-4bbb-a403-3c2b2846aba5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>newDescription</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Alone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Awakening</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Cinema</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Coming of age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Doctor Who</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Family bonds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Friendship</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Grandmother</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Hero</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Honk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Isolation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Music</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Naval warfare</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Peace</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Prostitution</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Real estate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Realms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Relationships</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Romeo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Shame</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Tree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Volleyball</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   newDescription\n",
       "0           Alone\n",
       "1       Awakening\n",
       "2          Cinema\n",
       "3   Coming of age\n",
       "4      Doctor Who\n",
       "5    Family bonds\n",
       "6      Friendship\n",
       "7     Grandmother\n",
       "8            Hero\n",
       "9            Honk\n",
       "10      Isolation\n",
       "11          Music\n",
       "12  Naval warfare\n",
       "13          Peace\n",
       "14   Prostitution\n",
       "15    Real estate\n",
       "16         Realms\n",
       "17  Relationships\n",
       "18          Romeo\n",
       "19          Shame\n",
       "20           Tree\n",
       "21     Volleyball"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme) WHERE t.description contains \":\\n\" \n",
    "MATCH (t2:Theme)\n",
    "WHERE t2.description = trim(split(t.description, \":\\n\")[1])\n",
    "WITH [t2, t] AS nodeList\n",
    "CALL apoc.refactor.mergeNodes(nodeList, {properties:\"discard\"}) YIELD node\n",
    "RETURN node.description AS newDescription\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5566b66-1ab3-4c79-9a0d-1baaf3038005",
   "metadata": {},
   "source": [
    "If the same item exists with different headings, merge those nodes together."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "51267d6c-6c94-4f49-9e2b-e00d82c32de5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>newDescription</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Child exploitation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Egotrip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Children's festival</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cosmic powers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Inventions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>Chūshingura</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>Fallen Angel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>Endaneleyay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>Looking out</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>Share house</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         newDescription\n",
       "0    Child exploitation\n",
       "1               Egotrip\n",
       "2   Children's festival\n",
       "3         Cosmic powers\n",
       "4            Inventions\n",
       "..                  ...\n",
       "63          Chūshingura\n",
       "64         Fallen Angel\n",
       "65          Endaneleyay\n",
       "66          Looking out\n",
       "67          Share house\n",
       "\n",
       "[68 rows x 1 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme) WHERE t.description contains \":\\n\" \n",
    "WITH trim(split(t.description, \":\\n\")[1]) AS newDescription, collect(t) AS nodeList\n",
    "CALL apoc.refactor.mergeNodes(nodeList, {properties:\"discard\"}) YIELD node\n",
    "SET node.description = trim(split(node.description, \":\\n\")[1])\n",
    "RETURN node.description AS newDescription\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "3abc457a-be6c-4de6-b517-0eadd504e3c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count(*)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   count(*)\n",
       "0         5"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme)<-[:HAS_THEME]-(m) WHERE t.description =~ \".*public figure.*\\..*\\.$\" DETACH DELETE m RETURN count(*)\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91d7fbbe-f490-43da-be68-ad97f72275c8",
   "metadata": {},
   "source": [
    "Sometimes the LLM delimited the themes with carriage return instead of commas."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "ee859394-eee6-49a1-a5f4-333bbea5845a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count(*)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   count(*)\n",
       "0        91"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "    MATCH (t:Theme) WHERE t.description contains \"\\n\" \n",
    "    WITH t, split(t.description, \"\\n\") AS split\n",
    "    UNWIND split AS desc\n",
    "    WITH t, desc\n",
    "    WHERE desc <> \"\"\n",
    "    AND size(desc) <= 50\n",
    "    WITH t, CASE WHEN desc =~ '\\d\\..*' THEN split(desc, \". \")[1] \n",
    "    WHEN desc ENDS WITH \",\" THEN substring(desc, 0, size(desc)-1) ELSE desc END as cleanDesc\n",
    "    MERGE (t2:Theme {description: cleanDesc})\n",
    "    WITH t, t2\n",
    "    MATCH (m:Movie)-[:HAS_THEME]->(t)\n",
    "    MERGE (m)-[:HAS_THEME]->(t2)\n",
    "    DETACH DELETE t\n",
    "    RETURN count(*)\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81e8fba2-f96e-4289-af11-196ff12c4529",
   "metadata": {},
   "source": [
    "Drop trailing periods unless there is more than one period in the description like \"A.I.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "4d125791-654a-4627-8365-e2e62c830823",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>newDescription</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [newDescription]\n",
       "Index: []"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme) WHERE NOT t.description =~ \".*.*\\..*\\.$\" AND t.description ENDS WITH '.' \n",
    "MATCH (t2:Theme)\n",
    "WHERE trim(t2.description) = substring(t.description, 0, size(t.description)-1)\n",
    "WITH [t2, t] AS nodeList\n",
    "CALL apoc.refactor.mergeNodes(nodeList, {properties:\"discard\"}) YIELD node\n",
    "RETURN node.description AS newDescription\"\"\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "3ae26ea7-b6ba-4ff0-b6ce-7bc4f4cab742",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t.description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [t.description]\n",
       "Index: []"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme) WHERE NOT t.description =~ \".*.*\\..*\\.$\" AND t.description ENDS WITH '.' \n",
    "SET t.description = substring(t.description, 0, size(t.description)-1)\n",
    "RETURN t.description\"\"\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "e77c7453-37ad-4a94-b13a-e239fe5bbc6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6eb8f62-b9bd-41c7-856a-42eeb1d0ac3a",
   "metadata": {},
   "source": [
    "We asked the LLM for memorable themes, settings, and public figures. Sometimes those terms came through in the output. Drop those themes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "75205340-20ad-408e-847c-ec4c5a57df18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t.description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [t.description]\n",
       "Index: []"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme) WHERE toLower(t.description) CONTAINS \"memorable themes\" \n",
    "OR toLower(t.description) CONTAINS \"settings\"\n",
    "OR toLower(t.description) CONTAINS \"public figure\"\n",
    "RETURN t.description ORDER BY t.description\"\"\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "716bfa6a-f1c2-4100-aab6-aeacd25f197b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme) WHERE t.description = \"Elon (public figure)\"\n",
    "SET t.description = \"Elon\" \"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "b60aba62-429b-4cdd-9b83-25af8af05615",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme) WHERE toLower(t.description) CONTAINS \"memorable themes\" \n",
    "OR toLower(t.description) CONTAINS \"settings\"\n",
    "OR toLower(t.description) CONTAINS \"public figure\"\n",
    "DETACH DELETE t\"\"\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "af9ca5a1-811f-4f57-9e4b-fb6ae125d0a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Based on the provided title and overview',\n",
       " 'Based on the title \"Another Alone\" and the overview provided',\n",
       " 'Based on the title and overview provided',\n",
       " \"but the overview is quite vague and doesn't provide enough context to extract more relevant phrases\"]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme) WHERE toLower(t.description) CONTAINS \"overview\" RETURN t.description AS description\"\"\")['description'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "e15f4579-079c-4f2d-bc2c-96ed5554b8e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme) WHERE toLower(t.description) CONTAINS \"overview\" DETACH DELETE t\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "5e963411-5422-4b6a-a95a-a024ad250899",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Alternate titles',\n",
       " 'Alétheia (from the title)',\n",
       " 'Based on the title \"Zakaria: a Hero in Memory\"',\n",
       " 'Based solely on the title \"ChayaBrikkho\"',\n",
       " 'Based solely on the title \"Good Looking Out\"',\n",
       " 'British title',\n",
       " 'Championship title',\n",
       " 'Entitled behavior',\n",
       " 'Flower (from title)',\n",
       " 'King title',\n",
       " 'Light heavyweight title',\n",
       " 'Lightweight title',\n",
       " 'Poetic intertitles',\n",
       " 'Reflective intertitles',\n",
       " 'Speculative intertitles',\n",
       " 'Subtitles',\n",
       " 'Tag Team Titles',\n",
       " 'Title',\n",
       " 'Title IX',\n",
       " 'Title contention',\n",
       " 'Title defense',\n",
       " 'Title eliminator',\n",
       " 'Title fight',\n",
       " 'Title match',\n",
       " 'Titles',\n",
       " 'Ultraviolent Title',\n",
       " 'Undisputed titles',\n",
       " 'World title',\n",
       " 'World title fight',\n",
       " 'World titles',\n",
       " 'epic title',\n",
       " 'title challenger',\n",
       " 'untitled']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme) WHERE toLower(t.description) CONTAINS \"title\" RETURN t.description AS description\"\"\")['description'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "b32cb33c-af15-48ee-b383-1ab1a15a885e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>deletedCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   deletedCount\n",
       "0             3"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme) \n",
    "WHERE t.description in ['Based on the title \"Zakaria: a Hero in Memory\"',\n",
    " 'Based solely on the title \"ChayaBrikkho\"',\n",
    " 'Based solely on the title \"Good Looking Out\"']\n",
    " DETACH DELETE t RETURN COUNT(*) AS deletedCount\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "3525544f-8045-4701-acea-175c9d7198a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['African themes',\n",
       " 'Christian themes',\n",
       " 'Feminist themes',\n",
       " 'Haunted theme park',\n",
       " 'LGBTQ themes',\n",
       " 'LGBTQ+ themes',\n",
       " 'Mature themes',\n",
       " 'Religious themes',\n",
       " 'Satya themes',\n",
       " 'Theme park',\n",
       " 'adult themes',\n",
       " 'dark themes',\n",
       " 'environmental themes',\n",
       " 'erotic themes',\n",
       " \"it's difficult to infer other relevant themes\",\n",
       " 'philosophical themes',\n",
       " 'social themes',\n",
       " 'themes']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme) WHERE toLower(t.description) CONTAINS \"theme\" RETURN t.description AS description\"\"\")['description'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "09e91988-dc9b-4707-801d-e8238ef7d481",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>deletedCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   deletedCount\n",
       "0             2"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme) \n",
    "WHERE t.description in [\"it's difficult to infer other relevant themes\",\n",
    " \"themes\"]\n",
    " DETACH DELETE t RETURN COUNT(*) AS deletedCount\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "d108439d-fe73-48ba-8c6c-f328302f43ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Irrelevant groups',\n",
       " 'but more context would be needed to provide additional relevant phrases']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme) WHERE toLower(t.description) CONTAINS \"relevant\" RETURN t.description AS description\"\"\")['description'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "2f604842-b305-4cd1-bb1e-b8f81bf512a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>deletedCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   deletedCount\n",
       "0             1"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme) \n",
    "WHERE t.description in ['but more context would be needed to provide additional relevant phrases']\n",
    " DETACH DELETE t RETURN COUNT(*) AS deletedCount\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "45570c1a-7f51-4f3f-8225-30c1974469d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Based on the limited information provided',\n",
       " 'Based on the very limited information provided']"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme) WHERE toLower(t.description) STARTS WITH \"based\" RETURN t.description AS description\"\"\")['description'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "c2b4eeab-4a8c-430d-8f4e-506f7f5ec78a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>deletedCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   deletedCount\n",
       "0             2"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"MATCH (t:Theme) WHERE toLower(t.description) STARTS WITH \"based\"\n",
    " DETACH DELETE t RETURN COUNT(*) AS deletedCount\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ea2cf7c-1896-469f-9aac-60351e01f971",
   "metadata": {},
   "source": [
    "# Generate embeddings\n",
    "Embed the themes for clustering and the movie text for testing RAG."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "1c4a11be-74f0-48ec-824e-128e721fa3a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = OpenAIEmbeddings(model=\"text-embedding-3-small\", openai_api_key=openai_api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "ebeb980a-a197-4396-8c37-3d27853e995b",
   "metadata": {},
   "outputs": [],
   "source": [
    "themes = gds.run_cypher(\"\"\"\n",
    "                        MATCH (t:Theme)\n",
    "                        WHERE t.embedding IS NULL\n",
    "                        RETURN t.description AS description\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "0962bdc0-4fca-4bcb-926c-aeda0d73c35a",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_text = gds.run_cypher(\"\"\"\n",
    "MATCH (m:Movie)\n",
    "RETURN m.tmdbId AS id, m.title + \". \" + m.overview AS movieText\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "09afe88c-48ff-4bcf-8923-8c37d82d01a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Acid barrel murderer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Altering time</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Combat swimmers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Corporate world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Detective inspector</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Edinburgh 2021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Enduring wonder</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Female pioneer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Financial risk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>First loves</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Friendly gesture</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Global-local connections</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Grassroots advocates</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Illustrations</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Lake Constance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Milan streets</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Mystical cafe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Parent conflict</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Revisionist Bicycle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Rituales de registro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Sputnik</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Travel chronicles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Urban awareness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Useless things</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>doctor's dilemma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>financial struggles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>vocations</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 description\n",
       "0       Acid barrel murderer\n",
       "1              Altering time\n",
       "2            Combat swimmers\n",
       "3            Corporate world\n",
       "4        Detective inspector\n",
       "5             Edinburgh 2021\n",
       "6            Enduring wonder\n",
       "7             Female pioneer\n",
       "8             Financial risk\n",
       "9                First loves\n",
       "10          Friendly gesture\n",
       "11  Global-local connections\n",
       "12      Grassroots advocates\n",
       "13             Illustrations\n",
       "14            Lake Constance\n",
       "15             Milan streets\n",
       "16             Mystical cafe\n",
       "17           Parent conflict\n",
       "18       Revisionist Bicycle\n",
       "19      Rituales de registro\n",
       "20                   Sputnik\n",
       "21         Travel chronicles\n",
       "22           Urban awareness\n",
       "23            Useless things\n",
       "24          doctor's dilemma\n",
       "25     financial struggles  \n",
       "26                 vocations"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "themes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "5c800cbc-1b7e-456a-ba95-b1e0ee24adc0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>movieText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>27296</td>\n",
       "      <td>The Gingerdead Man. An evil yet adorable Gingerbread man comes to life with the soul of a convicted killer, and this real life cookie monster wreaks havoc on the girl who sent the killer to the electric chair.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65456</td>\n",
       "      <td>The Atrocity Exhibition. A doctor in a mental research institution is driven insane by the spectacle of the horrors of the twentieth century.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>75594</td>\n",
       "      <td>Blind Turn. Samantha Holt had the perfect life with a handsome fiancée and a future that couldn’t look brighter. But an unexpected turn on a dark lonely road puts her on a collision course with Bruce Miller, a devoted family man who descends into madness when he loses everything he ever cared about. One year later, Bruce decides to give Samantha the punishment he believes she escaped. Samantha has struggled with her own demons since that fateful night, and Bruce will take her struggles to the edge when he implements a demented plan to teach her a twisted lesson that may cost her the ultimate price.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>187716</td>\n",
       "      <td>Universal Groove. Filmed in 1999, Universal Groove explores 90s rave culture through eight diverse characters seeking escape and self-discovery amidst drugs and dance, offering a nostalgic journey into the underground party scene of the era.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>269509</td>\n",
       "      <td>The Canterville Ghost. An American family moves in to the Canterville Chase, a London mansion that has been haunted by ghost Sir Simon De Canterville for 300 years.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16171</th>\n",
       "      <td>1256551</td>\n",
       "      <td>Agastya – Chapter 1. Saurya is haunted by a difficult past and grapples with his troubled mind. After a year, his brother embarks on a quest to find him. A touching story unfolds as Nisan delves into Saurya's history.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16172</th>\n",
       "      <td>1256574</td>\n",
       "      <td>Eye of the Fen. Far in the future, in a place that was once called Finland, there lives a people whose history is shrouded in mystery. A mythical song describes the past, but it is not enough for the Young One. Fortunately, there is a sect in nearby fen that preserves ancient knowledge.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16173</th>\n",
       "      <td>1256587</td>\n",
       "      <td>Mushrooms. n Mushrooms (2023), the artist trains her lens on the boreal forests of Northeast Asia. Beneath the canopy’s dappled light, the camera follows the transfer of energy and information through the micro-ecosystems on the forest’s floor, presenting the interplay of mushrooms, moss, insects, and soil as a grand dance.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16174</th>\n",
       "      <td>1256588</td>\n",
       "      <td>How I Trafficked $5 Million Worth of Cocaine | Posh Pete’s Full Story. \"Former cocaine smuggler Pieter Tritton returns for a second interview with Business Insider about his experience trafficking drugs from South America to the United Kingdom. Tritton says he started selling cocaine in the illegal rave scene in the UK in the 2000s. He then established a cartel connection and began importing cocaine to Europe in larger quantities. Tritton was arrested in Ecuador and served 10 years in prison there, first in Garcia Moreno in Quito, and later in Litoral Penitentiary in Guayaquil, which is one of the world's most violent and corrupt prisons.  \"He now works as a public speaker on the dangers of drugs, consults with the UK police force, and is writing a follow-up to his 2017 memoir, 'El Infierno: Drugs, Gangs, Riots and Murder: My time inside Ecuador's toughest prisons.'\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16175</th>\n",
       "      <td>1256606</td>\n",
       "      <td>Il n'y a pas d'Aventure. A romance novel author seeks inspiration over a glass of whiskey. While contemplating the customers around him, he has a revelation.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16176 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id  \\\n",
       "0        27296   \n",
       "1        65456   \n",
       "2        75594   \n",
       "3       187716   \n",
       "4       269509   \n",
       "...        ...   \n",
       "16171  1256551   \n",
       "16172  1256574   \n",
       "16173  1256587   \n",
       "16174  1256588   \n",
       "16175  1256606   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             movieText  \n",
       "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    The Gingerdead Man. An evil yet adorable Gingerbread man comes to life with the soul of a convicted killer, and this real life cookie monster wreaks havoc on the girl who sent the killer to the electric chair.  \n",
       "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        The Atrocity Exhibition. A doctor in a mental research institution is driven insane by the spectacle of the horrors of the twentieth century.  \n",
       "2                                                                                                                                                                                                                                                                                        Blind Turn. Samantha Holt had the perfect life with a handsome fiancée and a future that couldn’t look brighter. But an unexpected turn on a dark lonely road puts her on a collision course with Bruce Miller, a devoted family man who descends into madness when he loses everything he ever cared about. One year later, Bruce decides to give Samantha the punishment he believes she escaped. Samantha has struggled with her own demons since that fateful night, and Bruce will take her struggles to the edge when he implements a demented plan to teach her a twisted lesson that may cost her the ultimate price.  \n",
       "3                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    Universal Groove. Filmed in 1999, Universal Groove explores 90s rave culture through eight diverse characters seeking escape and self-discovery amidst drugs and dance, offering a nostalgic journey into the underground party scene of the era.  \n",
       "4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 The Canterville Ghost. An American family moves in to the Canterville Chase, a London mansion that has been haunted by ghost Sir Simon De Canterville for 300 years.  \n",
       "...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                ...  \n",
       "16171                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        Agastya – Chapter 1. Saurya is haunted by a difficult past and grapples with his troubled mind. After a year, his brother embarks on a quest to find him. A touching story unfolds as Nisan delves into Saurya's history.  \n",
       "16172                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  Eye of the Fen. Far in the future, in a place that was once called Finland, there lives a people whose history is shrouded in mystery. A mythical song describes the past, but it is not enough for the Young One. Fortunately, there is a sect in nearby fen that preserves ancient knowledge.  \n",
       "16173                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            Mushrooms. n Mushrooms (2023), the artist trains her lens on the boreal forests of Northeast Asia. Beneath the canopy’s dappled light, the camera follows the transfer of energy and information through the micro-ecosystems on the forest’s floor, presenting the interplay of mushrooms, moss, insects, and soil as a grand dance.  \n",
       "16174  How I Trafficked $5 Million Worth of Cocaine | Posh Pete’s Full Story. \"Former cocaine smuggler Pieter Tritton returns for a second interview with Business Insider about his experience trafficking drugs from South America to the United Kingdom. Tritton says he started selling cocaine in the illegal rave scene in the UK in the 2000s. He then established a cartel connection and began importing cocaine to Europe in larger quantities. Tritton was arrested in Ecuador and served 10 years in prison there, first in Garcia Moreno in Quito, and later in Litoral Penitentiary in Guayaquil, which is one of the world's most violent and corrupt prisons.  \"He now works as a public speaker on the dangers of drugs, consults with the UK police force, and is writing a follow-up to his 2017 memoir, 'El Infierno: Drugs, Gangs, Riots and Murder: My time inside Ecuador's toughest prisons.'\"  \n",
       "16175                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    Il n'y a pas d'Aventure. A romance novel author seeks inspiration over a glass of whiskey. While contemplating the customers around him, he has a revelation.  \n",
       "\n",
       "[16176 rows x 2 columns]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_text"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87cc5fbe-a720-432c-a16d-728cac499107",
   "metadata": {},
   "source": [
    "## Create indexes for the vector properties."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e70e4fe7-dd35-43e0-85dc-3b90fbbaa9c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"CREATE VECTOR INDEX theme_vectors IF NOT EXISTS \n",
    "                  FOR (t:Theme)\n",
    "                  ON (t.embedding)\n",
    "                  OPTIONS {indexConfig: \n",
    "                      {`vector.dimensions`: 1536,\n",
    "                       `vector.similarity_function`: 'cosine'\n",
    "                       }}\n",
    "                       \"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8ef363ac-fe42-4cf1-9c0e-bdbc74fabba8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"CREATE VECTOR INDEX movie_text_vectors IF NOT EXISTS \n",
    "                  FOR (m:Movie)\n",
    "                  ON (m.embedding)\n",
    "                  OPTIONS {indexConfig: \n",
    "                      {`vector.dimensions`: 1536,\n",
    "                       `vector.similarity_function`: 'cosine'\n",
    "                       }}\n",
    "                       \"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d24772d-0163-4263-b9fd-376962f16eb9",
   "metadata": {},
   "source": [
    "## Send vectors to Neo4j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "de11c422-e227-42b2-8c2b-aa3e918c227c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 5000.\n"
     ]
    }
   ],
   "source": [
    "for i in range(0, int(themes.shape[0]/5000) + 1):\n",
    "    theme_slice = themes.iloc[i*5000:(i+1)*5000].copy()\n",
    "    theme_slice['embedding'] = embeddings.embed_documents(theme_slice['description'].to_list())\n",
    "    gds.run_cypher(\"\"\"\n",
    "        UNWIND $themeData AS d\n",
    "        MATCH (t:Theme {description:d['description']})\n",
    "        CALL db.create.setNodeVectorProperty(t, 'embedding', d['embedding'])\n",
    "        RETURN count(*) AS updateCount\"\"\",\n",
    "                   {'themeData': theme_slice.to_dict('records')})\n",
    "    print(f\"Finished row {(i+1)*5000}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "6f4028dd-463b-472e-ba7e-0670e8053303",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n",
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 5000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 10000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished row 15000.\n",
      "Finished row 20000.\n"
     ]
    }
   ],
   "source": [
    "for i in range(0, int(movie_text.shape[0]/5000) + 1):\n",
    "    movie_slice = movie_text.iloc[i*5000:(i+1)*5000].copy()\n",
    "    movie_slice['embedding'] = embeddings.embed_documents(movie_slice['movieText'].to_list())\n",
    "    gds.run_cypher(\"\"\"\n",
    "        UNWIND $movieData AS d\n",
    "        MATCH (m:Movie {tmdbId:d['id']})\n",
    "        CALL db.create.setNodeVectorProperty(m, 'embedding', d['embedding'])\n",
    "        RETURN count(*) AS updateCount\"\"\",\n",
    "                   {'movieData': movie_slice.to_dict('records')})\n",
    "    print(f\"Finished row {(i+1)*5000}.\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chatbot",
   "language": "python",
   "name": "chatbot"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}