paw_demo/Step_1_Loading_and_Embedding_Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Add this directory to the path and load our functions\n",
    "import sys\n",
    "sys.path.append(\"../src/\")\n",
    "\n",
    "import paware\n",
    "\n",
    "import polars as pl\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Handling the Raw Data\n",
    "For the purposes of demonstration, we will work with a subset of the raw data that consists of posts in the following subreddits:\n",
    "\n",
    "* `r/wholefoods`\n",
    "* `r/TalesFromYourBank`\n",
    "* `r/GeneralMotors`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the Data\n",
    "\n",
    "The first step, is to load the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(147773, 14)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_parquet(\"../paw_demo/raw/raw_data.parquet\")\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we peek at the list of subreddits represented in the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3,)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>reddit_subreddit</th></tr><tr><td>str</td></tr></thead><tbody><tr><td>&quot;GeneralMotors&quot;</td></tr><tr><td>&quot;TalesFromYourB…</td></tr><tr><td>&quot;wholefoods&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3,)\n",
       "Series: 'reddit_subreddit' [str]\n",
       "[\n",
       "\t\"GeneralMotors\"\n",
       "\t\"TalesFromYourB…\n",
       "\t\"wholefoods\"\n",
       "]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"reddit_subreddit\"].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And we can peek at the data to get a sense of its structure:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 14)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>aware_post_type</th><th>aware_created_ts</th><th>reddit_id</th><th>reddit_name</th><th>reddit_created_utc</th><th>reddit_author</th><th>reddit_text</th><th>reddit_permalink</th><th>reddit_title</th><th>reddit_url</th><th>reddit_subreddit</th><th>reddit_link_id</th><th>reddit_parent_id</th><th>reddit_submission</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;comment&quot;</td><td>&quot;2023-06-10T22:…</td><td>&quot;jnqd263&quot;</td><td>&quot;t1_jnqd263&quot;</td><td>1686449528</td><td>&quot;Da_Bulls_312&quot;</td><td>&quot;I got to sit i…</td><td>&quot;/r/wholefoods/…</td><td>null</td><td>null</td><td>&quot;wholefoods&quot;</td><td>&quot;t3_145pvab&quot;</td><td>&quot;t3_145pvab&quot;</td><td>&quot;145pvab&quot;</td></tr><tr><td>&quot;comment&quot;</td><td>&quot;2023-08-24T12:…</td><td>&quot;jxkkawb&quot;</td><td>&quot;t1_jxkkawb&quot;</td><td>1692893821</td><td>&quot;Detroitbeardgu…</td><td>&quot;It wouldn&#x27;t hu…</td><td>&quot;/r/wholefoods/…</td><td>null</td><td>null</td><td>&quot;wholefoods&quot;</td><td>&quot;t3_15zz8jt&quot;</td><td>&quot;t1_jxke3u9&quot;</td><td>&quot;15zz8jt&quot;</td></tr><tr><td>&quot;comment&quot;</td><td>&quot;2023-08-24T20:…</td><td>&quot;jxmswe0&quot;</td><td>&quot;t1_jxmswe0&quot;</td><td>1692925155</td><td>&quot;cheesenuggets2…</td><td>&quot;Trigger Warnin…</td><td>&quot;/r/TalesFromYo…</td><td>null</td><td>null</td><td>&quot;TalesFromYourB…</td><td>&quot;t3_1607t9b&quot;</td><td>&quot;t1_jxluavb&quot;</td><td>&quot;1607t9b&quot;</td></tr><tr><td>&quot;comment&quot;</td><td>&quot;2023-03-30T20:…</td><td>&quot;jech82k&quot;</td><td>&quot;t1_jech82k&quot;</td><td>1680222210</td><td>&quot;StoreProfessio…</td><td>&quot;Weed acid and …</td><td>&quot;/r/wholefoods/…</td><td>null</td><td>null</td><td>&quot;wholefoods&quot;</td><td>&quot;t3_126g150&quot;</td><td>&quot;t3_126g150&quot;</td><td>&quot;126g150&quot;</td></tr><tr><td>&quot;comment&quot;</td><td>&quot;2023-07-08T21:…</td><td>&quot;jr8017h&quot;</td><td>&quot;t1_jr8017h&quot;</td><td>1688864468</td><td>&quot;Mariah0&quot;</td><td>&quot;They come in f…</td><td>&quot;/r/wholefoods/…</td><td>null</td><td>null</td><td>&quot;wholefoods&quot;</td><td>&quot;t3_14ukqnf&quot;</td><td>&quot;t3_14ukqnf&quot;</td><td>&quot;14ukqnf&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 14)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ aware_pos ┆ aware_cre ┆ reddit_id ┆ reddit_na ┆ … ┆ reddit_su ┆ reddit_li ┆ reddit_pa ┆ reddit_s │\n",
       "│ t_type    ┆ ated_ts   ┆ ---       ┆ me        ┆   ┆ breddit   ┆ nk_id     ┆ rent_id   ┆ ubmissio │\n",
       "│ ---       ┆ ---       ┆ str       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ n        │\n",
       "│ str       ┆ str       ┆           ┆ str       ┆   ┆ str       ┆ str       ┆ str       ┆ ---      │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ str      │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ comment   ┆ 2023-06-1 ┆ jnqd263   ┆ t1_jnqd26 ┆ … ┆ wholefood ┆ t3_145pva ┆ t3_145pva ┆ 145pvab  │\n",
       "│           ┆ 0T22:12:0 ┆           ┆ 3         ┆   ┆ s         ┆ b         ┆ b         ┆          │\n",
       "│           ┆ 8         ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ comment   ┆ 2023-08-2 ┆ jxkkawb   ┆ t1_jxkkaw ┆ … ┆ wholefood ┆ t3_15zz8j ┆ t1_jxke3u ┆ 15zz8jt  │\n",
       "│           ┆ 4T12:17:0 ┆           ┆ b         ┆   ┆ s         ┆ t         ┆ 9         ┆          │\n",
       "│           ┆ 1         ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ comment   ┆ 2023-08-2 ┆ jxmswe0   ┆ t1_jxmswe ┆ … ┆ TalesFrom ┆ t3_1607t9 ┆ t1_jxluav ┆ 1607t9b  │\n",
       "│           ┆ 4T20:59:1 ┆           ┆ 0         ┆   ┆ YourBank  ┆ b         ┆ b         ┆          │\n",
       "│           ┆ 5         ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ comment   ┆ 2023-03-3 ┆ jech82k   ┆ t1_jech82 ┆ … ┆ wholefood ┆ t3_126g15 ┆ t3_126g15 ┆ 126g150  │\n",
       "│           ┆ 0T20:23:3 ┆           ┆ k         ┆   ┆ s         ┆ 0         ┆ 0         ┆          │\n",
       "│           ┆ 0         ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ comment   ┆ 2023-07-0 ┆ jr8017h   ┆ t1_jr8017 ┆ … ┆ wholefood ┆ t3_14ukqn ┆ t3_14ukqn ┆ 14ukqnf  │\n",
       "│           ┆ 8T21:01:0 ┆           ┆ h         ┆   ┆ s         ┆ f         ┆ f         ┆          │\n",
       "│           ┆ 8         ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing\n",
    "\n",
    "Because we are (outside of this demo) working with a gigantic dataset, we will first break it up into separate files that each contain the data from an individual subreddit. This ensures that we preserve all of the important structure (all of the replies, and replies to replies, will be contained in the same file) but we are less likely to overload our memory when handling the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Get a list of the subreddits present in the dataset\n",
    "subreddit_list = df[\"reddit_subreddit\"].unique().to_list()\n",
    "\n",
    "## Write the data for each individual subreddit to distinct parquet files\n",
    "for subreddit in subreddit_list:\n",
    "    df_subreddit = df.filter(pl.col(\"reddit_subreddit\") == subreddit)\n",
    "    df_subreddit.write_parquet(f\"../paw_demo/raw_subs/{subreddit.lower()}_data.parquet\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embedding\n",
    "\n",
    "Now, we are ready to get a vector embedding of the data. We'll use our custom module to implment this.\n",
    "\n",
    "Here we will apply the General Text Embedding [thenlper/gte-base](https://huggingface.co/thenlper/gte-base) to generate vector representations of our data. The parameters we define are:\n",
    "\n",
    "* `CONFIG_NAME`: A name to distinguish this configuration.\n",
    "* `RAW_DATA_DIR`: The directory containing our raw data (in this case, after we've added sentiment data).\n",
    "* `EMBEDDED_SAVE_DIR`: The directory in which to save the files after embedding.\n",
    "* `BATCH_SIZE`: We're using the `embed_documents()` function for Hugging Face Embeddings provided by LangChain ([docs](https://github.com/langchain-ai/langchain/blob/ea43c669f28f4564cc23cb9cc1356e363508e69d/libs/community/langchain_community/embeddings/huggingface.py#L81)) and this is quite computationally and memory intensive. Breaking the data into batches helps avoid overloading RAM.\n",
    "* `CHUNK_WITH_METADATA`: Whether we should append information about the subreddit to the beginning of each text chink before embedding.\n",
    "* `CHUNK_OVERLAP_PCT`: The minimum overlap between adjacent chuncks of text to be embedded, as a percent."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading and chunking...\n",
      "Dropping 239 rows with reddit_text=='[deleted]'\n",
      "Dropping 40 rows with reddit_text=='[removed]'\n",
      "Dropping 0 rows that are likely bots or memes\n",
      "Dropping 0 rows with 'reddit_text'=='' and 'aware_post_type'=='comment'\n",
      "Dropping 0 rows with 'reddit_text'==' ' and 'aware_post_type'=='comment'\n",
      "Replacing 'reddit_text' with 'reddit_title' in 167 rows with 'reddit_text'=='' or 'reddit_text'==' '\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "sys:1: MapWithoutReturnDtypeWarning: Calling `map_elements` without specifying `return_dtype` can lead to unpredictable results. Specify `return_dtype` to silence this warning.\n",
      "/Users/marcos/MEGA/gitHub/erdos_paware/paw_demo/../src/data_processing/chunk.py:160: MapWithoutReturnDtypeWarning: Calling `map_elements` without specifying `return_dtype` can lead to unpredictable results. Specify `return_dtype` to silence this warning.\n",
      "  data_chunked = data_chunked.with_columns(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... done loading and chunking.\n",
      "\n",
      "Vectorizing and saving...\n",
      "Working on batch 0, rows 0 through 37642...\n",
      "Done with batch 0.\n",
      "Combining parquet files...\n",
      "... done vectorizing and saving.\n",
      "\n",
      "Loading and chunking...\n",
      "Dropping 1099 rows with reddit_text=='[deleted]'\n",
      "Dropping 119 rows with reddit_text=='[removed]'\n",
      "Dropping 67 rows that are likely bots or memes\n",
      "Dropping 0 rows with 'reddit_text'=='' and 'aware_post_type'=='comment'\n",
      "Dropping 0 rows with 'reddit_text'==' ' and 'aware_post_type'=='comment'\n",
      "Replacing 'reddit_text' with 'reddit_title' in 235 rows with 'reddit_text'=='' or 'reddit_text'==' '\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "sys:1: MapWithoutReturnDtypeWarning: Calling `map_elements` without specifying `return_dtype` can lead to unpredictable results. Specify `return_dtype` to silence this warning.\n",
      "/Users/marcos/MEGA/gitHub/erdos_paware/paw_demo/../src/data_processing/chunk.py:160: MapWithoutReturnDtypeWarning: Calling `map_elements` without specifying `return_dtype` can lead to unpredictable results. Specify `return_dtype` to silence this warning.\n",
      "  data_chunked = data_chunked.with_columns(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... done loading and chunking.\n",
      "\n",
      "Vectorizing and saving...\n",
      "Working on batch 0, rows 0 through 41559...\n",
      "Done with batch 0.\n",
      "Combining parquet files...\n",
      "... done vectorizing and saving.\n",
      "\n",
      "Loading and chunking...\n",
      "Dropping 700 rows with reddit_text=='[deleted]'\n",
      "Dropping 164 rows with reddit_text=='[removed]'\n",
      "Dropping 61 rows that are likely bots or memes\n",
      "Dropping 0 rows with 'reddit_text'=='' and 'aware_post_type'=='comment'\n",
      "Dropping 0 rows with 'reddit_text'==' ' and 'aware_post_type'=='comment'\n",
      "Replacing 'reddit_text' with 'reddit_title' in 631 rows with 'reddit_text'=='' or 'reddit_text'==' '\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "sys:1: MapWithoutReturnDtypeWarning: Calling `map_elements` without specifying `return_dtype` can lead to unpredictable results. Specify `return_dtype` to silence this warning.\n",
      "/Users/marcos/MEGA/gitHub/erdos_paware/paw_demo/../src/data_processing/chunk.py:160: MapWithoutReturnDtypeWarning: Calling `map_elements` without specifying `return_dtype` can lead to unpredictable results. Specify `return_dtype` to silence this warning.\n",
      "  data_chunked = data_chunked.with_columns(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... done loading and chunking.\n",
      "\n",
      "Vectorizing and saving...\n",
      "Working on batch 0, rows 0 through 92742...\n",
      "Done with batch 0.\n",
      "Combining parquet files...\n",
      "... done vectorizing and saving.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Define the embedding tool\n",
    "embedding_tool = paware.PawEmbedding(\n",
    "    CONFIG_NAME = \"demo\",\n",
    "    RAW_DATA_DIR = \"../paw_demo/raw_subs/\",\n",
    "    EMBEDDED_SAVE_DIR =  \"../paw_demo/embedded_subs/\",\n",
    "    BATCH_SIZE = 100000,\n",
    "    CHUNK_WITH_METADATA = True,\n",
    "    CHUNK_SIZE = 512,\n",
    "    CHUNK_OVERLAP_PCT = 0.2\n",
    ")\n",
    "\n",
    "## Embed the data\n",
    "embedding_tool.embed_from_subs(subs_dir=\"../paw_demo/raw_subs/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Verifying the Results\n",
    "\n",
    "We can load the data now, and verify that it includes the embedding data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 18)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>aware_post_type</th><th>aware_created_ts</th><th>reddit_id</th><th>reddit_name</th><th>reddit_created_utc</th><th>reddit_author</th><th>reddit_text</th><th>reddit_permalink</th><th>reddit_title</th><th>reddit_url</th><th>reddit_subreddit</th><th>reddit_link_id</th><th>reddit_parent_id</th><th>reddit_submission</th><th>reply_ids</th><th>is_short_question</th><th>text_chunk</th><th>vector</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>list[str]</td><td>bool</td><td>str</td><td>list[f64]</td></tr></thead><tbody><tr><td>&quot;submission&quot;</td><td>&quot;2024-01-05T10:…</td><td>&quot;18z96ud&quot;</td><td>&quot;t3_18z96ud&quot;</td><td>1704467742</td><td>&quot;Jmill2000&quot;</td><td>&quot;So I know GM m…</td><td>&quot;/r/GeneralMoto…</td><td>&quot;References&quot;</td><td>&quot;https://www.re…</td><td>&quot;GeneralMotors&quot;</td><td>null</td><td>null</td><td>null</td><td>[&quot;t1_kgfznkx&quot;, &quot;t1_kgg05vt&quot;]</td><td>false</td><td>&quot;General Motors…</td><td>[0.018657, 0.037279, … 0.037011]</td></tr><tr><td>&quot;submission&quot;</td><td>&quot;2024-01-05T10:…</td><td>&quot;18z96ud&quot;</td><td>&quot;t3_18z96ud&quot;</td><td>1704467742</td><td>&quot;Jmill2000&quot;</td><td>&quot;So I know GM m…</td><td>&quot;/r/GeneralMoto…</td><td>&quot;References&quot;</td><td>&quot;https://www.re…</td><td>&quot;GeneralMotors&quot;</td><td>null</td><td>null</td><td>null</td><td>[&quot;t1_kgfznkx&quot;, &quot;t1_kgg05vt&quot;]</td><td>false</td><td>&quot;General Motors…</td><td>[0.019537, 0.014495, … 0.008045]</td></tr><tr><td>&quot;comment&quot;</td><td>&quot;2024-01-05T10:…</td><td>&quot;kgfznkx&quot;</td><td>&quot;t1_kgfznkx&quot;</td><td>1704468972</td><td>&quot;warwolf0&quot;</td><td>&quot;Yea, that’s a …</td><td>&quot;/r/GeneralMoto…</td><td>null</td><td>null</td><td>&quot;GeneralMotors&quot;</td><td>&quot;t3_18z96ud&quot;</td><td>&quot;t3_18z96ud&quot;</td><td>&quot;18z96ud&quot;</td><td>null</td><td>false</td><td>&quot;General Motors…</td><td>[0.0408, 0.030147, … -0.004045]</td></tr><tr><td>&quot;comment&quot;</td><td>&quot;2024-01-05T10:…</td><td>&quot;kgg05vt&quot;</td><td>&quot;t1_kgg05vt&quot;</td><td>1704469154</td><td>&quot;tossedawaytheq…</td><td>&quot;Search Socrate…</td><td>&quot;/r/GeneralMoto…</td><td>null</td><td>null</td><td>&quot;GeneralMotors&quot;</td><td>&quot;t3_18z96ud&quot;</td><td>&quot;t3_18z96ud&quot;</td><td>&quot;18z96ud&quot;</td><td>[&quot;t1_kgg0rjf&quot;, &quot;t1_kgg79j0&quot;]</td><td>false</td><td>&quot;General Motors…</td><td>[0.02871, 0.029767, … 0.022664]</td></tr><tr><td>&quot;comment&quot;</td><td>&quot;2024-01-05T10:…</td><td>&quot;kgg0rjf&quot;</td><td>&quot;t1_kgg0rjf&quot;</td><td>1704469370</td><td>&quot;Ripinpasta69&quot;</td><td>&quot;Only works if …</td><td>&quot;/r/GeneralMoto…</td><td>null</td><td>null</td><td>&quot;GeneralMotors&quot;</td><td>&quot;t3_18z96ud&quot;</td><td>&quot;t1_kgg05vt&quot;</td><td>&quot;18z96ud&quot;</td><td>null</td><td>false</td><td>&quot;General Motors…</td><td>[0.012741, 0.031151, … 0.01205]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 18)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ aware_pos ┆ aware_cre ┆ reddit_id ┆ reddit_na ┆ … ┆ reply_ids ┆ is_short_ ┆ text_chun ┆ vector   │\n",
       "│ t_type    ┆ ated_ts   ┆ ---       ┆ me        ┆   ┆ ---       ┆ question  ┆ k         ┆ ---      │\n",
       "│ ---       ┆ ---       ┆ str       ┆ ---       ┆   ┆ list[str] ┆ ---       ┆ ---       ┆ list[f64 │\n",
       "│ str       ┆ str       ┆           ┆ str       ┆   ┆           ┆ bool      ┆ str       ┆ ]        │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ submissio ┆ 2024-01-0 ┆ 18z96ud   ┆ t3_18z96u ┆ … ┆ [\"t1_kgfz ┆ false     ┆ General   ┆ [0.01865 │\n",
       "│ n         ┆ 5T10:15:4 ┆           ┆ d         ┆   ┆ nkx\", \"t1 ┆           ┆ Motors,   ┆ 7, 0.037 │\n",
       "│           ┆ 2         ┆           ┆           ┆   ┆ _kgg05vt\" ┆           ┆ GM        ┆ 279, …   │\n",
       "│           ┆           ┆           ┆           ┆   ┆ ]         ┆           ┆           ┆ 0.037011 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ So I know ┆ ]        │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ GM…       ┆          │\n",
       "│ submissio ┆ 2024-01-0 ┆ 18z96ud   ┆ t3_18z96u ┆ … ┆ [\"t1_kgfz ┆ false     ┆ General   ┆ [0.01953 │\n",
       "│ n         ┆ 5T10:15:4 ┆           ┆ d         ┆   ┆ nkx\", \"t1 ┆           ┆ Motors,   ┆ 7, 0.014 │\n",
       "│           ┆ 2         ┆           ┆           ┆   ┆ _kgg05vt\" ┆           ┆ GM        ┆ 495, …   │\n",
       "│           ┆           ┆           ┆           ┆   ┆ ]         ┆           ┆           ┆ 0.008045 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ I think   ┆ ]        │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ that…     ┆          │\n",
       "│ comment   ┆ 2024-01-0 ┆ kgfznkx   ┆ t1_kgfznk ┆ … ┆ null      ┆ false     ┆ General   ┆ [0.0408, │\n",
       "│           ┆ 5T10:36:1 ┆           ┆ x         ┆   ┆           ┆           ┆ Motors,   ┆ 0.030147 │\n",
       "│           ┆ 2         ┆           ┆           ┆   ┆           ┆           ┆ GM        ┆ , … -0.0 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 04045]   │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ Yea,      ┆          │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ that’s …  ┆          │\n",
       "│ comment   ┆ 2024-01-0 ┆ kgg05vt   ┆ t1_kgg05v ┆ … ┆ [\"t1_kgg0 ┆ false     ┆ General   ┆ [0.02871 │\n",
       "│           ┆ 5T10:39:1 ┆           ┆ t         ┆   ┆ rjf\", \"t1 ┆           ┆ Motors,   ┆ , 0.0297 │\n",
       "│           ┆ 4         ┆           ┆           ┆   ┆ _kgg79j0\" ┆           ┆ GM        ┆ 67, …    │\n",
       "│           ┆           ┆           ┆           ┆   ┆ ]         ┆           ┆           ┆ 0.022664 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ Search    ┆ ]        │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ Socra…    ┆          │\n",
       "│ comment   ┆ 2024-01-0 ┆ kgg0rjf   ┆ t1_kgg0rj ┆ … ┆ null      ┆ false     ┆ General   ┆ [0.01274 │\n",
       "│           ┆ 5T10:42:5 ┆           ┆ f         ┆   ┆           ┆           ┆ Motors,   ┆ 1, 0.031 │\n",
       "│           ┆ 0         ┆           ┆           ┆   ┆           ┆           ┆ GM        ┆ 151, …   │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 0.01205] │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ Only      ┆          │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ works i…  ┆          │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_dir = \"../paw_demo/embedded_subs/config_demo/\"\n",
    "test = pl.read_parquet(\n",
    "    embedding_dir+\"vectorized_generalmotors_data_complete.parquet\")\n",
    "\n",
    "test.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice that the `is_short_question` flag has been added as part of preprocessing."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "aware",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}