langkit/examples/Logging_Text.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/whylabs/LanguageToolkit/blob/main/langkit/examples/Logging_Text.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install langkit\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "schema does not contain metadata, LangKit won't update metadata\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done initializing metrics.\n"
     ]
    }
   ],
   "source": [
    "import whylogs as why\n",
    "from langkit import light_metrics\n",
    "\n",
    "llm_schema = light_metrics.init()\n",
    "print(\"Done initializing metrics.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`light_metrics` is composed by the following modules:\n",
    "- `textstat`: Text quality, readability, complexity, and grade level.\n",
    "- `regexes`: Regex pattern matching for sensitive information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scenario 1: Feature Extraction\n",
    "\n",
    "Langkit can be used to extract features from text data.\n",
    "\n",
    "The following snippet will extract additional features from the text data. The llm_schema created previously will guide the feature extraction process. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prompt</th>\n",
       "      <th>response</th>\n",
       "      <th>prompt.flesch_reading_ease</th>\n",
       "      <th>response.flesch_reading_ease</th>\n",
       "      <th>prompt.automated_readability_index</th>\n",
       "      <th>response.automated_readability_index</th>\n",
       "      <th>prompt.aggregate_reading_level</th>\n",
       "      <th>response.aggregate_reading_level</th>\n",
       "      <th>prompt.syllable_count</th>\n",
       "      <th>response.syllable_count</th>\n",
       "      <th>...</th>\n",
       "      <th>prompt.letter_count</th>\n",
       "      <th>response.letter_count</th>\n",
       "      <th>prompt.polysyllable_count</th>\n",
       "      <th>response.polysyllable_count</th>\n",
       "      <th>prompt.monosyllable_count</th>\n",
       "      <th>response.monosyllable_count</th>\n",
       "      <th>prompt.difficult_words</th>\n",
       "      <th>response.difficult_words</th>\n",
       "      <th>prompt.has_patterns</th>\n",
       "      <th>response.has_patterns</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hello</td>\n",
       "      <td>World</td>\n",
       "      <td>36.62</td>\n",
       "      <td>121.22</td>\n",
       "      <td>2.6</td>\n",
       "      <td>2.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is your number?</td>\n",
       "      <td>my phone is +1 309-404-7587</td>\n",
       "      <td>92.80</td>\n",
       "      <td>117.16</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2.7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>16</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>phone number</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 prompt                     response  \\\n",
       "0                 Hello                        World   \n",
       "1  What is your number?  my phone is +1 309-404-7587   \n",
       "\n",
       "   prompt.flesch_reading_ease  response.flesch_reading_ease  \\\n",
       "0                       36.62                        121.22   \n",
       "1                       92.80                        117.16   \n",
       "\n",
       "   prompt.automated_readability_index  response.automated_readability_index  \\\n",
       "0                                 2.6                                   2.6   \n",
       "1                                 0.6                                   2.7   \n",
       "\n",
       "   prompt.aggregate_reading_level  response.aggregate_reading_level  \\\n",
       "0                             0.0                               0.0   \n",
       "1                             1.0                               2.0   \n",
       "\n",
       "   prompt.syllable_count  response.syllable_count  ...  prompt.letter_count  \\\n",
       "0                      2                        1  ...                    5   \n",
       "1                      5                        5  ...                   16   \n",
       "\n",
       "   response.letter_count  prompt.polysyllable_count  \\\n",
       "0                      5                          0   \n",
       "1                     20                          0   \n",
       "\n",
       "   response.polysyllable_count  prompt.monosyllable_count  \\\n",
       "0                            0                          0   \n",
       "1                            0                          3   \n",
       "\n",
       "   response.monosyllable_count  prompt.difficult_words  \\\n",
       "0                            1                       0   \n",
       "1                            5                       0   \n",
       "\n",
       "   response.difficult_words  prompt.has_patterns  response.has_patterns  \n",
       "0                         0                 None                   None  \n",
       "1                         0                 None           phone number  \n",
       "\n",
       "[2 rows x 26 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langkit import extract\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame({'prompt': ['Hello', 'What is your number?'], 'response': ['World','my phone is +1 309-404-7587']})\n",
    "\n",
    "enhanced_df = extract(df, schema=llm_schema)\n",
    "\n",
    "enhanced_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can also provide a __dictionary__:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'prompt': 'What is your number?',\n",
       " 'response': 'my phone is +1 309-404-7587',\n",
       " 'prompt.flesch_reading_ease': 92.8,\n",
       " 'response.flesch_reading_ease': 117.16,\n",
       " 'prompt.automated_readability_index': 0.6,\n",
       " 'response.automated_readability_index': 2.7,\n",
       " 'prompt.aggregate_reading_level': 1.0,\n",
       " 'response.aggregate_reading_level': 2.0,\n",
       " 'prompt.syllable_count': 5,\n",
       " 'response.syllable_count': 5,\n",
       " 'prompt.lexicon_count': 4,\n",
       " 'response.lexicon_count': 5,\n",
       " 'prompt.sentence_count': 1,\n",
       " 'response.sentence_count': 1,\n",
       " 'prompt.character_count': 17,\n",
       " 'response.character_count': 23,\n",
       " 'prompt.letter_count': 16,\n",
       " 'response.letter_count': 20,\n",
       " 'prompt.polysyllable_count': 0,\n",
       " 'response.polysyllable_count': 0,\n",
       " 'prompt.monosyllable_count': 3,\n",
       " 'response.monosyllable_count': 5,\n",
       " 'prompt.difficult_words': 0,\n",
       " 'response.difficult_words': 0,\n",
       " 'prompt.has_patterns': None,\n",
       " 'response.has_patterns': 'phone number'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enhanced_row = extract({\"prompt\": \"What is your number?\",\"response\": \"my phone is +1 309-404-7587\"},schema=llm_schema)\n",
    "enhanced_row\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scenario 2: Statistical Profiling with whylogs\n",
    "\n",
    "LangKit modules contain UDFs that automatically wire into the collection of UDFs on String features provided by whylogs by default.\n",
    "\n",
    "All we have to do is pass the schema to `why.log()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done profiling! Let's look at some of the metrics:\n",
      "prompt\n",
      "response\n",
      "\n",
      "Here is the summary for response metrics\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'counts/n': 1,\n",
       " 'counts/null': 0,\n",
       " 'counts/nan': 0,\n",
       " 'counts/inf': 0,\n",
       " 'types/integral': 0,\n",
       " 'types/fractional': 0,\n",
       " 'types/boolean': 0,\n",
       " 'types/string': 1,\n",
       " 'types/object': 0,\n",
       " 'types/tensor': 0,\n",
       " 'distribution/mean': 0.0,\n",
       " 'distribution/stddev': 0.0,\n",
       " 'distribution/n': 0,\n",
       " 'distribution/max': nan,\n",
       " 'distribution/min': nan,\n",
       " 'distribution/q_01': None,\n",
       " 'distribution/q_05': None,\n",
       " 'distribution/q_10': None,\n",
       " 'distribution/q_25': None,\n",
       " 'distribution/median': None,\n",
       " 'distribution/q_75': None,\n",
       " 'distribution/q_90': None,\n",
       " 'distribution/q_95': None,\n",
       " 'distribution/q_99': None,\n",
       " 'cardinality/est': 1.0,\n",
       " 'cardinality/upper_1': 1.000049929250618,\n",
       " 'cardinality/lower_1': 1.0,\n",
       " 'udf/has_patterns:counts/n': 1,\n",
       " 'udf/has_patterns:counts/null': 1,\n",
       " 'udf/has_patterns:counts/nan': 0,\n",
       " 'udf/has_patterns:counts/inf': 0,\n",
       " 'udf/has_patterns:types/integral': 0,\n",
       " 'udf/has_patterns:types/fractional': 0,\n",
       " 'udf/has_patterns:types/boolean': 0,\n",
       " 'udf/has_patterns:types/string': 0,\n",
       " 'udf/has_patterns:types/object': 0,\n",
       " 'udf/has_patterns:types/tensor': 0,\n",
       " 'udf/has_patterns:distribution/mean': 0.0,\n",
       " 'udf/has_patterns:distribution/stddev': 0.0,\n",
       " 'udf/has_patterns:distribution/n': 0,\n",
       " 'udf/has_patterns:distribution/max': nan,\n",
       " 'udf/has_patterns:distribution/min': nan,\n",
       " 'udf/has_patterns:distribution/q_01': None,\n",
       " 'udf/has_patterns:distribution/q_05': None,\n",
       " 'udf/has_patterns:distribution/q_10': None,\n",
       " 'udf/has_patterns:distribution/q_25': None,\n",
       " 'udf/has_patterns:distribution/median': None,\n",
       " 'udf/has_patterns:distribution/q_75': None,\n",
       " 'udf/has_patterns:distribution/q_90': None,\n",
       " 'udf/has_patterns:distribution/q_95': None,\n",
       " 'udf/has_patterns:distribution/q_99': None,\n",
       " 'udf/has_patterns:cardinality/est': 0.0,\n",
       " 'udf/has_patterns:cardinality/upper_1': 0.0,\n",
       " 'udf/has_patterns:cardinality/lower_1': 0.0,\n",
       " 'udf/has_patterns:frequent_items/frequent_strings': [],\n",
       " 'udf/automated_readability_index:counts/n': 1,\n",
       " 'udf/automated_readability_index:counts/null': 0,\n",
       " 'udf/automated_readability_index:counts/nan': 0,\n",
       " 'udf/automated_readability_index:counts/inf': 0,\n",
       " 'udf/automated_readability_index:types/integral': 0,\n",
       " 'udf/automated_readability_index:types/fractional': 1,\n",
       " 'udf/automated_readability_index:types/boolean': 0,\n",
       " 'udf/automated_readability_index:types/string': 0,\n",
       " 'udf/automated_readability_index:types/object': 0,\n",
       " 'udf/automated_readability_index:types/tensor': 0,\n",
       " 'udf/automated_readability_index:distribution/mean': 7.3,\n",
       " 'udf/automated_readability_index:distribution/stddev': 0.0,\n",
       " 'udf/automated_readability_index:distribution/n': 1,\n",
       " 'udf/automated_readability_index:distribution/max': 7.3,\n",
       " 'udf/automated_readability_index:distribution/min': 7.3,\n",
       " 'udf/automated_readability_index:distribution/q_01': 7.3,\n",
       " 'udf/automated_readability_index:distribution/q_05': 7.3,\n",
       " 'udf/automated_readability_index:distribution/q_10': 7.3,\n",
       " 'udf/automated_readability_index:distribution/q_25': 7.3,\n",
       " 'udf/automated_readability_index:distribution/median': 7.3,\n",
       " 'udf/automated_readability_index:distribution/q_75': 7.3,\n",
       " 'udf/automated_readability_index:distribution/q_90': 7.3,\n",
       " 'udf/automated_readability_index:distribution/q_95': 7.3,\n",
       " 'udf/automated_readability_index:distribution/q_99': 7.3,\n",
       " 'udf/automated_readability_index:cardinality/est': 1.0,\n",
       " 'udf/automated_readability_index:cardinality/upper_1': 1.000049929250618,\n",
       " 'udf/automated_readability_index:cardinality/lower_1': 1.0,\n",
       " 'udf/automated_readability_index:frequent_items/frequent_strings': [FrequentItem(value='7.300000', est=1, upper=1, lower=1)],\n",
       " 'udf/aggregate_reading_level:counts/n': 1,\n",
       " 'udf/aggregate_reading_level:counts/null': 0,\n",
       " 'udf/aggregate_reading_level:counts/nan': 0,\n",
       " 'udf/aggregate_reading_level:counts/inf': 0,\n",
       " 'udf/aggregate_reading_level:types/integral': 0,\n",
       " 'udf/aggregate_reading_level:types/fractional': 1,\n",
       " 'udf/aggregate_reading_level:types/boolean': 0,\n",
       " 'udf/aggregate_reading_level:types/string': 0,\n",
       " 'udf/aggregate_reading_level:types/object': 0,\n",
       " 'udf/aggregate_reading_level:types/tensor': 0,\n",
       " 'udf/aggregate_reading_level:distribution/mean': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/stddev': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/n': 1,\n",
       " 'udf/aggregate_reading_level:distribution/max': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/min': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/q_01': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/q_05': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/q_10': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/q_25': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/median': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/q_75': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/q_90': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/q_95': 0.0,\n",
       " 'udf/aggregate_reading_level:distribution/q_99': 0.0,\n",
       " 'udf/aggregate_reading_level:cardinality/est': 1.0,\n",
       " 'udf/aggregate_reading_level:cardinality/upper_1': 1.000049929250618,\n",
       " 'udf/aggregate_reading_level:cardinality/lower_1': 1.0,\n",
       " 'udf/aggregate_reading_level:frequent_items/frequent_strings': [FrequentItem(value='0.000000', est=1, upper=1, lower=1)],\n",
       " 'udf/syllable_count:counts/n': 1,\n",
       " 'udf/syllable_count:counts/null': 0,\n",
       " 'udf/syllable_count:counts/nan': 0,\n",
       " 'udf/syllable_count:counts/inf': 0,\n",
       " 'udf/syllable_count:types/integral': 1,\n",
       " 'udf/syllable_count:types/fractional': 0,\n",
       " 'udf/syllable_count:types/boolean': 0,\n",
       " 'udf/syllable_count:types/string': 0,\n",
       " 'udf/syllable_count:types/object': 0,\n",
       " 'udf/syllable_count:types/tensor': 0,\n",
       " 'udf/syllable_count:distribution/mean': 1.0,\n",
       " 'udf/syllable_count:distribution/stddev': 0.0,\n",
       " 'udf/syllable_count:distribution/n': 1,\n",
       " 'udf/syllable_count:distribution/max': 1.0,\n",
       " 'udf/syllable_count:distribution/min': 1.0,\n",
       " 'udf/syllable_count:distribution/q_01': 1.0,\n",
       " 'udf/syllable_count:distribution/q_05': 1.0,\n",
       " 'udf/syllable_count:distribution/q_10': 1.0,\n",
       " 'udf/syllable_count:distribution/q_25': 1.0,\n",
       " 'udf/syllable_count:distribution/median': 1.0,\n",
       " 'udf/syllable_count:distribution/q_75': 1.0,\n",
       " 'udf/syllable_count:distribution/q_90': 1.0,\n",
       " 'udf/syllable_count:distribution/q_95': 1.0,\n",
       " 'udf/syllable_count:distribution/q_99': 1.0,\n",
       " 'udf/syllable_count:cardinality/est': 1.0,\n",
       " 'udf/syllable_count:cardinality/upper_1': 1.000049929250618,\n",
       " 'udf/syllable_count:cardinality/lower_1': 1.0,\n",
       " 'udf/syllable_count:frequent_items/frequent_strings': [FrequentItem(value='1', est=1, upper=1, lower=1)],\n",
       " 'udf/lexicon_count:counts/n': 1,\n",
       " 'udf/lexicon_count:counts/null': 0,\n",
       " 'udf/lexicon_count:counts/nan': 0,\n",
       " 'udf/lexicon_count:counts/inf': 0,\n",
       " 'udf/lexicon_count:types/integral': 1,\n",
       " 'udf/lexicon_count:types/fractional': 0,\n",
       " 'udf/lexicon_count:types/boolean': 0,\n",
       " 'udf/lexicon_count:types/string': 0,\n",
       " 'udf/lexicon_count:types/object': 0,\n",
       " 'udf/lexicon_count:types/tensor': 0,\n",
       " 'udf/lexicon_count:distribution/mean': 1.0,\n",
       " 'udf/lexicon_count:distribution/stddev': 0.0,\n",
       " 'udf/lexicon_count:distribution/n': 1,\n",
       " 'udf/lexicon_count:distribution/max': 1.0,\n",
       " 'udf/lexicon_count:distribution/min': 1.0,\n",
       " 'udf/lexicon_count:distribution/q_01': 1.0,\n",
       " 'udf/lexicon_count:distribution/q_05': 1.0,\n",
       " 'udf/lexicon_count:distribution/q_10': 1.0,\n",
       " 'udf/lexicon_count:distribution/q_25': 1.0,\n",
       " 'udf/lexicon_count:distribution/median': 1.0,\n",
       " 'udf/lexicon_count:distribution/q_75': 1.0,\n",
       " 'udf/lexicon_count:distribution/q_90': 1.0,\n",
       " 'udf/lexicon_count:distribution/q_95': 1.0,\n",
       " 'udf/lexicon_count:distribution/q_99': 1.0,\n",
       " 'udf/lexicon_count:cardinality/est': 1.0,\n",
       " 'udf/lexicon_count:cardinality/upper_1': 1.000049929250618,\n",
       " 'udf/lexicon_count:cardinality/lower_1': 1.0,\n",
       " 'udf/lexicon_count:frequent_items/frequent_strings': [FrequentItem(value='1', est=1, upper=1, lower=1)],\n",
       " 'udf/sentence_count:counts/n': 1,\n",
       " 'udf/sentence_count:counts/null': 0,\n",
       " 'udf/sentence_count:counts/nan': 0,\n",
       " 'udf/sentence_count:counts/inf': 0,\n",
       " 'udf/sentence_count:types/integral': 1,\n",
       " 'udf/sentence_count:types/fractional': 0,\n",
       " 'udf/sentence_count:types/boolean': 0,\n",
       " 'udf/sentence_count:types/string': 0,\n",
       " 'udf/sentence_count:types/object': 0,\n",
       " 'udf/sentence_count:types/tensor': 0,\n",
       " 'udf/sentence_count:distribution/mean': 1.0,\n",
       " 'udf/sentence_count:distribution/stddev': 0.0,\n",
       " 'udf/sentence_count:distribution/n': 1,\n",
       " 'udf/sentence_count:distribution/max': 1.0,\n",
       " 'udf/sentence_count:distribution/min': 1.0,\n",
       " 'udf/sentence_count:distribution/q_01': 1.0,\n",
       " 'udf/sentence_count:distribution/q_05': 1.0,\n",
       " 'udf/sentence_count:distribution/q_10': 1.0,\n",
       " 'udf/sentence_count:distribution/q_25': 1.0,\n",
       " 'udf/sentence_count:distribution/median': 1.0,\n",
       " 'udf/sentence_count:distribution/q_75': 1.0,\n",
       " 'udf/sentence_count:distribution/q_90': 1.0,\n",
       " 'udf/sentence_count:distribution/q_95': 1.0,\n",
       " 'udf/sentence_count:distribution/q_99': 1.0,\n",
       " 'udf/sentence_count:cardinality/est': 1.0,\n",
       " 'udf/sentence_count:cardinality/upper_1': 1.000049929250618,\n",
       " 'udf/sentence_count:cardinality/lower_1': 1.0,\n",
       " 'udf/sentence_count:frequent_items/frequent_strings': [FrequentItem(value='1', est=1, upper=1, lower=1)],\n",
       " 'udf/character_count:counts/n': 1,\n",
       " 'udf/character_count:counts/null': 0,\n",
       " 'udf/character_count:counts/nan': 0,\n",
       " 'udf/character_count:counts/inf': 0,\n",
       " 'udf/character_count:types/integral': 1,\n",
       " 'udf/character_count:types/fractional': 0,\n",
       " 'udf/character_count:types/boolean': 0,\n",
       " 'udf/character_count:types/string': 0,\n",
       " 'udf/character_count:types/object': 0,\n",
       " 'udf/character_count:types/tensor': 0,\n",
       " 'udf/character_count:distribution/mean': 6.0,\n",
       " 'udf/character_count:distribution/stddev': 0.0,\n",
       " 'udf/character_count:distribution/n': 1,\n",
       " 'udf/character_count:distribution/max': 6.0,\n",
       " 'udf/character_count:distribution/min': 6.0,\n",
       " 'udf/character_count:distribution/q_01': 6.0,\n",
       " 'udf/character_count:distribution/q_05': 6.0,\n",
       " 'udf/character_count:distribution/q_10': 6.0,\n",
       " 'udf/character_count:distribution/q_25': 6.0,\n",
       " 'udf/character_count:distribution/median': 6.0,\n",
       " 'udf/character_count:distribution/q_75': 6.0,\n",
       " 'udf/character_count:distribution/q_90': 6.0,\n",
       " 'udf/character_count:distribution/q_95': 6.0,\n",
       " 'udf/character_count:distribution/q_99': 6.0,\n",
       " 'udf/character_count:cardinality/est': 1.0,\n",
       " 'udf/character_count:cardinality/upper_1': 1.000049929250618,\n",
       " 'udf/character_count:cardinality/lower_1': 1.0,\n",
       " 'udf/character_count:frequent_items/frequent_strings': [FrequentItem(value='6', est=1, upper=1, lower=1)],\n",
       " 'udf/letter_count:counts/n': 1,\n",
       " 'udf/letter_count:counts/null': 0,\n",
       " 'udf/letter_count:counts/nan': 0,\n",
       " 'udf/letter_count:counts/inf': 0,\n",
       " 'udf/letter_count:types/integral': 1,\n",
       " 'udf/letter_count:types/fractional': 0,\n",
       " 'udf/letter_count:types/boolean': 0,\n",
       " 'udf/letter_count:types/string': 0,\n",
       " 'udf/letter_count:types/object': 0,\n",
       " 'udf/letter_count:types/tensor': 0,\n",
       " 'udf/letter_count:distribution/mean': 5.0,\n",
       " 'udf/letter_count:distribution/stddev': 0.0,\n",
       " 'udf/letter_count:distribution/n': 1,\n",
       " 'udf/letter_count:distribution/max': 5.0,\n",
       " 'udf/letter_count:distribution/min': 5.0,\n",
       " 'udf/letter_count:distribution/q_01': 5.0,\n",
       " 'udf/letter_count:distribution/q_05': 5.0,\n",
       " 'udf/letter_count:distribution/q_10': 5.0,\n",
       " 'udf/letter_count:distribution/q_25': 5.0,\n",
       " 'udf/letter_count:distribution/median': 5.0,\n",
       " 'udf/letter_count:distribution/q_75': 5.0,\n",
       " 'udf/letter_count:distribution/q_90': 5.0,\n",
       " 'udf/letter_count:distribution/q_95': 5.0,\n",
       " 'udf/letter_count:distribution/q_99': 5.0,\n",
       " 'udf/letter_count:cardinality/est': 1.0,\n",
       " 'udf/letter_count:cardinality/upper_1': 1.000049929250618,\n",
       " 'udf/letter_count:cardinality/lower_1': 1.0,\n",
       " 'udf/letter_count:frequent_items/frequent_strings': [FrequentItem(value='5', est=1, upper=1, lower=1)],\n",
       " 'udf/polysyllable_count:counts/n': 1,\n",
       " 'udf/polysyllable_count:counts/null': 0,\n",
       " 'udf/polysyllable_count:counts/nan': 0,\n",
       " 'udf/polysyllable_count:counts/inf': 0,\n",
       " 'udf/polysyllable_count:types/integral': 1,\n",
       " 'udf/polysyllable_count:types/fractional': 0,\n",
       " 'udf/polysyllable_count:types/boolean': 0,\n",
       " 'udf/polysyllable_count:types/string': 0,\n",
       " 'udf/polysyllable_count:types/object': 0,\n",
       " 'udf/polysyllable_count:types/tensor': 0,\n",
       " 'udf/polysyllable_count:distribution/mean': 0.0,\n",
       " 'udf/polysyllable_count:distribution/stddev': 0.0,\n",
       " 'udf/polysyllable_count:distribution/n': 1,\n",
       " 'udf/polysyllable_count:distribution/max': 0.0,\n",
       " 'udf/polysyllable_count:distribution/min': 0.0,\n",
       " 'udf/polysyllable_count:distribution/q_01': 0.0,\n",
       " 'udf/polysyllable_count:distribution/q_05': 0.0,\n",
       " 'udf/polysyllable_count:distribution/q_10': 0.0,\n",
       " 'udf/polysyllable_count:distribution/q_25': 0.0,\n",
       " 'udf/polysyllable_count:distribution/median': 0.0,\n",
       " 'udf/polysyllable_count:distribution/q_75': 0.0,\n",
       " 'udf/polysyllable_count:distribution/q_90': 0.0,\n",
       " 'udf/polysyllable_count:distribution/q_95': 0.0,\n",
       " 'udf/polysyllable_count:distribution/q_99': 0.0,\n",
       " 'udf/polysyllable_count:cardinality/est': 1.0,\n",
       " 'udf/polysyllable_count:cardinality/upper_1': 1.000049929250618,\n",
       " 'udf/polysyllable_count:cardinality/lower_1': 1.0,\n",
       " 'udf/polysyllable_count:frequent_items/frequent_strings': [FrequentItem(value='0', est=1, upper=1, lower=1)],\n",
       " 'udf/monosyllable_count:counts/n': 1,\n",
       " 'udf/monosyllable_count:counts/null': 0,\n",
       " 'udf/monosyllable_count:counts/nan': 0,\n",
       " 'udf/monosyllable_count:counts/inf': 0,\n",
       " 'udf/monosyllable_count:types/integral': 1,\n",
       " 'udf/monosyllable_count:types/fractional': 0,\n",
       " 'udf/monosyllable_count:types/boolean': 0,\n",
       " 'udf/monosyllable_count:types/string': 0,\n",
       " 'udf/monosyllable_count:types/object': 0,\n",
       " 'udf/monosyllable_count:types/tensor': 0,\n",
       " 'udf/monosyllable_count:distribution/mean': 1.0,\n",
       " 'udf/monosyllable_count:distribution/stddev': 0.0,\n",
       " 'udf/monosyllable_count:distribution/n': 1,\n",
       " 'udf/monosyllable_count:distribution/max': 1.0,\n",
       " 'udf/monosyllable_count:distribution/min': 1.0,\n",
       " 'udf/monosyllable_count:distribution/q_01': 1.0,\n",
       " 'udf/monosyllable_count:distribution/q_05': 1.0,\n",
       " 'udf/monosyllable_count:distribution/q_10': 1.0,\n",
       " 'udf/monosyllable_count:distribution/q_25': 1.0,\n",
       " 'udf/monosyllable_count:distribution/median': 1.0,\n",
       " 'udf/monosyllable_count:distribution/q_75': 1.0,\n",
       " 'udf/monosyllable_count:distribution/q_90': 1.0,\n",
       " 'udf/monosyllable_count:distribution/q_95': 1.0,\n",
       " 'udf/monosyllable_count:distribution/q_99': 1.0,\n",
       " 'udf/monosyllable_count:cardinality/est': 1.0,\n",
       " 'udf/monosyllable_count:cardinality/upper_1': 1.000049929250618,\n",
       " 'udf/monosyllable_count:cardinality/lower_1': 1.0,\n",
       " 'udf/monosyllable_count:frequent_items/frequent_strings': [FrequentItem(value='1', est=1, upper=1, lower=1)],\n",
       " 'udf/difficult_words:counts/n': 1,\n",
       " 'udf/difficult_words:counts/null': 0,\n",
       " 'udf/difficult_words:counts/nan': 0,\n",
       " 'udf/difficult_words:counts/inf': 0,\n",
       " 'udf/difficult_words:types/integral': 1,\n",
       " 'udf/difficult_words:types/fractional': 0,\n",
       " 'udf/difficult_words:types/boolean': 0,\n",
       " 'udf/difficult_words:types/string': 0,\n",
       " 'udf/difficult_words:types/object': 0,\n",
       " 'udf/difficult_words:types/tensor': 0,\n",
       " 'udf/difficult_words:distribution/mean': 0.0,\n",
       " 'udf/difficult_words:distribution/stddev': 0.0,\n",
       " 'udf/difficult_words:distribution/n': 1,\n",
       " 'udf/difficult_words:distribution/max': 0.0,\n",
       " 'udf/difficult_words:distribution/min': 0.0,\n",
       " 'udf/difficult_words:distribution/q_01': 0.0,\n",
       " 'udf/difficult_words:distribution/q_05': 0.0,\n",
       " 'udf/difficult_words:distribution/q_10': 0.0,\n",
       " 'udf/difficult_words:distribution/q_25': 0.0,\n",
       " 'udf/difficult_words:distribution/median': 0.0,\n",
       " 'udf/difficult_words:distribution/q_75': 0.0,\n",
       " 'udf/difficult_words:distribution/q_90': 0.0,\n",
       " 'udf/difficult_words:distribution/q_95': 0.0,\n",
       " 'udf/difficult_words:distribution/q_99': 0.0,\n",
       " 'udf/difficult_words:cardinality/est': 1.0,\n",
       " 'udf/difficult_words:cardinality/upper_1': 1.000049929250618,\n",
       " 'udf/difficult_words:cardinality/lower_1': 1.0,\n",
       " 'udf/difficult_words:frequent_items/frequent_strings': [FrequentItem(value='0', est=1, upper=1, lower=1)]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = why.log({\"prompt\": \"Hello,\", \"response\": \"World!\"}, schema=llm_schema)\n",
    "print(\"Done profiling! Let's look at some of the metrics:\")\n",
    "\n",
    "view = results.view()\n",
    "for col_name in view.get_columns():\n",
    "    print(col_name)\n",
    "print()\n",
    "print(\"Here is the summary for response metrics\")\n",
    "view.get_column(\"response\").to_summary_dict()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "whylabs-textmetricstoolkit-EeFODeF5-py3.8",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}