Concept_Bottleneck_Model_(Clean).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "V100",
      "machine_shape": "hm",
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#Overview\n"
      ],
      "metadata": {
        "id": "0GXR5QY841og"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "This rough notebook implements an example of a text bottleneck model. It is meant to be run as a google colab notebook. It currently downloads the cebab dataset and trains a TBM over it as an example."
      ],
      "metadata": {
        "id": "9u1a9OlE6WpR"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Imports and downloads"
      ],
      "metadata": {
        "id": "RBc6Lr79HQwp"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "vla5J1aZwGxt"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "#Downloading necessary packages\n",
        "!pip install openai     #Finetuning GPT-3\n",
        "!pip install transformers #For GPT-2 tokenizer for cost calculations\n",
        "!pip install swifter      #Useful for evaluation, gives a loading bar and parallelizes calls to API"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "#Expanding display size for convenience\n",
        "pd.options.display.max_colwidth = 1000\n",
        "\n",
        "import random\n",
        "random.seed(42)\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import json\n",
        "import openai\n",
        "import os\n",
        "import swifter\n",
        "from tenacity import retry, stop_after_attempt, wait_random_exponential, retry_if_exception_type\n",
        "from tqdm import tqdm"
      ],
      "metadata": {
        "id": "F1CW_09CwVw-"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "openai.api_key = input(\"Provide openai API Key\")\n",
        "os.environ['OPENAI_API_KEY']=openai.api_key"
      ],
      "metadata": {
        "id": "0m_fdRNGwaUN"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# API Helper Functions"
      ],
      "metadata": {
        "id": "4aQothlgHUJI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "import time\n",
        "\n",
        "alertIncrements = 20\n",
        "totalCost = 0\n",
        "totalDavinciTokens = 0\n",
        "totalGpt3TurboTokens = 0\n",
        "totalGpt3Turbo16kTokens = 0\n",
        "totalGpt4Tokens = 0\n",
        "totalCurieTokens = 0\n",
        "alertCounter = 0\n",
        "prompt_generations_log = []\n",
        "\n",
        "def enumerate_requests(limitLength, lastX):\n",
        "  for i, (x,y) in enumerate(prompt_generations_log[-lastX:]):\n",
        "    print(\"---\")\n",
        "    print(\"{} : {}, {}\".format(i, x, count_gpt2_tokens(y)))\n",
        "    print(y[:limitLength])\n",
        "\n",
        "# Function to check if an alertIncrement increase has been reached\n",
        "def check_alert_increment():\n",
        "    global alertCounter, totalCost, alertIncrements\n",
        "    if totalCost >= (alertCounter + 1) * alertIncrements:\n",
        "        alertCounter += 1\n",
        "        print(\"=======================================\")\n",
        "        print(\"=======================================\")\n",
        "        print(f\"Alert: Total cost has reached ${alertCounter * alertIncrements:.2f}\")\n",
        "        print(\"=======================================\")\n",
        "        print(\"=======================================\")\n",
        "\n",
        "#Function to count GPT-2 tokens\n",
        "#Importing GPT-2 tokenizer\n",
        "from transformers import GPT2Tokenizer, GPT2LMHeadModel\n",
        "\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "\n",
        "#Count tokens (gpt-3 uses the same tokenizer as gpt-2)\n",
        "def count_gpt2_tokens(text):\n",
        "    tokens = tokenizer.tokenize(text)\n",
        "    return len(tokens)\n",
        "\n",
        "def accountingCost(prompt, generation, model):\n",
        "    global totalCost, totalDavinciTokens, totalCurieTokens, totalGpt4Tokens, prompt_generations_log, totalGpt3TurboTokens, totalGpt3Turbo16kTokens\n",
        "    costPerTokenDavinci = 0.02/1000\n",
        "    costPerTokenCurie = 0.002/1000\n",
        "    costPerTokenGpt3TurboPrompt16k = 0.003/1000\n",
        "    costPerTokenGpt3TurboCompletion16k = 0.004/1000\n",
        "    costPerTokenGpt3TurboPrompt = 0.0015/1000\n",
        "    costPerTokenGpt3TurboCompletion = 0.002/1000\n",
        "    costPerTokenGpt4Prompt = 0.03/1000\n",
        "    costPerTokenGpt4Generation = 0.06/1000\n",
        "\n",
        "    promptTokens = count_gpt2_tokens(prompt)\n",
        "    genTokens = count_gpt2_tokens(generation)\n",
        "    numTokens = promptTokens + genTokens\n",
        "    if \"curie\" in model:\n",
        "        totalCurieTokens += numTokens\n",
        "        cost = costPerTokenCurie * numTokens\n",
        "    if \"davinci\" in model:\n",
        "        totalDavinciTokens += numTokens\n",
        "        cost = costPerTokenDavinci * numTokens\n",
        "    if \"3.5-turbo\" in model:\n",
        "      if \"16k\" in model:\n",
        "        totalGpt3Turbo16kTokens += numTokens\n",
        "        cost = costPerTokenGpt3TurboPrompt16k * promptTokens\n",
        "        cost += costPerTokenGpt3TurboCompletion16k * genTokens\n",
        "      else:\n",
        "        totalGpt3TurboTokens += numTokens\n",
        "        cost = costPerTokenGpt3TurboPrompt * promptTokens\n",
        "        cost += costPerTokenGpt3TurboCompletion * genTokens\n",
        "    if \"gpt-4\" in model:\n",
        "        totalGpt4Tokens += numTokens\n",
        "        cost = costPerTokenGpt4Prompt * promptTokens\n",
        "        cost += costPerTokenGpt4Generation * genTokens\n",
        "    totalCost += cost\n",
        "    prompt_generations_log.append((cost, prompt + generation))\n",
        "\n",
        "    # Check if an alertIncrement increase has been reached\n",
        "    check_alert_increment()\n",
        "\n",
        "@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(100), retry=(retry_if_exception_type(openai.error.RateLimitError) | retry_if_exception_type(openai.error.APIError) | retry_if_exception_type(openai.error.ServiceUnavailableError) | retry_if_exception_type(openai.error.APIConnectionError)))\n",
        "def completion(model, temp, prompt, stopToken=\"###\", maxtokens=200, format=\"completion\"):\n",
        "    # If the format is \"completion\"\n",
        "    if format == \"completion\":\n",
        "        response = openai.Completion.create(\n",
        "            engine=model,\n",
        "            prompt=prompt,\n",
        "            max_tokens=maxtokens,\n",
        "            stop=[stopToken],\n",
        "            temperature=temp\n",
        "        )\n",
        "        generated_text = response['choices'][0]['text']\n",
        "    # If the format is \"chat\"\n",
        "    elif format == \"chat\":\n",
        "        # Construct the messages list for the chat format\n",
        "        messages = [\n",
        "            {\"role\": \"system\", \"content\": \"You are an expert data annotator\"},\n",
        "            {\"role\": \"user\", \"content\": prompt}\n",
        "        ]\n",
        "\n",
        "        response = openai.ChatCompletion.create(\n",
        "            model=model,\n",
        "            messages=messages,\n",
        "            max_tokens=maxtokens,\n",
        "            stop=[stopToken],\n",
        "            temperature=temp\n",
        "        )\n",
        "        generated_text = response['choices'][0]['message']['content']\n",
        "    else:\n",
        "        raise ValueError(\"Unsupported format. Please use either 'completion' or 'chat'.\")\n",
        "\n",
        "    accountingCost(prompt, generated_text, model)\n",
        "    return generated_text\n",
        "\n",
        "\n",
        "@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(10), retry=(retry_if_exception_type(openai.error.RateLimitError) | retry_if_exception_type(openai.error.APIError)))\n",
        "def call_api(model, chunk, stopTokens, temp, max_tokens):\n",
        "    response = openai.Completion.create(\n",
        "        model=model,\n",
        "        prompt=chunk,\n",
        "        max_tokens=max_tokens,\n",
        "        stop=stopTokens,\n",
        "        temperature=temp\n",
        "    )\n",
        "    # Call accountingCost for each prompt and generated text\n",
        "    for choice in response.choices:\n",
        "        accountingCost(chunk[choice.index], choice.text, model)\n",
        "    return response\n",
        "\n",
        "@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(10), retry=(retry_if_exception_type(openai.error.RateLimitError) | retry_if_exception_type(openai.error.APIError)))\n",
        "def batchCompletion(batchSizes, model, temp, prompts, stopTokens=[\"###\"], waitTime=30, tokenLimit=1000):\n",
        "    results = []\n",
        "    listPrompts = list(prompts)\n",
        "    for i in tqdm(range(0, len(prompts), batchSizes)):\n",
        "        #time.sleep(waitTime)\n",
        "        chunk = listPrompts[i:i + batchSizes]\n",
        "        # batching\n",
        "        responses = [(\"\",\"\")] * len(chunk)\n",
        "        try:\n",
        "          response = call_api(\n",
        "              model=model,\n",
        "              chunk=chunk,\n",
        "              stopTokens=stopTokens,\n",
        "              temp=temp,\n",
        "              max_tokens=tokenLimit\n",
        "          )\n",
        "\n",
        "          # match completions to prompts by index\n",
        "          for choice in response.choices:\n",
        "              responses[choice.index] = (chunk[choice.index], choice.text)\n",
        "        except openai.error.InvalidRequestError:\n",
        "          print(\"batch prompting length error\")\n",
        "        results.extend(responses)\n",
        "    return results"
      ],
      "metadata": {
        "id": "Z3XqL9eFwh_G"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Text Bottleneck Model Code"
      ],
      "metadata": {
        "id": "Su-TI7Q0Hduw"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Pipeline:\n",
        "\n",
        "Concept generation\n",
        "1. Gather most misclassified examples based on current concept set\n",
        "2. Collect into concept generation prompt and generate concept\n",
        "3. Improve prompt to avoid common failure cases\n",
        "4. Trial the concept on a small sample to see if the concept improves accuracy\n",
        "5. Admit into concept set if it passes\n",
        "\n",
        "Concept Measurement\n",
        "1. Collect examples into batches to maximize prompt space\n",
        "2. Batch generate: snippet extraction, rationalization, final answer"
      ],
      "metadata": {
        "id": "xqKPwmEnA6xg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# @title\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.neighbors import NearestNeighbors\n",
        "from sklearn.metrics import mean_squared_error, accuracy_score\n",
        "from sklearn.linear_model import LinearRegression\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn import metrics\n",
        "\n",
        "from random import choice\n",
        "\n",
        "from concurrent.futures import ThreadPoolExecutor\n",
        "\n",
        "from sklearn.metrics import accuracy_score, mean_squared_error\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from IPython.display import clear_output\n",
        "import ipywidgets as widgets\n",
        "import textwrap\n",
        "import datetime\n",
        "from tqdm import tqdm\n",
        "\n",
        "\n",
        "class SupervisedConceptSpace():\n",
        "  #Initialize supervised concept space\n",
        "  def __init__(self, train_filepath, test_filepath, is_regression,\n",
        "               training_configs, name, description, label_guide,\n",
        "               label_column='label', text_column='text',\n",
        "               examplesMaxLength = 450, automated=False, modelChoices=None):\n",
        "    \"\"\"\n",
        "    Initializes the SupervisedConceptSpace class.\n",
        "\n",
        "    Args:\n",
        "    train_filepath (str): Path to the training data file.\n",
        "    test_filepath (str): Path to the testing data file.\n",
        "    is_regression (bool): Whether the task is a regression task.\n",
        "    training_configs (tuple): Configuration for training (train size limit (int), test size limit (int)).\n",
        "    name (str): Name of the dataset.\n",
        "    description (str): Description of the dataset.\n",
        "    label_guide (dict): Guide for the labels in the dataset.\n",
        "    label_column (str, optional): Name of the label column in the dataset. Defaults to 'label'.\n",
        "    text_column (str, optional): Name of the text column in the dataset. Defaults to 'text'.\n",
        "    examplesMaxLength (int, optional): Maximum length of the examples. Defaults to 450.\n",
        "\n",
        "    Returns:\n",
        "    None.\n",
        "    \"\"\"\n",
        "    train_size_limit, test_size_limit = training_configs\n",
        "    self.train_df_full = pd.read_csv(train_filepath)\n",
        "    self.test_df_full = pd.read_csv(test_filepath)\n",
        "    self.train_df = self.train_df_full.sample(min(train_size_limit, len(self.train_df_full)))\n",
        "    self.test_df = self.test_df_full.sample(min(test_size_limit, len(self.test_df_full)))\n",
        "    self.name = name\n",
        "    self.description = description\n",
        "    self.label_guide = label_guide\n",
        "    self.label_col = label_column\n",
        "    self.text_col = text_column\n",
        "    self.concepts_list = []\n",
        "    self.concepts_list_unfiltered = []\n",
        "    self.is_regression = is_regression\n",
        "    self.examplesMaxLength = examplesMaxLength\n",
        "    self.flags = []\n",
        "\n",
        "    #Printing width for interactive printing\n",
        "    self.printWidth = 150\n",
        "    self.automated = automated\n",
        "\n",
        "    #Logging\n",
        "    self.logs = []\n",
        "    self.dashboard = {}\n",
        "    self.dashboard['remaining_iterations'] = -1\n",
        "\n",
        "    DEFAULT_GENERATION_MODEL = \"gpt-4-0613\"\n",
        "    DEFAULT_GENERATION_FORMAT = \"chat\"\n",
        "\n",
        "    DEFAULT_MEASUREMENT_MODEL = \"gpt-4-0613\"\n",
        "    DEFAULT_MEASUREMENT_FORMAT = \"chat\"\n",
        "\n",
        "    if modelChoices:\n",
        "      DEFAULT_GENERATION_MODEL = modelChoices['DEFAULT_GENERATION_MODEL']\n",
        "      DEFAULT_GENERATION_FORMAT = modelChoices['DEFAULT_GENERATION_FORMAT']\n",
        "      DEFAULT_MEASUREMENT_MODEL = modelChoices['DEFAULT_MEASUREMENT_MODEL']\n",
        "      DEFAULT_MEASUREMENT_FORMAT = modelChoices['DEFAULT_MEASUREMENT_FORMAT']\n",
        "\n",
        "    self.generation_model = DEFAULT_GENERATION_MODEL\n",
        "    self.generation_format = DEFAULT_GENERATION_FORMAT\n",
        "    self.measurement_model = DEFAULT_MEASUREMENT_MODEL\n",
        "    self.measurement_format = DEFAULT_MEASUREMENT_FORMAT\n",
        "\n",
        "  # ==== Functions for sampling during Concept Generation ====\n",
        "\n",
        "  def sample_randomly(self, n_samples_max):\n",
        "    \"\"\"\n",
        "    Samples a specified number of examples from the training data, ensuring a balanced representation of labels.\n",
        "\n",
        "    Args:\n",
        "    n_samples_max (int): Maximum number of samples to take.\n",
        "\n",
        "    Returns:\n",
        "    samples (DataFrame): Sampled data.\n",
        "    \"\"\"\n",
        "    MAX_TOTAL_TEXT_LENGTH = 4000\n",
        "\n",
        "    df = self.train_df\n",
        "    labels = set(df[self.label_col])\n",
        "    samples_per_label = n_samples_max // len(labels)\n",
        "    dfs = []\n",
        "\n",
        "    for label in labels:\n",
        "        this_label_df = df[df[self.label_col] == label]\n",
        "        dfs.append(this_label_df.sample(min(samples_per_label, len(this_label_df))))\n",
        "\n",
        "    samples = pd.concat(dfs)\n",
        "\n",
        "    #Truncate texts\n",
        "    samples['text'] = samples['text'].str.slice(0,self.examplesMaxLength)\n",
        "\n",
        "    # Calculate the total text length\n",
        "    total_text_length = samples['text'].str.len().sum() * 1.2\n",
        "\n",
        "    # Remove examples from the label(s) with the highest number of texts until the total text length is within the limit\n",
        "    while total_text_length > MAX_TOTAL_TEXT_LENGTH:\n",
        "        label_counts = samples[self.label_col].value_counts()\n",
        "        max_label = label_counts.idxmax()\n",
        "\n",
        "        # Get the index of the first occurrence of the max_label\n",
        "        max_label_index = samples[samples[self.label_col] == max_label].index[0]\n",
        "\n",
        "        # Remove the row with the max_label\n",
        "        removed_text_length = len(samples.loc[max_label_index, 'text'])\n",
        "        samples = samples.drop(max_label_index)\n",
        "\n",
        "        # Update the total text length\n",
        "        total_text_length -= removed_text_length * 1.2\n",
        "\n",
        "    return samples\n",
        "\n",
        "  def get_neighborhood_mse(_, neighbor_idxs, df, train_pred, label_col):\n",
        "    \"\"\"\n",
        "    Calculates the mean squared error for a given neighborhood of data points.\n",
        "\n",
        "    Args:\n",
        "    neighbor_idxs (list): Indices of the neighbors.\n",
        "    df (DataFrame): DataFrame containing the data.\n",
        "    train_pred (array): Predicted labels for the training data.\n",
        "    label_col (str): Name of the label column.\n",
        "\n",
        "    Returns:\n",
        "    mse (float): Mean squared error.\n",
        "    \"\"\"\n",
        "    true_labels = df.loc[neighbor_idxs, label_col]\n",
        "    pred_labels = train_pred[neighbor_idxs]\n",
        "    mse = mean_squared_error(true_labels, pred_labels)\n",
        "    return mse\n",
        "\n",
        "  def get_neighborhood_accuracy(_, neighbor_idxs, df, featureSet, label_col, clf):\n",
        "    \"\"\"\n",
        "    Calculates the accuracy for a given neighborhood of data points.\n",
        "\n",
        "    Args:\n",
        "    neighbor_idxs (list): Indices of the neighbors.\n",
        "    df (DataFrame): DataFrame containing the data.\n",
        "    featureSet (list): List of features to use for prediction.\n",
        "    label_col (str): Name of the label column.\n",
        "    clf (Classifier): Classifier to use for prediction.\n",
        "\n",
        "    Returns:\n",
        "    acc (float): Accuracy.\n",
        "    \"\"\"\n",
        "    X = df.loc[neighbor_idxs, featureSet]\n",
        "    y_true = df.loc[neighbor_idxs, label_col]\n",
        "    y_pred = clf.predict(X)\n",
        "    acc = accuracy_score(y_true, y_pred)\n",
        "    return acc\n",
        "\n",
        "  def sample_high_error_neighbors(self, train_pred, n_samples_max, n_neighbors, max_length = None):\n",
        "      \"\"\"\n",
        "      Samples a specified number of examples from the training data that have high prediction error.\n",
        "\n",
        "      Args:\n",
        "      train_pred (array): Predicted labels for the training data.\n",
        "      n_samples_max (int): Maximum number of samples to take.\n",
        "      n_neighbors (int): Number of neighbors to consider.\n",
        "\n",
        "      Returns:\n",
        "      samples (DataFrame): Sampled data.\n",
        "      \"\"\"\n",
        "      if \"gpt-4\" in self.generation_model:\n",
        "        MAX_TOTAL_TEXT_LENGTH = 8000\n",
        "      else:\n",
        "        MAX_TOTAL_TEXT_LENGTH = 4000\n",
        "\n",
        "      if not max_length:\n",
        "        max_length = self.examplesMaxLength\n",
        "\n",
        "      concept_features = [c['Concept Name'] for c in self.concepts_list]\n",
        "      df_new = self.train_df.copy().reset_index(drop=True)\n",
        "\n",
        "      knn = NearestNeighbors().fit(np.array(df_new[concept_features]))\n",
        "      df_new['neighbors'] = pd.Series(knn.kneighbors(np.array(df_new[concept_features]), n_neighbors=n_neighbors)[1].tolist())\n",
        "\n",
        "      if self.is_regression:\n",
        "        df_new['neighborhood_loss'] = df_new['neighbors'].apply(self.get_neighborhood_mse, args=(df_new, train_pred, self.label_col))\n",
        "        idx_order = np.argsort(df_new['neighborhood_loss'])[::-1]\n",
        "      else:\n",
        "        df_new['neighborhood_acc'] = df_new['neighbors'].apply(self.get_neighborhood_accuracy, args=(df_new, concept_features, self.label_col, self.clf))\n",
        "        idx_order = np.argsort(df_new['neighborhood_acc'])\n",
        "      # Get the top n_samples_max rows\n",
        "      df_new = df_new.iloc[idx_order[:n_samples_max]]\n",
        "\n",
        "      #Truncate texts\n",
        "      df_new['text'] = df_new['text'].str.slice(0,max_length)\n",
        "\n",
        "      # Calculate the total text length\n",
        "      total_text_length = df_new['text'].str.len().sum() * 1.2\n",
        "\n",
        "      # Remove examples from the label(s) with the highest number of texts until the total text length is within the limit\n",
        "      while total_text_length > MAX_TOTAL_TEXT_LENGTH:\n",
        "          label_counts = df_new[self.label_col].value_counts()\n",
        "          max_label = label_counts.idxmax()\n",
        "\n",
        "          # Get the index of the first occurrence of the max_label\n",
        "          max_label_index = df_new[df_new[self.label_col] == max_label].index[0]\n",
        "\n",
        "          # Remove the row with the max_label\n",
        "          removed_text_length = len(df_new.loc[max_label_index, 'text'])\n",
        "          df_new = df_new.drop(max_label_index)\n",
        "\n",
        "          # Update the total text length\n",
        "          total_text_length -= removed_text_length\n",
        "\n",
        "      samples = df_new\n",
        "      if len(samples) < 4:\n",
        "        self.log(\"=== Too few examples, trying again with smaller text sizes ===\")\n",
        "        return self.sample_high_error_neighbors(\n",
        "            train_pred, n_samples_max, n_neighbors, int(max_length * 0.8))\n",
        "      if samples[self.label_col].nunique() == 1:\n",
        "        self.flags.append(sample)\n",
        "      else:\n",
        "        self.flags.append(\"clear\")\n",
        "      return samples\n",
        "\n",
        "  def generate_concept(self, samples, verbose = False):\n",
        "    \"\"\"\n",
        "    Generates a new concept by interacting with the user or using an automated process.\n",
        "\n",
        "    Args:\n",
        "    samples (DataFrame): Sampled data.\n",
        "    verbose (bool, optional): Whether to print verbose output. Defaults to False.\n",
        "\n",
        "    Returns:\n",
        "    conceptJson (dict): Generated concept.\n",
        "    \"\"\"\n",
        "    try:\n",
        "      texts = samples[self.text_col].str.slice(0,self.examplesMaxLength).to_list()\n",
        "      labels = samples[self.label_col].to_list()\n",
        "      input_type = -1\n",
        "\n",
        "      #------ Interactive Elements ---------\n",
        "      self.log(\"======= Concept Information =======\")\n",
        "      if len(self.concepts_list) > 0:\n",
        "        featureSet = [c['Concept Name'] for c in self.concepts_list]\n",
        "        X_train = self.train_df[featureSet]\n",
        "        y_train = self.train_df['label']\n",
        "\n",
        "        if self.is_regression:\n",
        "          model = LinearRegression()\n",
        "        else:\n",
        "          model = LogisticRegression()\n",
        "\n",
        "        model.fit(X_train, y_train)\n",
        "\n",
        "        coef_dict = {}\n",
        "        coefs = model.coef_[0] if len(model.coef_.shape) > 1 else model.coef_\n",
        "        for coef, concept in sorted(zip(coefs, self.concepts_list), key = lambda x: abs(x[0])):\n",
        "          self.log(f\"Feature: {concept['Concept Name']}, Coefficient:{coef}\")\n",
        "\n",
        "          button = widgets.Button(description=\"Show details\")\n",
        "\n",
        "          # Define what happens when button is clicked\n",
        "          def on_button_clicked(b, coef=coef, concept=concept):\n",
        "              self.log(f\"Clicked for feature: {concept['Concept Name']}, Coefficient: {coef}\")\n",
        "\n",
        "          # Attach the click event to the button\n",
        "          button.on_click(on_button_clicked)\n",
        "\n",
        "          # Display the button\n",
        "          display(button)\n",
        "      else:\n",
        "        self.log(\"Currently No Concepts\")\n",
        "\n",
        "      self.log(\"======= Concept Generation =======\")\n",
        "\n",
        "\n",
        "      while input_type not in [0, 1, 2]:\n",
        "        if self.automated:\n",
        "          input_type = '0'\n",
        "        else:\n",
        "          input_type = input(\"Write 0 for automated concept generation, Write 1 for suggesting a concept, Write 2 to exit and evaluate\")\n",
        "        if input_type.isdigit():\n",
        "          input_type = int(input_type)\n",
        "      if input_type == 0:\n",
        "        conceptGenerated = self.generate_differences_binary(texts, labels, verbose)\n",
        "      elif input_type == 1:\n",
        "        conceptGenerated = self.recommend_concept_assistant(texts, labels, verbose)\n",
        "      elif input_type == 2:\n",
        "        self.display_results()\n",
        "        return None\n",
        "\n",
        "      cleanedConcept = conceptGenerated.replace(\"\\n\",\"\").replace(\"“\",\"\\\"\").replace(\"”\",\"\\\"\")\n",
        "      conceptJson = json.loads(cleanedConcept)\n",
        "\n",
        "      self.concepts_list_unfiltered.append(conceptJson)\n",
        "      return conceptJson\n",
        "    except json.JSONDecodeError:\n",
        "      self.log(\"\"\"=== JSON DECODE ERROR in concept generation ===\"\"\")\n",
        "      return self.generate_concept(samples, verbose)\n",
        "\n",
        "  def generate_differences_binary(self, texts, labels, verbose=False):\n",
        "    \"\"\"\n",
        "    Generates a binary concept based on the differences between the examples.\n",
        "\n",
        "    Args:\n",
        "    texts (list): List of text examples.\n",
        "    labels (list): Corresponding labels for the text examples.\n",
        "    verbose (bool, optional): Whether to print verbose output. Defaults to False.\n",
        "\n",
        "    Returns:\n",
        "    response (str): Generated concept.\n",
        "    \"\"\"\n",
        "    # The method first formats the examples and labels into a string, and then\n",
        "    # uses the OpenAI API to generate a concept based on the examples and labels.\n",
        "    # The generated concept is returned as a string.\n",
        "\n",
        "    if not self.is_regression:\n",
        "      examples = [(text, self.label_guide[str(label)]) for text,label in zip(texts, labels)]\n",
        "      examples = sorted(examples, key=lambda x: x[1])\n",
        "      formatted_str = [\"text:{}\\nrating: {}\".format(x,y) for x,y in examples]\n",
        "      focus_str = f\"Focus on concepts that help you distinguish between the following labels from the examples above. The concept should be useful for identifying texts with the following label: {choice([y for x,y in examples])}, you are encouraged to extract keywords and inspiration from the example above.\"\n",
        "    else:\n",
        "      examples = [(text, str(label)) for text,label in zip(texts, labels)]\n",
        "      examples = sorted(examples, key=lambda x: x[1])\n",
        "      formatted_str = [\"text:{}\\nrating: {}\".format(x,y) for x,y in examples]\n",
        "      focus_str = \"\"\n",
        "    examples_string = \"\\n\\n\".join(formatted_str)\n",
        "\n",
        "\n",
        "    rejected_concepts = [x for x in concept_tree.concepts_list_unfiltered if\n",
        "                             x['Concept Name'] not in [y['Concept Name'] for\n",
        "                             y in concept_tree.concepts_list]]\n",
        "\n",
        "    rejected_concepts_str = ''\n",
        "    current_concepts_str = ''\n",
        "    for i in range(len(self.concepts_list)):\n",
        "      current_concepts_str += str(i+1)+'. '+self.concepts_list[i]['Concept Name'] + \":\" +self.concepts_list[i]['Concept Description']+\", possible responses: \"+str(self.concepts_list[i]['Possible Responses'])+'\\n'\n",
        "    for i in range(len(rejected_concepts)):\n",
        "      rejected_concepts_str += str(i+1)+'. '+rejected_concepts[i]['Concept Name'] + \":\" +rejected_concepts[i]['Concept Description']+\", possible responses: \"+str(rejected_concepts[i]['Possible Responses'])+'\\n'\n",
        "\n",
        "    prompt = f\"\"\"Concept Feature Engineering Task\n",
        "\n",
        "Below we are given a text dataset with accompanying labels. Our task is to identify a concept in the text that could be associated with the label. This is because we want to find the main factors that can be used to explain the label.\n",
        "\n",
        "To do this, we will examine a sample of texts that have different labels so that we can look at the different characteristics that exist for one label and compare it to another. Good concepts are those that separate texts with one label from another.\n",
        "\n",
        "After looking at these texts and finding a difference, we will define a concept definition JSON\n",
        "Each full concept definition comes with a concept name, description, question, response set, and response guide. The concept description provides an intuitive overview of the concept. The concept question is our tool for measuring the concept, this will be graded by a human annotator. The possible responses list the possible responses to the question and the response guide provides information on what each rating means. We also include a response mapping to help with data processing.\n",
        "\n",
        "Below are some examples of concepts for different datasets.\n",
        "\n",
        "1. A possible concept for a dataset assigning toxicity scores to social media texts\n",
        "{{\"Concept Name\": \"explicit language\",\n",
        "\"Concept Description\": \"'Explicit language' refers to the use of words, phrases, or expressions that are offensive, vulgar, or inappropriate for general audiences. This may include profanity, obscenities, slurs, sexually explicit or lewd language, and derogatory or discriminatory terms targeted at specific groups or individuals.\",\n",
        "\"Concept Question\": \"What is the nature of the language used in the text?\",\n",
        "\"Possible Responses\": [\"explicit\", \"strong\",\"non-explicit\", \"uncertain\"],\n",
        "\"Response Guide\": {{\n",
        "\"explicit\": \"The text contains explicit language, such as profanity, obscenities, slurs, sexually explicit or lewd language, or derogatory terms targeted at specific groups or individuals.\",\n",
        "\"strong\": \"The text contains strong language but not explicit language, it may contain terms that some viewers might find mature.\",\n",
        "\"non-explicit\": \"The text is free from explicit language and is appropriate for general audiences.\",\n",
        "\"uncertain\": \"It is difficult to determine the nature of the language used in the text or if any explicit terms are used.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"explicit\": 2,\n",
        "\"strong\":1,\n",
        "\"non-explicit\": -1,\n",
        "\"uncertain\": 0\n",
        "}}\n",
        "}}###\n",
        "\n",
        "2. A possible concept for evaluating the sentiment of product reviews on ecommerce site\n",
        "{{\"Concept Name\": \"good build quality\",\n",
        "\"Concept Description\": \"Build quality refers to the craftsmanship, durability, and overall construction of a product. It encompasses aspects such as materials used, design, manufacturing techniques, and attention to detail. A product with good build quality is typically considered to be well-made, sturdy, and long-lasting, while a product with poor build quality may be prone to defects or wear out quickly.\",\n",
        "\"Concept Question\": \"What does the review say about the build quality of the product?\",\n",
        "\"Possible Responses\": [\"positive\", \"negative\", \"uncertain\", \"not applicable\"],\n",
        "\"Response Guide\": {{\n",
        "\"high\": \"Review mentions aspects such as well-made, sturdy, durable, high-quality materials, excellent craftsmanship, etc.\",\n",
        "\"low\": \"Review mentions aspects such as poor construction, flimsy, cheap materials, bad design, easily breakable, etc.\",\n",
        "\"uncertain\": \"Review does not mention build quality, the information is ambiguous or vague, or it has both positive and negative aspects mentioned like 'the product is sturdy but uses cheap materials'.\",\n",
        "\"not applicable\": \"The review does not mention the build quality of the product at all.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"high\": 1,\n",
        "\"low\": -1,\n",
        "\"uncertain\": 0,\n",
        "\"not applicable\": 0\n",
        "}}\n",
        "}}###\n",
        "\n",
        "3. A useful concept for scam detection for emails\n",
        "{{\n",
        "\"Concept Name\": \"Extremely generous offer\",\n",
        "\"Concept Description\": \"The concept 'Extremely generous offer' refers to situations where the text describes an offer that seems too good to be true, such as promises of large financial gains, disproportionate rewards, or substantial benefits with seemingly little to no risk or effort required. These can often be indicative of scams or deceptive practices.\",\n",
        "\"Concept Question\": \"What type of offer is described in the text?\",\n",
        "\"Possible Responses\": [\"extremely generous offer\", \"ordinary offer\", \"no offer\", \"uncertain\"],\n",
        "\"Response Guide\": {{\n",
        "\"extremely generous offer\": \"The text describes an offer that is disproportionately rewarding or beneficial with seemingly little to no risk or effort. This could include promises of large financial returns with minimal investment, 'free' gifts that require payment information, or rewards that are disproportionate to the effort required.\",\n",
        "\"ordinary offer\": \"The text describes a typical or ordinary offer. For instance, normal sales or discounts, standard business offerings, or fair trades.\",\n",
        "\"no offer\": \"The text does not describe any offer.\",\n",
        "\"uncertain\": \"It is difficult to determine the type of offer described in the text. The text might be vague, ambiguous, or lack sufficient context.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"extremely generous offer\": 1,\n",
        "\"ordinary offer\": -2,\n",
        "\"no offer\": -1,\n",
        "\"uncertain\": 0\n",
        "}}\n",
        "}}\n",
        "\n",
        "---\n",
        "\n",
        "In the task, we will generate concepts for the {self.name} dataset\n",
        "\n",
        "Below is an explanation of the dataset and the labels therein:\n",
        "\n",
        "Description: {self.description}\n",
        "\n",
        "Label Guide: {self.label_guide}\n",
        "\n",
        "Below are some example texts along with their labels. These are examples that the model has a hard time classifying.\n",
        "---\n",
        "{examples_string}\n",
        "---\n",
        "As a reminder we already have the following concepts which are useful:\n",
        "{current_concepts_str}\n",
        "The following concepts have been rejected by the system, avoid making similar ones:\n",
        "{rejected_concepts_str}\n",
        "\n",
        "Keeping in mind the pointers above, create a concept below that is distinct from the current set of concepts. Additionally, make sure that all possible responses can be mapped to an integer.\n",
        "Make sure that the concept is as relevant to the labels in the {self.name} dataset. As the examples shown start to look more similar, we can start being more specific, picking out particular details tied to the label we notice in the examples above.\n",
        "Below is the description of the dataset.\n",
        "Description: {self.description}\n",
        "Additional Notes: {focus_str}\n",
        "Use strict adherence to the json format. Do not create duplicate concepts.\n",
        "Definition:\"\"\"\n",
        "\n",
        "    if self.generation_format == \"completion\":\n",
        "      formatted_prompt = prompt\n",
        "      response = completion(model=self.generation_model,\n",
        "                            temp=1,\n",
        "                            prompt= formatted_prompt,\n",
        "                            maxtokens=500,\n",
        "                            format=\"completion\"\n",
        "                            )\n",
        "    elif self.generation_format == \"chat\":\n",
        "      formatted_prompt = prompt\n",
        "      response = completion(model=self.generation_model,\n",
        "                            temp=1,\n",
        "                            prompt= formatted_prompt,\n",
        "                            maxtokens=500,\n",
        "                            format=\"chat\"\n",
        "                            )\n",
        "\n",
        "    if verbose:\n",
        "      self.log(prompt)\n",
        "    self.log(response)\n",
        "    return response\n",
        "\n",
        "  def measure_concept_prompt(self,concept,text):\n",
        "    \"\"\"\n",
        "    Generates the prompt for measuring a concept.\n",
        "\n",
        "    Args:\n",
        "    concept (dict): Concept to measure.\n",
        "    text (str): Text to measure the concept on.\n",
        "\n",
        "    Returns:\n",
        "    prompt (str): Generated prompt.\n",
        "    \"\"\"\n",
        "    # The method first formats the concept and text into a string,\n",
        "    # and then returns the string as the prompt for measuring the concept.\n",
        "    conceptString = str({k: concept[k] for k in ('Concept Name', 'Concept Description', 'Concept Question', 'Response Guide')})\n",
        "\n",
        "    prompt = \"\"\"Answer the following question about the text below by selecting from the following choices. Before answering the question, extract any potentially relevant snippets of the text that can serve as evidence for each classification. After that, compare the snippets against the response guide to come up with a final decision.\n",
        "\n",
        "Format your response as a JSON with string keys and string values. Below is an example of a valid JSON response. The JSON contains keys for snippets, thoughts, and answer. End your response with ###\n",
        "---\n",
        "Text: Text\n",
        "Response JSON:{{\"snippets\": {{\n",
        "\"classification 1\" : [\"Snippet 1\", \"Snippet 2\", ...],\n",
        "\"classification 2\" : [\"Snippet 3\", \"Snippet 4\", ...]\n",
        "...\n",
        "}},\n",
        "\"thoughts\": \"In this section, you weigh evidence based on the text and the extracted snippets to come to a final decision with the response guide as a reference. Be as objective as possible and ignore irrelevant information. Focus only on the snippets and avoid making guesses.\",\n",
        "\"answer\": \"An answer from the response guide goes here. In answering the question, ignore irrelevant information and avoid making assumptions.\"}}###\n",
        "---\n",
        "An example of this task being performed can be seen below. Note that the snippet should be as short as possible and be no greater than 10 words.\n",
        "\n",
        "Concept:\n",
        "{{\n",
        "\"Concept Name\": \"good build quality\",\n",
        "\"Concept Description\": \"build quality refers to the craftsmanship, durability, and overall construction of a product. It encompasses aspects such as materials used, design, manufacturing techniques, and attention to detail. A product with good build quality is typically considered to be well-made, sturdy, and long-lasting, while a product with poor build quality may be prone to defects or wear out quickly.\",\n",
        "\"Concept Question\": \"What does the review say about the build quality of the product?\",\n",
        "\"Possible Responses\": [\"high\", \"low\", \"uncertain\", \"not applicable\"],\n",
        "\"Response Guide\": {{\n",
        "\"high\": \"Review mentions aspects such as well-made, sturdy, durable, high-quality materials, excellent craftsmanship, etc.\",\n",
        "\"low\": \"Review mentions aspects such as poor construction, flimsy, cheap materials, bad design, easily breakable, etc.\",\n",
        "\"uncertain\": \"Review does not mention build quality, the information is ambiguous or vague, or it has both positive and negative aspects mentioned like 'the product is sturdy but uses cheap materials'.\",\n",
        "\"not applicable\": \"The review does not mention the build quality of the product at all.\"\n",
        "}}\n",
        "}}\n",
        "\n",
        "Text: \"This product has a great design and is really easy to use. It is also very durable.\"\n",
        "Response JSON:{{\"snippets\":{{\n",
        "\"high\": [\"It is also very durable\", \"really easy to use\"],\n",
        "\"low\": [],\n",
        "\"uncertain\": []\n",
        "}},\n",
        "\"thoughts\": \"Two snippets for high. The first is related to durability which is an aspect of good build quality. The second is related to ease of use which is not related to good build quality. Overall the text describes good build quality.\",\n",
        "\"answer\": \"positive\"\n",
        "}}###\n",
        "---\n",
        "Text: \"Was excited for it to finally arrive, got here in nice sturdy packaging. Opened it up though and it smelled kind of weird? goes away after a while but otherwise an ok product. Saw some print aberrations it didn't interfere much with use.\"\n",
        "Response JSON:{{\"snippets\":{{\n",
        "\"high\": [\"got here in nice sturdy packaging\"],\n",
        "\"low\": [\"Saw some print aberrations\"],\n",
        "\"uncertain\": []\n",
        "}},\n",
        "\"thoughts\": \"One snippet for high, one snippet for low. The low snippet comes with a caveat that it 'didn't interfere much with use'. In the positive snippet, 'Sturdy' only refers to the packaging, not the product. We do not have strong enough evidence for a high classification.\",\n",
        "\"answer\": \"low\"\n",
        "}}###\n",
        "---\n",
        "Text: \"A big fan of the product. Serves me well during workouts but I go through them like hotcakes. Don't expect it to last long compared to other brands but you get what you pay for. It does the job though.\"\n",
        "Response JSON:{{\"snippets\":\n",
        "{{\n",
        "\"high\": [\"Serves me well during workouts\"],\n",
        "\"low\": [\"Don't expect it to last long compared to other brands\", \"I go through them like hotcakes\"],\n",
        "\"uncertain\": [\"you get what you pay for\"]\n",
        "}},\n",
        "\"thoughts\": \"One high snippet, two low snippets, one uncertain snippet. The high snippet is about utility which is not related to build quality. The first and second low snippet relates to durability, an aspect of build quality.\",\n",
        "\"answer\": \"negative\"\n",
        "}}###\n",
        "---\n",
        "Text: \"Very disappointing. I was excited to order this but when it arrived I was shocked at how poorly it worked. Deceptive advertising at its finest.\"\n",
        "Response JSON:{{\"snippets\": {{\n",
        "\"high\": [],\n",
        "\"low\": [\"Very disappointing\", \"shocked at how poorly it worked\", \"Deceptive advertising\"],\n",
        "\"uncertain\": []\n",
        "}},\n",
        "\"thoughts\": \"Three low spans. The first is related to overall judgment which is irrelevant, the second is related to functionality which is irrelevant, and the third is related to marketing/advertising which is also irrelevant. None are related to build quality.\",\n",
        "\"answer\": \"not applicable\"\n",
        "}}###\n",
        "---\n",
        "Perform the task below, keeping in mind to limit snippets to 10 words and ignoring irrelevant information. Return a valid JSON response ending with ###\n",
        "Concept: {}\n",
        "\n",
        "Text: {}\n",
        "Response JSON:\"\"\".format(conceptString, text)\n",
        "\n",
        "    return prompt\n",
        "\n",
        "  def measure_concept_prompt_batched(self, concept, texts):\n",
        "    \"\"\"\n",
        "    Generates the prompts for measuring a concept on multiple texts.\n",
        "\n",
        "    Args:\n",
        "    concept (dict): Concept to measure.\n",
        "    texts (list): List of texts to measure the concept on.\n",
        "\n",
        "    Returns:\n",
        "    batch_sizes (list): Sizes of the batches.\n",
        "    prompts (list): List of generated prompts.\n",
        "    \"\"\"\n",
        "    # The method first formats the concept and each text into a string, and then\n",
        "    #returns a list of the strings as the prompts for measuring the concept.\n",
        "    #It also returns a list of the sizes of the batches.\n",
        "    conceptString = str({k: concept[k] for k in ('Concept Name', 'Concept Description', 'Concept Question', 'Response Guide')})\n",
        "\n",
        "    base_prompt = \"\"\"Answer the following question about the texts below by selecting from the following choices. Before answering the question, extract any potentially relevant snippets of the text that can serve as evidence for each classification. After that, compare the snippets against the response guide to come up with a final decision.\n",
        "\n",
        "Format your response as a list of JSON objects with string keys and string values. Below is an example of a valid JSON response. Each JSON object contains keys for snippets, thoughts, and answer. End your response with ###\n",
        "---\n",
        "Text 1: Text\n",
        "Text 2: Text\n",
        "Text 3: Text\n",
        "\n",
        "Response JSON:[\n",
        "{{\"text\": \"Text 1\", \"snippets\": {{\n",
        "\"classification 1\" : [\"Snippet 1\", \"Snippet 2\", ...],\n",
        "\"classification 2\" : [\"Snippet 3\", \"Snippet 4\", ...]\n",
        "...\n",
        "}},\n",
        "\"thoughts\": \"In this section, you weigh evidence based on the text and the extracted snippets to come to a final decision with the response guide as a reference. Be as objective as possible and ignore irrelevant information. Focus only on the snippets and avoid making guesses.\",\n",
        "\"answer\": \"An answer from the response guide goes here. In answering the question, ignore irrelevant information and avoid making assumptions.\"}},\n",
        "{{\"text\": \"Text 2\", \"snippets\": {{\n",
        "\"classification 1\" : [\"Snippet 1\", \"Snippet 2\", ...],\n",
        "\"classification 2\" : [\"Snippet 3\", \"Snippet 4\", ...]\n",
        "...\n",
        "}},\n",
        "\"thoughts\": \"...\",\n",
        "\"answer\": \"...\"}},\n",
        "{{\"text\": \"Text 3\", \"snippets\": {{\n",
        "\"classification 1\" : [\"Snippet 1\", \"Snippet 2\", ...],\n",
        "\"classification 2\" : [\"Snippet 3\", \"Snippet 4\", ...]\n",
        "...\n",
        "}}\n",
        "]###\n",
        "---\n",
        "Below is an example of the task being performed with the concept \"build quality\":\n",
        "\n",
        "Concept:\n",
        "{{\n",
        "\"Concept Name\": \"good build quality\",\n",
        "\"Concept Description\": \"build quality refers to the craftsmanship, durability, and overall construction of a product. It encompasses aspects such as materials used, design, manufacturing techniques, and attention to detail. A product with good build quality is typically considered to be well-made, sturdy, and long-lasting, while a product with poor build quality may be prone to defects or wear out quickly.\",\n",
        "\"Concept Question\": \"What does the review say about the build quality of the product?\",\n",
        "\"Possible Responses\": [\"high\", \"low\", \"uncertain\", \"not applicable\"],\n",
        "\"Response Guide\": {{\n",
        "\"high\": \"Review mentions aspects such as well-made, sturdy, durable, high-quality materials, excellent craftsmanship, etc.\",\n",
        "\"low\": \"Review mentions aspects such as poor construction, flimsy, cheap materials, bad design, easily breakable, etc.\",\n",
        "\"uncertain\": \"Review does not mention build quality, the information is ambiguous or vague, or it has both positive and negative aspects mentioned like 'the product is sturdy but uses cheap materials'.\",\n",
        "\"not applicable\": \"The review does not mention the build quality of the product at all.\"\n",
        "}}\n",
        "}}\n",
        "\n",
        "Text 1: \"This product has a great design and is really easy to use. It is also very durable.\"\n",
        "Text 2: \"Was excited for it to finally arrive, got here in nice sturdy packaging. Opened it up though and it smelled kind of weird? goes away after a while but otherwise an ok product. Saw some print aberrations it didn't interfere much with use.\"\n",
        "Text 3: \"A big fan of the product. Serves me well during workouts but I go through them like hotcakes. Don't expect it to last long compared to other brands but you get what you pay for. It does the job though.\"\n",
        "Text 4: \"Very disappointing. I was excited to order this but when it arrived I was shocked at how poorly it worked. Deceptive advertising at its finest.\"\n",
        "\n",
        "Response JSON:[\n",
        "{{\"text\": \"Text 1\", \"snippets\": {{\n",
        "\"high\": [\"It is also very durable\", \"really easy to use\"],\n",
        "\"low\": [],\n",
        "\"uncertain\": []\n",
        "}},\n",
        "\"thoughts\": \"Two snippets for high. The first is related to durability which is an aspect of good build quality. The second is related to ease of use which is not related to good build quality. Overall the text describes good build quality..\",\n",
        "\"answer\": \"high\"}},\n",
        "\n",
        "{{\"text\": \"Text 2\", \"snippets\": {{\n",
        "\"high\": [\"got here in nice sturdy packaging\"],\n",
        "\"low\": [\"Saw some print aberrations\"],\n",
        "\"uncertain\": []\n",
        "}},\n",
        "\"thoughts\": \"One snippet for high, one snippet for low. The low snippet mentions defects in manufacturing. In the high snippet, 'Sturdy' only refers to the packaging, not the product. The balance of evidence leans towards a classification of low.\",\n",
        "\"answer\": \"low\"}},\n",
        "\n",
        "{{\"text\": \"Text 3\", \"snippets\":\n",
        "{{\n",
        "\"high\": [\"Serves me well during workouts\"],\n",
        "\"low\": [\"Don't expect it to last long compared to other brands\", \"I go through them like hotcakes\"],\n",
        "\"uncertain\": [\"you get what you pay for\"]\n",
        "}},\n",
        "\"thoughts\": \"One high snippet, two low snippets, one uncertain snippet. The high snippet is about utility which is not related to build quality. The low snippet relates to durability, an aspect of build quality.\",\n",
        "\"answer\": \"low\"}},\n",
        "\n",
        "{{\"text\": \"Text 4\", \"snippets\":\n",
        "{{\n",
        "\"high\": [],\n",
        "\"low\": [\"Very disappointing\", \"shocked at how poorly it worked\", \"Deceptive advertising\"],\n",
        "\"uncertain\": []\n",
        "}},\n",
        "\"thoughts\": \"Three low spans. The first is related to overall judgment which is irrelevant, the second is related to functionality which is irrelevant, and the third is related to marketing/advertising which is also irrelevant. None are related to build quality.\",\n",
        "\"answer\": \"uncertain\"\n",
        "}}\n",
        "]###\n",
        "---\n",
        "Perform the task below, keeping in mind to limit snippets to 10 words and ignoring irrelevant information. Return a valid list of JSON objects ending with ###. Note that some texts are truncated due to text length. You must have an output for each piece of text.\n",
        "Concept: {}\n",
        "\n",
        "\"\"\".format(conceptString)\n",
        "\n",
        "    base_prompt_token_length = count_gpt2_tokens(base_prompt)\n",
        "\n",
        "    # Batch texts into groups no greater than 3000 tokens\n",
        "    batched_texts = []\n",
        "    batch_sizes = []\n",
        "    current_batch = \"\"\n",
        "    current_batch_allocated_tokens = 0\n",
        "    current_size = 0\n",
        "\n",
        "    for text in texts:\n",
        "        text_tokens = count_gpt2_tokens(text)\n",
        "        allocation = text_tokens * 2.5 + 200\n",
        "        if current_batch_allocated_tokens + allocation + base_prompt_token_length < 3500:\n",
        "            current_size += 1\n",
        "            current_batch += \"Text {}: {}\\n\".format(current_size, text)\n",
        "            current_batch_allocated_tokens += allocation\n",
        "        else:\n",
        "            if (current_size) > 0:\n",
        "              batched_texts.append(current_batch)\n",
        "              batch_sizes.append(current_size)\n",
        "\n",
        "            current_size = 1\n",
        "            current_batch = \"Text {}: {}\\n\".format(current_size, text[:2000])\n",
        "            current_batch_allocated_tokens = allocation\n",
        "\n",
        "    if current_batch:\n",
        "        batched_texts.append(current_batch)\n",
        "        batch_sizes.append(current_size)\n",
        "\n",
        "    # Add batched texts to the prompts\n",
        "    prompts = []\n",
        "    for batch, size in zip(batched_texts, batch_sizes):\n",
        "        prompt = base_prompt + batch + f\"\\nLabel for all {size} items:\\n Response JSON:[\\n{{\"\n",
        "        prompts.append(prompt)\n",
        "\n",
        "\n",
        "    self.log(\"batch sizing: {}\".format(str(batch_sizes[:5])))\n",
        "    return batch_sizes, prompts\n",
        "\n",
        "  def answer_extractor(self, responseText, response_mapping, nullable = False):\n",
        "    \"\"\"\n",
        "    Extracts the answer from the response of the OpenAI API.\n",
        "\n",
        "    Args:\n",
        "    responseText (str): Response from the OpenAI API.\n",
        "    response_mapping (dict): Mapping of responses to scores.\n",
        "    nullable (bool, optional): Whether to return None for invalid responses. Defaults to False.\n",
        "\n",
        "    Returns:\n",
        "    tuple: Extracted answer, score, snippets, and thoughts.\n",
        "    \"\"\"\n",
        "    # The method first cleans the response text and parses it into a JSON object.\n",
        "    # It then extracts the answer, score, snippets,\n",
        "    # and thoughts from the JSON object and returns them as a tuple.\n",
        "    try:\n",
        "        cleanedResponse = responseText.replace(\"\\n\", \"\").replace(\"“\", \"\\\"\").replace(\"”\", \"\\\"\")\n",
        "        jsonResponse = json.loads(cleanedResponse)\n",
        "        for item in ['snippets', 'thoughts','answer']:\n",
        "          if item not in jsonResponse.keys():\n",
        "            return (\"na\", 0, cleanedResponse, 'Invalid JSON response generated - missing keys')\n",
        "        answer = jsonResponse['answer']\n",
        "        score = response_mapping.get(answer, 0)\n",
        "        return (answer, score, jsonResponse['snippets'], jsonResponse['thoughts'])\n",
        "    except json.JSONDecodeError:\n",
        "        if nullable:\n",
        "          score = None\n",
        "        else:\n",
        "          score = 0\n",
        "        return (\"na\", 0, cleanedResponse, 'Invalid JSON response generated')\n",
        "\n",
        "  def answer_extractor_batched(self, responseText, response_mapping, batch_size, nullable = False):\n",
        "    \"\"\"\n",
        "    Extracts the answers from the response of the OpenAI API for multiple texts.\n",
        "\n",
        "    Args:\n",
        "    responseText (str): Response from the OpenAI API.\n",
        "    response_mapping (dict): Mapping of responses to scores.\n",
        "    batch_size (int): Number of texts in the batch.\n",
        "    nullable (bool, optional): Whether to return None for invalid responses. Defaults to False.\n",
        "\n",
        "    Returns:\n",
        "    list: List of extracted answers, scores, snippets, and thoughts for each text in the batch.\n",
        "    \"\"\"\n",
        "    # The method first cleans the response text and parses it into a list of JSON objects.\n",
        "    # It then extracts the answer, score, snippets, and thoughts from each JSON object\n",
        "    # and returns them as a list of tuples.\n",
        "    try:\n",
        "        extracted = []\n",
        "        cleanedResponse = \"[{\" + responseText.replace(\"\\n\", \"\").replace(\"“\", \"\\\"\").replace(\"”\", \"\\\"\").replace(\"],}\", \"]}\")\n",
        "        jsonResponse = json.loads(cleanedResponse)\n",
        "\n",
        "        for item in jsonResponse:\n",
        "          answer = item['answer'].lower()\n",
        "          try:\n",
        "            score = response_mapping.get(answer, 0)\n",
        "          except:\n",
        "            if nullable:\n",
        "              score = None\n",
        "            else:\n",
        "              score = 0\n",
        "          extracted.append((answer, score, item['snippets'], item['thoughts']))\n",
        "\n",
        "        if len(extracted) != batch_size:\n",
        "          return [(\"na\", 0, cleanedResponse, 'Invalid JSON response generated: Invalid size')]*batch_size\n",
        "        return extracted\n",
        "    except json.JSONDecodeError:\n",
        "        if nullable:\n",
        "          score = None\n",
        "        else:\n",
        "          score = 0\n",
        "        self.log(\"Json Decode Error in batch prompt\")\n",
        "        return [(\"na\", 0, cleanedResponse, 'Invalid JSON response generated: JSON Decode error')]*batch_size\n",
        "    except KeyError:\n",
        "        if nullable:\n",
        "          score = None\n",
        "        else:\n",
        "          score = 0\n",
        "        self.log(\"Key error in batch prompt\")\n",
        "        return [(\"na\", 0, cleanedResponse, 'Invalid JSON response generated: Key Error')]*batch_size\n",
        "\n",
        "  def measure_concept(self,texts,concept, batching=True):\n",
        "    \"\"\"\n",
        "    Measures a concept by interacting with the OpenAI API.\n",
        "\n",
        "    Args:\n",
        "    texts (list): List of texts to measure the concept on.\n",
        "    concept (dict): Concept to measure.\n",
        "    batching (bool, optional): Whether to batch the texts. Defaults to True.\n",
        "\n",
        "    Returns:\n",
        "    extracted (list): List of extracted answers, scores, snippets, and thoughts for each text.\n",
        "    \"\"\"\n",
        "    # The method first generates the prompts for measuring the concept on the texts.\n",
        "    # It then sends the prompts to the OpenAI API and extracts the answers, scores,\n",
        "    # snippets, and thoughts from the responses. The extracted information is returned as a list of tuples.\n",
        "    MODEL = self.measurement_model\n",
        "    TEMP = 0\n",
        "    BATCHSIZES = 20\n",
        "    truncated_texts = [x[0:self.examplesMaxLength] for x in texts]\n",
        "\n",
        "    if self.measurement_format == \"chat\":\n",
        "      prompts = [self.measure_concept_prompt(concept, text) for text in truncated_texts]\n",
        "      with ThreadPoolExecutor(max_workers=10) as executor:  # max_workers determines how many threads will be spawned\n",
        "            responses = list(tqdm(executor.map(lambda p: completion(MODEL, TEMP, p, stopToken=\"###\", maxtokens=1000, format=\"chat\"), prompts), total=len(prompts)))\n",
        "      extracted = [self.answer_extractor(x, concept['Response Mapping']) for x in responses]\n",
        "    elif self.measurement_format == \"completion\":\n",
        "      if batching:\n",
        "        batch_sizes, prompts = self.measure_concept_prompt_batched(concept, truncated_texts)\n",
        "        responses = batchCompletion(BATCHSIZES,MODEL,TEMP,prompts,stopTokens=[\"###\"],tokenLimit=1500, waitTime = 1)\n",
        "        extracted_list = [self.answer_extractor_batched(x[1], concept['Response Mapping'], batch_sizes[i]) for i,(x,batch_size) in enumerate(zip(responses, batch_sizes))]\n",
        "        extracted = [item for sublist in extracted_list for item in sublist]\n",
        "      else:\n",
        "        prompts = [\"{\" + self.measure_concept_prompt(concept, text) for text in truncated_texts]\n",
        "        responses = batchCompletion(BATCHSIZES,MODEL,TEMP,prompts,stopTokens=[\"###\"], waitTime = 1)\n",
        "        extracted = [self.answer_extractor(\"{\" + x[1], concept['Response Mapping']) for x in responses]\n",
        "\n",
        "    return extracted\n",
        "\n",
        "  def label_dataset(self, dataset, concept):\n",
        "    \"\"\"\n",
        "    Labels a dataset with a given concept, skipping already labeled texts.\n",
        "\n",
        "    Args:\n",
        "    dataset (DataFrame): Dataset to label.\n",
        "    concept (dict): Concept to use for labeling.\n",
        "\n",
        "    Returns:\n",
        "    None.\n",
        "    \"\"\"\n",
        "    concept_name = concept['Concept Name']\n",
        "    label_col_name = \"{}_label\".format(concept_name)\n",
        "\n",
        "    # Check if the dataset already contains labeled data for this concept\n",
        "    if label_col_name in dataset.columns:\n",
        "        # Only proceed with rows where the label column is not already populated\n",
        "        to_label_idx = dataset[dataset[label_col_name].isna()].index\n",
        "        subset_to_label = dataset.loc[to_label_idx]\n",
        "    else:\n",
        "        subset_to_label = dataset\n",
        "\n",
        "    # Proceed with labeling only if there are rows to label\n",
        "    if not subset_to_label.empty:\n",
        "        answers = self.measure_concept(subset_to_label[self.text_col], concept)\n",
        "\n",
        "        # Only update the rows that were actually labeled\n",
        "        dataset.loc[subset_to_label.index, concept_name] = [x[1] for x in answers]\n",
        "        dataset.loc[subset_to_label.index, label_col_name] = [x[0] for x in answers]\n",
        "        dataset.loc[subset_to_label.index, \"{}_snippets\".format(concept_name)] = [x[2] for x in answers]\n",
        "        dataset.loc[subset_to_label.index, \"{}_thoughts\".format(concept_name)] = [x[3] for x in answers]\n",
        "\n",
        "\n",
        "\n",
        "  def add_concept(self, concept, trial_size, threshold, messages=\"\"):\n",
        "    \"\"\"\n",
        "    Adds a concept to the list of concepts.\n",
        "\n",
        "    Args:\n",
        "    concept (dict): Concept to add.\n",
        "    trial_size (int): Size of the trial for evaluating the concept.\n",
        "    threshold (float): Threshold for accepting the concept.\n",
        "    messages (str, optional): Messages to display during the process. Defaults to \"\".\n",
        "\n",
        "    Returns:\n",
        "    bool: Whether the concept was added to the list.\n",
        "    \"\"\"\n",
        "    # The method first evaluates the concept on a trial set of examples.\n",
        "    # It then interacts with the user for feedback and adds the concept to the\n",
        "    # list if it is accepted. The method returns a boolean\n",
        "    # indicating whether the concept was added to the list.\n",
        "    userFeedback = 0\n",
        "    while userFeedback in [0,1,2]:\n",
        "      #---- Interactive elements ----\n",
        "      self.refresh_dashboard()\n",
        "      self.log(textwrap.fill(\"Generated Concept: \" + json.dumps(concept, indent=2), break_long_words=False,replace_whitespace=False, width=self.printWidth))\n",
        "      #---- Interactive elements ----\n",
        "      if self.automated:\n",
        "        concept = self.modify_concept_assistant(concept)\n",
        "        userInput = '0'\n",
        "      else:\n",
        "        userInput = input(\"Write 0 to evaluate concept, Write 1 to add this concept to the concepts list, Write 2 to modify, write anything else to reject it: \")\n",
        "        if userInput.isdigit():\n",
        "            userFeedback = int(userInput)\n",
        "        else:\n",
        "          userFeedback = -1\n",
        "      if userFeedback == 0:\n",
        "        concept_name = concept['Concept Name']\n",
        "\n",
        "\n",
        "\n",
        "        trial_df = self.train_df.sample(trial_size)\n",
        "        self.trial_df = trial_df\n",
        "        sampled_indices = trial_df.index\n",
        "        # Label the trial_df\n",
        "        self.label_dataset(trial_df, concept)\n",
        "        # Update the corresponding rows in train_df\n",
        "        concept_tree.train_df[concept_name] = np.nan\n",
        "        concept_tree.train_df[f'{concept_name}_label'] = np.nan\n",
        "        concept_tree.train_df[f'{concept_name}_snippets'] = np.nan\n",
        "        concept_tree.train_df[f'{concept_name}_thoughts'] = np.nan\n",
        "\n",
        "        self.train_df.loc[sampled_indices] = trial_df\n",
        "\n",
        "        self.visualize(trial_df, concept_name)\n",
        "\n",
        "        featureSet_1 = [c['Concept Name'] for c in self.concepts_list]\n",
        "        featureSet_2 = featureSet_1.copy()\n",
        "        featureSet_2.append(concept_name)\n",
        "\n",
        "        scores = []\n",
        "\n",
        "        self.log(\"=== Evaluating concept in trial ===\")\n",
        "        le = LabelEncoder()\n",
        "\n",
        "        for featureSet in [featureSet_1, featureSet_2]:\n",
        "          if len(featureSet) == 0:\n",
        "              if self.is_regression:\n",
        "                  mean_score = trial_df[self.label_col].mean()\n",
        "                  scores.append(np.mean((mean_score - trial_df[self.label_col]) ** 2))\n",
        "              else:\n",
        "                  most_common_label = trial_df[self.label_col].value_counts().idxmax()\n",
        "                  scores.append((trial_df[self.label_col] == most_common_label).mean())\n",
        "          else:\n",
        "              if self.is_regression:\n",
        "                  #model = xgb.XGBRegressor(n_estimators = 20, max_depth=3, min_samples_leaf=5, subsample=0.8, gamma=0.5)\n",
        "                  y_train = trial_df[self.label_col]\n",
        "                  model = LinearRegression()\n",
        "                  model.fit(trial_df[featureSet], y_train)\n",
        "                  preds = model.predict(trial_df[featureSet])\n",
        "                  scores.append(np.mean((preds - y_train) ** 2))\n",
        "              else:\n",
        "                  #model = xgb.XGBClassifier(n_estimators = 20, max_depth=3, min_samples_leaf=5, subsample=0.8, gamma=0.5)\n",
        "                  y_train = le.fit_transform(trial_df[self.label_col])\n",
        "                  model = LogisticRegression()\n",
        "                  model.fit(trial_df[featureSet], y_train)\n",
        "                  preds = model.predict(trial_df[featureSet])\n",
        "                  scores.append((preds == y_train).mean())\n",
        "        self.log(\"Score 1: {}\".format(scores[0]))\n",
        "        self.log(\"Score 2: {}\".format(scores[1]))\n",
        "\n",
        "        if self.is_regression:\n",
        "          delta = -(scores[1] - scores[0])/scores[0]\n",
        "        else:\n",
        "          delta = (scores[1] - scores[0])/scores[0]\n",
        "        self.log(\"delta: {}\".format(delta))\n",
        "\n",
        "        if self.automated:\n",
        "          if delta >= threshold:\n",
        "            userInput = '1'\n",
        "          else:\n",
        "            userInput = 'exit'\n",
        "            userFeedback = 'exit'\n",
        "        else:\n",
        "          userInput = input(\"Write 1 to add this concept to the concepts list, Write 2 to modify, write anything else to reject it: \")\n",
        "        if userInput.isdigit():\n",
        "          userFeedback = int(userInput)\n",
        "\n",
        "      if userFeedback == 1:\n",
        "        self.log(\"=== Accepted to concepts list ===\")\n",
        "        self.concepts_list.append(concept)\n",
        "        self.label_dataset(self.train_df, concept)\n",
        "        return True\n",
        "      elif userFeedback == 2:\n",
        "        concept = self.modify_concept_assistant(concept)\n",
        "\n",
        "\n",
        "    self.log(\"=== Rejected from concepts list ===\")\n",
        "    return False\n",
        "\n",
        "\n",
        "  def score_test_set(self):\n",
        "    \"\"\"\n",
        "    Scores the test set based on the current list of concepts.\n",
        "\n",
        "    Returns:\n",
        "    None.\n",
        "    \"\"\"\n",
        "    # The method first measures each concept for each example in the test set.\n",
        "    # It then calculates an overall score for the test set based on the concept scores.\n",
        "    featureSet = []\n",
        "    for concept in self.concepts_list:\n",
        "      concept_name = concept['Concept Name']\n",
        "      if concept_name not in self.test_df.columns:\n",
        "        self.label_dataset(self.test_df, concept)\n",
        "        featureSet.append(concept_name)\n",
        "        if self.is_regression:\n",
        "            #model = xgb.XGBRegressor(n_estimators=100, max_depth=3, min_samples_leaf=3, subsample=0.5, gamma=2)\n",
        "            model = LinearRegression()\n",
        "            model.fit(self.train_df[featureSet], self.train_df[self.label_col])\n",
        "            preds = model.predict(self.test_df[featureSet])\n",
        "            self.log(\"Current feature set score: {}\".format(np.mean((preds - self.test_df[self.label_col]) ** 2)))\n",
        "        else:\n",
        "            #model = xgb.XGBClassifier(n_estimators=100, max_depth=3, min_samples_leaf=3, subsample=0.5, gamma=2)\n",
        "            model = LogisticRegression()\n",
        "            model.fit(self.train_df[featureSet], self.train_df[self.label_col])\n",
        "            preds = model.predict(self.test_df[featureSet])\n",
        "            self.log(\"Current feature set accuracy: {}\".format((preds == self.test_df[self.label_col]).mean()))\n",
        "      else:\n",
        "        featureSet.append(concept_name)\n",
        "\n",
        "    if self.is_regression:\n",
        "      model = LinearRegression()\n",
        "    else:\n",
        "      model = LogisticRegression()\n",
        "    model.fit(self.train_df[featureSet], self.train_df[self.label_col])\n",
        "    featureSet = [c['Concept Name'] for c in self.concepts_list]\n",
        "    pred_test = model.predict(self.test_df[featureSet])\n",
        "    self.test_df['preds'] = preds\n",
        "\n",
        "    if self.is_regression:\n",
        "      mse = np.mean((pred_test - self.test_df[self.label_col]).pow(2))\n",
        "      self.log('test mse:',mse)\n",
        "    else:\n",
        "      acc = self.clf.score(self.test_df[featureSet], self.test_df[self.label_col])\n",
        "      self.log('test acc:',acc)\n",
        "\n",
        "\n",
        "  def iterate(self, trial_size, threshold, remaining_iters = -1, verbose=False):\n",
        "    \"\"\"\n",
        "    Performs one iteration of the concept generation and evaluation process.\n",
        "\n",
        "    Args:\n",
        "    trial_size (int): Size of the trial for evaluating the concepts.\n",
        "    threshold (float): Threshold for accepting the concepts.\n",
        "    verbose (bool): Whether to print verbose output.\n",
        "\n",
        "    Returns:\n",
        "    None.\n",
        "    \"\"\"\n",
        "    # The method first updates the classifier based on the current list of concepts.\n",
        "    # It then generates a new concept, improves it, evaluates it, and adds it to the\n",
        "    # list of concepts if it is useful. The method also scores the test set based on\n",
        "    # the final list of concepts and writes the results to files.\n",
        "\n",
        "    #update classifier\n",
        "    featureSet = [c['Concept Name'] for c in self.concepts_list]\n",
        "\n",
        "    if len(self.concepts_list) == 0:\n",
        "      samples = self.sample_randomly(20)\n",
        "    else:\n",
        "      if self.is_regression:\n",
        "        #self.clf = xgb.XGBRegressor(n_estimators = 25, max_depth=3, min_samples_leaf=5, subsample=0.8, gamma=0.5)\n",
        "        self.clf = LinearRegression()\n",
        "      else:\n",
        "        #self.clf = xgb.XGBClassifier(n_estimators = 25, max_depth=3, min_samples_leaf=5, subsample=0.8, gamma=0.5)\n",
        "        self.clf = LogisticRegression()\n",
        "\n",
        "      self.clf.fit(self.train_df[featureSet], self.train_df[self.label_col])\n",
        "      pred_train = self.clf.predict(self.train_df[featureSet])\n",
        "      samples = self.sample_high_error_neighbors(pred_train,20,10)\n",
        "\n",
        "    if not self.automated:\n",
        "      userFeedback = -1\n",
        "      while userFeedback not in [0,1]:\n",
        "        userInput = input(\"Write 0 to generate additional concept, Write 1 to finish and evaluate: \")\n",
        "        if userInput.isdigit():\n",
        "          userFeedback = int(userInput)\n",
        "      if userFeedback == 1:\n",
        "        self.display_results()\n",
        "        return\n",
        "      elif userFeedback == 0:\n",
        "        new_concept = self.generate_concept(samples, verbose)\n",
        "    else:\n",
        "      new_concept = self.generate_concept(samples, verbose)\n",
        "\n",
        "    if new_concept:\n",
        "      #Prevent name conflict\n",
        "      if new_concept['Concept Name'] in featureSet:\n",
        "        new_concept['Concept Name'] += str(len(featureSet))\n",
        "      #If concept added, add to internal clf\n",
        "      if self.add_concept(new_concept, trial_size, threshold):\n",
        "        featureSet = [c['Concept Name'] for c in self.concepts_list]\n",
        "        if self.is_regression:\n",
        "          #self.clf = xgb.XGBRegressor(n_estimators = 25, max_depth=3, min_samples_leaf=5, subsample=0.8, gamma=0.5)\n",
        "          self.clf = LinearRegression()\n",
        "        else:\n",
        "          #self.clf = xgb.XGBClassifier(n_estimators = 25, max_depth=3, min_samples_leaf=5, subsample=0.8, gamma=0.5)\n",
        "          self.clf = LogisticRegression()\n",
        "        self.clf.fit(self.train_df[featureSet], self.train_df[self.label_col])\n",
        "        pred_train = self.clf.predict(self.train_df[featureSet])\n",
        "        if self.is_regression:\n",
        "          score = np.mean((pred_train - self.train_df[self.label_col]).pow(2))\n",
        "        else:\n",
        "          score = self.clf.score(self.train_df[featureSet], self.train_df[self.label_col])\n",
        "        self.log(f\"Internal training set score: {score}\")\n",
        "\n",
        "    if self.automated:\n",
        "      remaining_iters = remaining_iters - 1\n",
        "      self.dashboard['remaining_iterations'] = remaining_iters\n",
        "      if remaining_iters >= 1:\n",
        "        self.iterate(trial_size, threshold, remaining_iters, verbose)\n",
        "\n",
        "\n",
        "  def visualize(self,df,concept):\n",
        "    \"\"\"\n",
        "    Visualizes the distribution of a concept in the data.\n",
        "\n",
        "    Args:\n",
        "    df (DataFrame): DataFrame containing the data.\n",
        "    concept (str): Concept to visualize.\n",
        "\n",
        "    Returns:\n",
        "    None.\n",
        "    \"\"\"\n",
        "    # The method first extracts examples of each unique value of the concept from\n",
        "    # the data. It then prints these examples to provide a visual representation\n",
        "    # of the distribution of the concept in the data.\n",
        "\n",
        "    value_examples = '\\n'\n",
        "    for i in df[concept].unique():\n",
        "      value_examples += '----Some examples of '+concept+' with value '+ str(i) + '----\\n'\n",
        "      value_df = df[df[concept]==i]\n",
        "      examples = value_df.sample(min(5,len(value_df)))\n",
        "      for ex in examples[self.text_col]:\n",
        "        value_examples += ex+'\\n'\n",
        "    self.log(value_examples)\n",
        "\n",
        "  def write_regression_results(self, train_features, test_features):\n",
        "    \"\"\"\n",
        "    Writes the results of the regression model to files.\n",
        "\n",
        "    Args:\n",
        "    train_features (DataFrame): Training features.\n",
        "    test_features (DataFrame): Testing features.\n",
        "\n",
        "    Returns:\n",
        "    None.\n",
        "    \"\"\"\n",
        "    # The method first trains a Linear Regression model and an XGBoost model on\n",
        "    # the training features. It then predicts the labels for the train and test data\n",
        "    # and calculates the Mean Squared Error for both train and test data. The method\n",
        "    # saves the accuracies to a CSV file and also saves the coefficients for Linear\n",
        "    # Regression and feature importances for XGBoost in one file.\n",
        "\n",
        "    # Get the 'label' columns for train and test data\n",
        "    train_labels = self.train_df[self.label_col]\n",
        "    test_labels = self.test_df[self.label_col]\n",
        "\n",
        "    # Create and train a Linear Regression model\n",
        "    model_lr = LinearRegression()\n",
        "    model_lr.fit(train_features, train_labels)\n",
        "\n",
        "    # Predict the labels for the train and test data\n",
        "    train_preds_lr = model_lr.predict(train_features)\n",
        "    test_preds_lr = model_lr.predict(test_features)\n",
        "\n",
        "    # Calculate the Mean Squared Error for both train and test data\n",
        "    train_mse_lr = mean_squared_error(train_labels, train_preds_lr)\n",
        "    test_mse_lr = mean_squared_error(test_labels, test_preds_lr)\n",
        "\n",
        "    # Train and test XGBoost model\n",
        "    model_xgb = xgb.XGBRegressor()\n",
        "    model_xgb.fit(train_features, train_labels)\n",
        "    train_preds_xgb = model_xgb.predict(train_features)\n",
        "    test_preds_xgb = model_xgb.predict(test_features)\n",
        "\n",
        "    # Calculate the Mean Squared Error for both train and test data\n",
        "    train_mse_xgb = mean_squared_error(train_labels, train_preds_xgb)\n",
        "    test_mse_xgb = mean_squared_error(test_labels, test_preds_xgb)\n",
        "\n",
        "    # Save accuracies to a CSV file\n",
        "    accuracies = pd.DataFrame({\n",
        "        'model': ['Linear Regression', 'XGBoost'],\n",
        "        'train_mse': [train_mse_lr, train_mse_xgb],\n",
        "        'test_mse': [test_mse_lr, test_mse_xgb]\n",
        "    })\n",
        "    accuracies.to_csv(f\"{self.name}_accuracy.csv\", index=False)\n",
        "\n",
        "    # Save the coefficients for Linear Regression and feature importances for XGBoost in one file\n",
        "    coefficients_lr = pd.DataFrame({'feature': train_features.columns, 'coefficient': model_lr.coef_, 'model': 'Linear Regression'})\n",
        "    importances_xgb = pd.DataFrame({'feature': train_features.columns, 'coefficient': model_xgb.feature_importances_, 'model': 'XGBoost'})\n",
        "    coefficients = pd.concat([coefficients_lr, importances_xgb], ignore_index=True)\n",
        "    coefficients.to_csv(f\"{self.name}_coefficients.csv\", index=False)\n",
        "\n",
        "  def write_classification_results(self, train_features, test_features):\n",
        "    \"\"\"\n",
        "    Writes the results of the classification model to files.\n",
        "\n",
        "    Args:\n",
        "    train_features (DataFrame): Training features.\n",
        "    test_features (DataFrame): Testing features.\n",
        "\n",
        "    Returns:\n",
        "    None.\n",
        "    \"\"\"\n",
        "    # The method first trains a Logistic Regression model and an XGBoost model\n",
        "    # on the training features. It then predicts the labels for the train and\n",
        "    # test data and calculates the accuracy for both train and test data.\n",
        "    # The method saves the accuracies to a CSV file and also saves the coefficients\n",
        "    # for Logistic Regression and feature importances for XGBoost in one file.\n",
        "\n",
        "\n",
        "    # Get the 'label' columns for train and test data\n",
        "    train_labels = self.train_df[self.label_col]\n",
        "    test_labels = self.test_df[self.label_col]\n",
        "\n",
        "    # Create and train a Logistic Regression model\n",
        "    model_lr = LogisticRegression()\n",
        "    model_lr.fit(train_features, train_labels)\n",
        "\n",
        "    # Predict the labels for the train and test data\n",
        "    train_preds_lr = model_lr.predict(train_features)\n",
        "    test_preds_lr = model_lr.predict(test_features)\n",
        "\n",
        "    # Calculate the accuracy for both train and test data\n",
        "    train_acc_lr = accuracy_score(train_labels, train_preds_lr)\n",
        "    test_acc_lr = accuracy_score(test_labels, test_preds_lr)\n",
        "\n",
        "    # Train and test XGBoost model\n",
        "    model_xgb = xgb.XGBClassifier()\n",
        "    model_xgb.fit(train_features, train_labels)\n",
        "    train_preds_xgb = model_xgb.predict(train_features)\n",
        "    test_preds_xgb = model_xgb.predict(test_features)\n",
        "\n",
        "    # Calculate the accuracy for both train and test data\n",
        "    train_acc_xgb = accuracy_score(train_labels, train_preds_xgb)\n",
        "    test_acc_xgb = accuracy_score(test_labels, test_preds_xgb)\n",
        "\n",
        "    # Save accuracies to a CSV file\n",
        "    accuracies = pd.DataFrame({\n",
        "        'model': ['Logistic Regression', 'XGBoost'],\n",
        "        'train_accuracy': [train_acc_lr, train_acc_xgb],\n",
        "        'test_accuracy': [test_acc_lr, test_acc_xgb]\n",
        "    })\n",
        "    accuracies.to_csv(f\"{self.name}_accuracy.csv\", index=False)\n",
        "\n",
        "    # Save the coefficients for Logistic Regression and feature importances for XGBoost in one file\n",
        "    if model_lr.coef_.shape[0] == 1 and len(model_lr.classes_) == 2:\n",
        "        coef_reshaped = np.vstack((model_lr.coef_, -model_lr.coef_))\n",
        "    else:\n",
        "        coef_reshaped = model_lr.coef_\n",
        "\n",
        "    coefficients_lr = pd.DataFrame(coef_reshaped, columns=train_features.columns)\n",
        "    coefficients_lr['class'] = model_lr.classes_\n",
        "    coefficients_lr = coefficients_lr.melt(id_vars=['class'], var_name='feature', value_name='coefficient')\n",
        "    coefficients_lr['model'] = 'Logistic Regression'\n",
        "\n",
        "    importances_xgb = pd.DataFrame({'feature': train_features.columns, 'coefficient': model_xgb.feature_importances_, 'model': 'XGBoost'})\n",
        "    coefficients = pd.concat([coefficients_lr, importances_xgb], ignore_index=True)\n",
        "    coefficients.to_csv(f\"{self.name}_coefficients.csv\", index=False)\n",
        "\n",
        "\n",
        "  def write_results(self):\n",
        "    \"\"\"\n",
        "    Writes the results of the concept generation and evaluation process to files.\n",
        "\n",
        "    Returns:\n",
        "    None.\n",
        "    \"\"\"\n",
        "    # The method first saves the training and testing data, the list of concepts,\n",
        "    # and the scores to CSV files. It then calculates the correlation matrix for\n",
        "    # the concept scores and saves it to a CSV file. The method also generates a\n",
        "    # heatmap plot of the correlation matrix and saves it to a PNG file.\n",
        "\n",
        "    # Save training and testing dataframes\n",
        "    self.train_df.to_csv(f\"{self.name}_train_df.csv\", index=False)\n",
        "    self.test_df.to_csv(f\"{self.name}_test_df.csv\", index=False)\n",
        "\n",
        "    # Save the concepts list\n",
        "    concepts_df = pd.DataFrame(self.concepts_list)\n",
        "    concepts_df.to_csv(f\"{self.name}_concepts.csv\", index=False)\n",
        "    unfiltered_concepts_df = pd.DataFrame(self.concepts_list_unfiltered)\n",
        "    unfiltered_concepts_df.to_csv(f\"{self.name}_concepts_unfiltered.csv\", index=False)\n",
        "\n",
        "    # Prepare train and test data\n",
        "    concept_features = [c['Concept Name'] for c in self.concepts_list]\n",
        "    train_features = self.train_df[concept_features]\n",
        "    test_features = self.test_df[concept_features]\n",
        "\n",
        "    # Calculate the correlation matrix\n",
        "    corr_matrix = test_features.corr()\n",
        "\n",
        "    # Save correlations between the concept scores\n",
        "    corr_matrix.to_csv(f\"{self.name}_correlations.csv\")\n",
        "\n",
        "    # Set up a figure and axes for the plot\n",
        "    fig, ax = plt.subplots(figsize=(10, 10))\n",
        "\n",
        "    # Create a custom colormap\n",
        "    cmap = sns.diverging_palette(220, 20, as_cmap=True)\n",
        "\n",
        "    # Create the heatmap plot\n",
        "    sns.heatmap(corr_matrix, cmap=cmap, annot=True, fmt=\".2f\", linewidths=.5, ax=ax)\n",
        "\n",
        "    # Set the title for the plot\n",
        "    ax.set_title(\"Correlation Matrix Heatmap\")\n",
        "\n",
        "    # Save the correlation plot\n",
        "    plt.savefig(f\"{self.name}_correlation_plot.png\")\n",
        "\n",
        "    # Show the plot\n",
        "    plt.show()\n",
        "\n",
        "    # Distribution of concept scores\n",
        "    train_features.hist(figsize=(15, 15))\n",
        "    plt.savefig(f\"{self.name}_distribution_of_concept_scores.png\")\n",
        "    plt.show()\n",
        "\n",
        "    if self.is_regression:\n",
        "        self.write_regression_results(train_features, test_features)\n",
        "    else:\n",
        "        self.write_classification_results(train_features, test_features)\n",
        "\n",
        "  def train(self, n_iters, trial_size, threshold, verbose):\n",
        "    \"\"\"\n",
        "    Trains the concept generation and evaluation process.\n",
        "\n",
        "    Args:\n",
        "    n_iters (int): Number of iterations to perform.\n",
        "    trial_size (int): Size of the trial for evaluating the concepts.\n",
        "    threshold (float): Threshold for accepting the concepts.\n",
        "    verbose (bool): Whether to print verbose output.\n",
        "\n",
        "    Returns:\n",
        "    None.\n",
        "    \"\"\"\n",
        "    # The method first sets up the concept generation and evaluation process.\n",
        "    # It then performs a specified number of iterations, with each iteration\n",
        "    # involving concept generation, improvement, and evaluation. After all\n",
        "    # iterations are complete, the method scores the test set based on the\n",
        "    # final list of concepts and writes the results to files.\n",
        "\n",
        "    input_type = input(\"Write 0 for interactive mode, Write 1 for automatic mode: \")\n",
        "    if input_type.isdigit():\n",
        "      input_type = int(input_type)\n",
        "\n",
        "    if input_type == 1:\n",
        "      self.automated = True\n",
        "      self.dashboard['remaining_iterations'] = n_iters\n",
        "      self.iterate(trial_size, threshold, n_iters, verbose)\n",
        "      self.display_results()\n",
        "    else:\n",
        "      self.automated = False\n",
        "      self.iterate(trial_size, threshold, -1, verbose)\n",
        "\n",
        "  def display_results(self):\n",
        "    \"\"\"\n",
        "    Displays the results of the concept generation and evaluation process.\n",
        "\n",
        "    Returns:\n",
        "    None.\n",
        "    \"\"\"\n",
        "    # The method first scores the test set based on the final list of concepts.\n",
        "    # It then writes the results to files and displays the results in a graphical format.\n",
        "    self.score_test_set()\n",
        "    self.write_results()\n",
        "\n",
        "    results = {}\n",
        "    results['test'] = pd.read_csv('{}_test_df.csv'.format(self.name))\n",
        "    results['train'] = pd.read_csv('{}_train_df.csv'.format(self.name))\n",
        "    results['concepts'] = pd.read_csv('{}_concepts.csv'.format(self.name))\n",
        "    results['correlations'] = pd.read_csv('{}_correlations.csv'.format(self.name))\n",
        "    results['coefficients'] = pd.read_csv('{}_coefficients.csv'.format(self.name))\n",
        "    results['accuracy'] = pd.read_csv('{}_accuracy.csv'.format(self.name))\n",
        "\n",
        "    display(results['test'].sample(5))\n",
        "    display(results['concepts'].head(10))\n",
        "    display(results['correlations'].head(10))\n",
        "    display(results['coefficients'].tail(20))\n",
        "    display(results['accuracy'].head(10))\n",
        "\n",
        "    return results\n",
        "\n",
        "  def extract_output_list_batched(self, responseText, batch_size):\n",
        "    \"\"\"\n",
        "    Extracts the output list from the response of the OpenAI API for multiple texts.\n",
        "\n",
        "    Args:\n",
        "    responseText (str): Response from the OpenAI API.\n",
        "    batch_size (int): Number of texts in the batch.\n",
        "\n",
        "    Returns:\n",
        "    list: List of extracted outputs for each text in the batch.\n",
        "    \"\"\"\n",
        "    # The method first cleans the response text and parses it into a list of outputs. It then returns the list of outputs.\n",
        "\n",
        "    try:\n",
        "        extracted = []\n",
        "        if self.generation_format == 'completion':\n",
        "          responseText = \"[\" + responseText\n",
        "        cleanedResponse = responseText.replace(\"\\n\", \"\").replace(\"“\", \"\\\"\").replace(\"”\", \"\\\"\").replace(\"],}\", \"]}\")\n",
        "        for response in json.loads(cleanedResponse):\n",
        "            extracted.append(response[1])\n",
        "        if len(extracted) != batch_size:\n",
        "            return [\"Invalid output generated: Invalid size\"]*batch_size\n",
        "        return extracted\n",
        "    except json.JSONDecodeError:\n",
        "        return [\"Invalid output generated: Bad generation\"]*batch_size\n",
        "\n",
        "\n",
        "  def modify_concept_assistant(self, concept):\n",
        "    \"\"\"\n",
        "    Modifies a concept based on user feedback.\n",
        "\n",
        "    Args:\n",
        "    concept (dict): Concept to modify.\n",
        "\n",
        "    Returns:\n",
        "    new_concept (dict): Modified concept.\n",
        "    \"\"\"\n",
        "    # The method first interacts with the user to get feedback on the concept.\n",
        "    # It then uses the OpenAI API to suggest improvements to the concept based\n",
        "    # on the user feedback. The improved concept is returned as a dictionary.\n",
        "    instructionPrompt = lambda x: f\"\"\"Concept Improvement Task\n",
        "\n",
        "We have a concept that needs to be improved. The goal of this task is to identify any issues with the current concept and suggest improvements to make it more valid, clear, well-phrased, and properly formatted in JSON.\n",
        "In this task, we will return information about any potential problems in the concept along with the improved concept.\n",
        "Note it is also possible that the concept requires no further improvement (even minor ones), in which case, we will return the original concept with \"None\" for the other responses.\n",
        "\n",
        "Consider the following error cases while improving the concept:\n",
        "\n",
        "1. Lack of validity: Ensure that the responses are mutually exclusive and collectively exhaustive.\n",
        "- Example of a concept that is not mutually exclusive:\n",
        "Input JSON:\n",
        "{{\n",
        "\"Concept Name\": \"review sentiment\",\n",
        "\"Concept Description\": \"The sentiment expressed towards the product in the review. It could be positive, negative, or neutral.\",\n",
        "\"Concept Question\": \"What is the overall feeling towards the product?\",\n",
        "\"Possible Responses\": [\"positive\", \"somewhat positive\", \"negative\"],\n",
        "\"Response Guide\": {{\n",
        "\"positive\": \"The reviewer expresses a positive opinion on the product, such as praising its quality, performance, or value.\",\n",
        "\"somewhat positive\": \"The reviewer expresses a somewhat positive opinion on the product, such as mentioning some good aspects but also pointing out some flaws.\",\n",
        "\"negative\": \"The reviewer expresses a negative opinion on the product, such as criticizing its quality, performance, or value.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"positive\": 1,\n",
        "\"somewhat positive\": 0.5,\n",
        "\"negative\": -1\n",
        "}}\n",
        "}}\n",
        "User Feedback: The concept above contains non-mutually exclusive responses 'positive' and 'somewhat positive\n",
        "Response: {{\n",
        "\"Errors\": \"The concept above contains non-mutually exclusive responses 'positive' and 'somewhat positive'\",\n",
        "\"Fix\": \"We can address this by either combining 'positive' and 'somewhat positive' into a single response or defining clearer distinctions between them.\",\n",
        "\"New Concept\": {{\n",
        "\"Concept Name\": \"review sentiment\",\n",
        "\"Concept Description\": \"The sentiment expressed towards the product in the review. It could be positive, negative, or neutral.\",\n",
        "\"Concept Question\": \"What is the overall sentiment expressed towards the product in the review?\",\n",
        "\"Possible Responses\": [\"positive\", \"negative\", \"neutral\"],\n",
        "\"Response Guide\": {{\n",
        "\"positive\": \"The reviewer expresses a positive opinion on the product, such as praising its quality, performance, or value.\",\n",
        "\"negative\": \"The reviewer expresses a negative opinion on the product, such as criticizing its quality, performance, or value.\",\n",
        "\"neutral\": \"The reviewer does not express a clear positive or negative opinion on the product, or the review contains a mix of positive and negative aspects.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"positive\": 1,\n",
        "\"negative\": -1,\n",
        "\"neutral\": 0\n",
        "}}\n",
        "}}\n",
        "}}###\n",
        "\n",
        "- Example of a concept that is not collectively exhaustive:\n",
        "Input JSON:\n",
        "{{\n",
        "\"Concept Name\": \"product availability\",\n",
        "\"Concept Description\": \"The availability of the product as described in the review.\",\n",
        "\"Concept Question\": \"Is the product available?\",\n",
        "\"Possible Responses\": [\"available\", \"unavailable\"],\n",
        "\"Response Guide\": {{\n",
        "\"available\": \"The reviewer mentions that the product is available, in stock, or easy to find.\",\n",
        "\"unavailable\": \"The reviewer mentions that the product is unavailable, out of stock, or hard to find.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"available\": 1,\n",
        "\"unavailable\": -1\n",
        "}}\n",
        "}}\n",
        "User Feedback: The concept above contains a non-collectively exhaustive response set because it may be possible that a piece of text does not strictly match some criteria in the response guide, such as the availability of the product.\n",
        "Response: {{\n",
        "\"Errors\": \"The concept above contains a non-collectively exhaustive response set because it may be possible that a piece of text does not strictly match some criteria in the response guide,\",\n",
        "\"Fix\": \"We can address this by adding a 'uncertain' response to cover cases where the availability is not clearly mentioned, and a 'not applicable' response for cases where the text does not discuss a product.\",\n",
        "\"New Concept\": {{\n",
        "\"Concept Name\": \"product availability\",\n",
        "\"Concept Description\": \"The availability of the product as described in the review.\",\n",
        "\"Concept Question\": \"What does the review say about the product's availability?\",\n",
        "\"Possible Responses\": [\"available\", \"unavailable\", \"uncertain\", \"not applicable\"],\n",
        "\"Response Guide\": {{\n",
        "\"available\": \"The reviewer mentions that the product is available, in stock, or easy to find.\",\n",
        "\"unavailable\": \"The reviewer mentions that the product is unavailable, out of stock, or hard to find.\",\n",
        "\"uncertain\": \"The review contains mixed information or no information that makes it difficult to determine the availability of the product.\",\n",
        "\"not applicable\": \"The reviewer is not discussing a product or anything else that could be described by this concept.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"available\": 1,\n",
        "\"unavailable\": -1,\n",
        "\"uncertain\": 0,\n",
        "\"not applicable\": 0\n",
        "}}\n",
        "}}\n",
        "}}###\n",
        "\n",
        "2. Poor phrasing: Avoid leading questions and provide rich examples in the response guide.\n",
        "- Example of a leading question:\n",
        "Input JSON:\n",
        "{{\n",
        "\"Concept Name\": \"product cost\",\n",
        "\"Concept Description\": \"The cost of the product as described in the review.\",\n",
        "\"Concept Question\": \"Is the product expensive?\",\n",
        "\"Possible Responses\": [\"expensive\", \"affordable\", \"uncertain\", \"not applicable\"],\n",
        "\"Response Guide\": {{\n",
        "\"expensive\": \"The reviewer thinks the product is expensive.\",\n",
        "\"affordable\": \"The reviewer thinks the product is affordable.\",\n",
        "\"uncertain\": \"The reviewer does not provide a clear opinion on the product's cost.\",\n",
        "\"not applicable\": \"The reviewer is not discussing a product or anything else that could be described by this concept.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"expensive\": 1,\n",
        "\"affordable\": -1,\n",
        "\"uncertain\": 0,\n",
        "\"not applicable\": 0\n",
        "}}\n",
        "}}\n",
        "User Feedback: The concept above contains a leading question and a bad response guide. To fix this, we can simply add more items to the response set.\n",
        "Response: {{\n",
        "\"Errors\": \"The concept contains a leading question and a bad response guide\",\n",
        "\"Fix\": \"We can address this by changing the question to be more neutral and asking about the reviewer's description of the product's cost.\",\n",
        "\"New Concept\": {{\n",
        "\"Concept Name\": \"product cost\",\n",
        "\"Concept Description\": \"The cost of the product as described in the review, in terms of whether the product is perceived as expensive or affordable.\",\n",
        "\"Concept Question\": \"How does the reviewer describe the cost of the product?\",\n",
        "\"Possible Responses\": [\"expensive\", \"affordable\", \"uncertain\", \"not applicable\"],\n",
        "\"Response Guide\": {{\n",
        "\"expensive\": \"The reviewer describes the product as costly, high-priced, or not worth the money.\",\n",
        "\"affordable\": \"The reviewer describes the product as reasonably priced, good value for money, or budget-friendly.\",\n",
        "\"uncertain\": \"The reviewer does not provide a clear opinion or information on the product's cost.\",\n",
        "\"not applicable\": \"The reviewer is not discussing a product or anything else that could be described by this concept.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"expensive\": 1,\n",
        "\"affordable\": -1,\n",
        "\"uncertain\": 0,\n",
        "\"not applicable\": 0\n",
        "}}\n",
        "}}\n",
        "}}###\n",
        "\n",
        "In addition to the errors above, some other problems could be:\n",
        "\n",
        "1. Lack of detail in the response guide. As much as possible, the response guide should contain detailed examples. This issue can be fixed by making the response guide more specific to allow annotators to be more objective about answering the question.\n",
        "2. All responses should map to integers. They should also be sorted so that the ordering is meaningful. Related responses should be close to each other when mapped to integers.\n",
        "3. Too many categories\n",
        "\n",
        "---\n",
        "Try to make the concept as helpful as possible for prediction task in the dataset, Keep in mind the label guide and the dataset description:\n",
        "{self.label_guide}\n",
        "{self.description}\n",
        "---\n",
        "Below is the concept for you to improve. Complete the response.\n",
        "{concept}\n",
        "User Feedback: {x}\n",
        "Response:\n",
        "\"\"\"\n",
        "\n",
        "    if self.automated:\n",
        "      userComment = \"Repair any potential issues if any are detected. Do not repair any issues related to mutual exclusivity/collective exhaustion. Do not change the response set.\"\n",
        "    else:\n",
        "      userComment = input(\"Write down notes for modifications to concept:\")\n",
        "    userPrompt = instructionPrompt(userComment)\n",
        "    self.log(userComment)\n",
        "\n",
        "    try:\n",
        "      response = completion(self.generation_model,\n",
        "                              1,\n",
        "                              userPrompt,\n",
        "                              maxtokens=2000,\n",
        "                              format=self.generation_format)\n",
        "      cleanedResponse = response.replace(\"\\n\",\"\").replace(\"“\",\"\\\"\").replace(\"”\",\"\\\"\")\n",
        "      self.log(cleanedResponse)\n",
        "\n",
        "      responseJson = json.loads(cleanedResponse)\n",
        "\n",
        "      for item in ['Errors', 'Fix','New Concept']:\n",
        "          if item not in responseJson.keys():\n",
        "            self.log(\"=== JSONDecodeError in response json . Retrying===\")\n",
        "            return self.modify_concept_assistant(concept)\n",
        "\n",
        "      problem = responseJson['Errors']\n",
        "      fix = responseJson['Fix']\n",
        "      new_concept = responseJson['New Concept']\n",
        "\n",
        "      self.log(\"=== System Response ===\")\n",
        "      self.log(fix)\n",
        "      self.log(json.dumps(new_concept, indent=2))\n",
        "\n",
        "      for item in ['Concept Name', 'Concept Description','Concept Question', 'Possible Responses', 'Response Guide', \"Response Mapping\"]:\n",
        "          if item not in new_concept.keys():\n",
        "            self.log(\"=== JSONDecodeError in concept modification. Retrying ===\")\n",
        "            return self.modify_concept_assistant(concept)\n",
        "\n",
        "      return new_concept\n",
        "    except json.JSONDecodeError:\n",
        "      self.log(\"=== JSONDecodeError in concept modification. Retrying ===\")\n",
        "      return self.modify_concept_assistant(concept)\n",
        "\n",
        "\n",
        "  def recommend_concept_assistant(self, texts, labels, verbose=True):\n",
        "    \"\"\"\n",
        "    Recommends a concept based on the given texts and labels.\n",
        "\n",
        "    Args:\n",
        "    texts (list): List of text examples.\n",
        "    labels (list): Corresponding labels for the text examples.\n",
        "    verbose (bool, optional): Whether to print verbose output. Defaults to True.\n",
        "\n",
        "    Returns:\n",
        "    new_concept (dict): Recommended concept.\n",
        "    \"\"\"\n",
        "    # The method first formats the examples and labels into a string, and then\n",
        "    # uses the OpenAI API to generate a concept based on the examples and labels.\n",
        "    # The generated concept is returned as a dictionary.\n",
        "\n",
        "    examples = [(text, self.label_guide[str(label)]) for text,label in zip(texts, labels)]\n",
        "    examples = sorted(examples, key=lambda x: x[1])\n",
        "    formatted_str = [\"text:{}\\nrating: {}\".format(x,y) for x,y in examples]\n",
        "    focus_str = f\"Focus on concepts that help you distinguish between the following labels from the examples above. The concept should be useful for identifying texts with the following label: {choice([y for x,y in examples])}, you are encouraged to extract keywords and inspiration from the example above.\"\n",
        "    examples_string = \"\\n\\n\".join(formatted_str)\n",
        "\n",
        "    rejected_concepts = [x for x in concept_tree.concepts_list_unfiltered if\n",
        "                             x['Concept Name'] not in [y['Concept Name'] for\n",
        "                             y in concept_tree.concepts_list]]\n",
        "\n",
        "    rejected_concepts_str = ''\n",
        "    current_concepts_str = ''\n",
        "    for i in range(len(self.concepts_list)):\n",
        "      current_concepts_str += str(i+1)+'. '+self.concepts_list[i]['Concept Name'] + \":\" +self.concepts_list[i]['Concept Description']+\", possible responses: \"+str(self.concepts_list[i]['Possible Responses'])+'\\n'\n",
        "    for i in range(len(rejected_concepts)):\n",
        "      rejected_concepts_str += str(i+1)+'. '+rejected_concepts[i]['Concept Name'] + \":\" +rejected_concepts[i]['Concept Description']+\", possible responses: \"+str(rejected_concepts[i]['Possible Responses'])+'\\n'\n",
        "\n",
        "    if verbose:\n",
        "      self.log(f\"Description: {self.description}\")\n",
        "      self.log(f\"Label Guide: {self.label_guide}\")\n",
        "      self.log(f\"Examples: {examples_string}\")\n",
        "      self.log(f\"Rejected Concepts: {rejected_concepts_str}\")\n",
        "      self.log(f\"Current Concepts: {current_concepts_str}\")\n",
        "\n",
        "    prompt = lambda x: f\"\"\"Concept Feature Engineering Task\n",
        "\n",
        "Below we are given a text dataset with accompanying labels. Our task is to identify a concept in the text that could be associated with the label. This is because we want to find the main factors that can be used to explain the label.\n",
        "\n",
        "To do this, we will examine a sample of texts that have different labels so that we can look at the different characteristics that exist for one label and compare it to another. Good concepts are those that separate texts with one label from another.\n",
        "\n",
        "After looking at these texts and finding a difference, we will define a concept definition JSON\n",
        "Each full concept definition comes with a concept name, description, question, response set, and response guide. The concept description provides an intuitive overview of the concept. The concept question is our tool for measuring the concept, this will be graded by a human annotator. The possible responses list the possible responses to the question and the response guide provides information on what each rating means. We also include a response mapping to help with data processing.\n",
        "\n",
        "Below are some examples of concepts for different datasets.\n",
        "\n",
        "1. A possible concept for a dataset assigning toxicity scores to social media texts\n",
        "{{\"Concept Name\": \"explicit language\",\n",
        "\"Concept Description\": \"'Explicit language' refers to the use of words, phrases, or expressions that are offensive, vulgar, or inappropriate for general audiences. This may include profanity, obscenities, slurs, sexually explicit or lewd language, and derogatory or discriminatory terms targeted at specific groups or individuals.\",\n",
        "\"Concept Question\": \"What is the nature of the language used in the text?\",\n",
        "\"Possible Responses\": [\"explicit\", \"strong\",\"non-explicit\", \"uncertain\"],\n",
        "\"Response Guide\": {{\n",
        "\"explicit\": \"The text contains explicit language, such as profanity, obscenities, slurs, sexually explicit or lewd language, or derogatory terms targeted at specific groups or individuals.\",\n",
        "\"strong\": \"The text contains strong language but not explicit language, it may contain terms that some viewers might find mature.\",\n",
        "\"non-explicit\": \"The text is free from explicit language and is appropriate for general audiences.\",\n",
        "\"uncertain\": \"It is difficult to determine the nature of the language used in the text or if any explicit terms are used.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"explicit\": 2,\n",
        "\"strong\":1,\n",
        "\"non-explicit\": -1,\n",
        "\"uncertain\": 0\n",
        "}}\n",
        "}}###\n",
        "\n",
        "2. A possible concept for evaluating the sentiment of product reviews on ecommerce site\n",
        "{{\"Concept Name\": \"good build quality\",\n",
        "\"Concept Description\": \"Build quality refers to the craftsmanship, durability, and overall construction of a product. It encompasses aspects such as materials used, design, manufacturing techniques, and attention to detail. A product with good build quality is typically considered to be well-made, sturdy, and long-lasting, while a product with poor build quality may be prone to defects or wear out quickly.\",\n",
        "\"Concept Question\": \"What does the review say about the build quality of the product?\",\n",
        "\"Possible Responses\": [\"positive\", \"negative\", \"uncertain\", \"not applicable\"],\n",
        "\"Response Guide\": {{\n",
        "\"high\": \"Review mentions aspects such as well-made, sturdy, durable, high-quality materials, excellent craftsmanship, etc.\",\n",
        "\"low\": \"Review mentions aspects such as poor construction, flimsy, cheap materials, bad design, easily breakable, etc.\",\n",
        "\"uncertain\": \"Review does not mention build quality, the information is ambiguous or vague, or it has both positive and negative aspects mentioned like 'the product is sturdy but uses cheap materials'.\",\n",
        "\"not applicable\": \"The review does not mention the build quality of the product at all.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"high\": 1,\n",
        "\"low\": -1,\n",
        "\"uncertain\": 0,\n",
        "\"not applicable\": 0\n",
        "}}\n",
        "}}###\n",
        "\n",
        "3. A useful concept for scam detection for emails\n",
        "{{\n",
        "\"Concept Name\": \"Extremely generous offer\",\n",
        "\"Concept Description\": \"The concept 'Extremely generous offer' refers to situations where the text describes an offer that seems too good to be true, such as promises of large financial gains, disproportionate rewards, or substantial benefits with seemingly little to no risk or effort required. These can often be indicative of scams or deceptive practices.\",\n",
        "\"Concept Question\": \"What type of offer is described in the text?\",\n",
        "\"Possible Responses\": [\"extremely generous offer\", \"ordinary offer\", \"no offer\", \"uncertain\"],\n",
        "\"Response Guide\": {{\n",
        "\"extremely generous offer\": \"The text describes an offer that is disproportionately rewarding or beneficial with seemingly little to no risk or effort. This could include promises of large financial returns with minimal investment, 'free' gifts that require payment information, or rewards that are disproportionate to the effort required.\",\n",
        "\"ordinary offer\": \"The text describes a typical or ordinary offer. For instance, normal sales or discounts, standard business offerings, or fair trades.\",\n",
        "\"no offer\": \"The text does not describe any offer.\",\n",
        "\"uncertain\": \"It is difficult to determine the type of offer described in the text. The text might be vague, ambiguous, or lack sufficient context.\"\n",
        "}},\n",
        "\"Response Mapping\": {{\n",
        "\"extremely generous offer\": 1,\n",
        "\"ordinary offer\": -2,\n",
        "\"no offer\": -1,\n",
        "\"uncertain\": 0\n",
        "}}\n",
        "}}\n",
        "\n",
        "---\n",
        "\n",
        "In the task, we will generate concepts for the {self.name} dataset\n",
        "\n",
        "Below is an explanation of the dataset and the labels therein:\n",
        "\n",
        "Description: {self.description}\n",
        "\n",
        "Label Guide: {self.label_guide}\n",
        "\n",
        "Below are some example texts along with their labels. These are examples that the model has a hard time classifying.\n",
        "---\n",
        "{examples_string}\n",
        "---\n",
        "As a reminder we already have the following concepts which are useful:\n",
        "{current_concepts_str[:3000]}\n",
        "The following concepts have been rejected by the system, avoid making similar ones:\n",
        "{rejected_concepts_str[:3000]}\n",
        "\n",
        "Keeping in mind the pointers above, create a concept below that is distinct from the current set of concepts. Additionally, make sure that all possible responses can be mapped to an integer.\n",
        "Make sure that the concept is as relevant to the labels in the {self.name} dataset. As the examples shown start to look more similar, we can start being more specific, picking out particular details tied to the label we notice in the examples above.\n",
        "Below is the description of the dataset.\n",
        "Description: {self.description}\n",
        "Below are some additional notes to pay attention to during concept generation\n",
        "Additional Notes: {x}\n",
        "Use strict adherence to the json format. Do not create duplicate concepts.\n",
        "Definition:\"\"\"\n",
        "\n",
        "    newMessages = []\n",
        "    #---- Interactive Elements ----\n",
        "    self.refresh_dashboard()\n",
        "    self.log(\"==== Concept Generation Prompt ====\")\n",
        "    self.log(textwrap.fill(prompt(\"[ Guidance Is inserted here ]\"), break_long_words=False,replace_whitespace=False, width=self.printWidth))\n",
        "    #---- Interactive Elements ----\n",
        "    userComment = input(\"Write down some guidance notes for concept generation:\")\n",
        "    userPrompt = prompt(userComment)\n",
        "    self.log(userComment)\n",
        "\n",
        "    response = completion(self.generation_model,\n",
        "                            1,\n",
        "                            userPrompt,\n",
        "                            maxtokens=2000,\n",
        "                            format=\"chat\")\n",
        "\n",
        "    cleanedResponse = response.replace(\"\\n\",\"\").replace(\"“\",\"\\\"\").replace(\"”\",\"\\\"\")\n",
        "    self.log(cleanedResponse)\n",
        "\n",
        "    new_concept = json.loads(cleanedResponse)\n",
        "\n",
        "    self.log(\"=== System Response ===\")\n",
        "    self.log(json.dumps(new_concept, indent=2))\n",
        "\n",
        "    return response\n",
        "\n",
        "  def refresh_dashboard(self):\n",
        "    clear_output()\n",
        "    self.log(\"===== Concept Bottleck Model =====\")\n",
        "    self.log(f\"Current Concepts: {[x['Concept Name'] for x in self.concepts_list]}\")\n",
        "    self.log(f\"Total Generated: {len(self.concepts_list_unfiltered)}\")\n",
        "    self.log(f\"Total Accepted: {len(self.concepts_list)}\")\n",
        "    self.log(f\"Remaining iterations: {self.dashboard['remaining_iterations']}\")\n",
        "    self.log(\"==================================\")\n",
        "\n",
        "  def log(self, text, tags=(), verbose=True):\n",
        "    timeStr = str(datetime.datetime.now(datetime.timezone(\n",
        "        datetime.timedelta(hours=-5))))\n",
        "\n",
        "    self.logs.append((timeStr, tags, text))\n",
        "    if verbose:\n",
        "      print(text)"
      ],
      "metadata": {
        "id": "HamTvN57oNJG"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluation"
      ],
      "metadata": {
        "id": "HPEQ-80eHpx9"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import Data"
      ],
      "metadata": {
        "id": "TsmDaGGuHt0I"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title\n",
        "!pip install datasets"
      ],
      "metadata": {
        "id": "gtOoATJqWUKY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8c98837c-23e4-4455-9c09-a20bcd744022"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-2.14.6-py3-none-any.whl (493 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m493.7/493.7 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.23.5)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
            "Collecting dill<0.3.8,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.1)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Collecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.8.6)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.17.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (3.3.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.12.4)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.5.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.7.22)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Installing collected packages: dill, multiprocess, datasets\n",
            "Successfully installed datasets-2.14.6 dill-0.3.7 multiprocess-0.70.15\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "configsDict = {}"
      ],
      "metadata": {
        "id": "a65Pmg6pJQlc"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title\n",
        "from datasets import load_dataset\n",
        "from sklearn.model_selection import train_test_split\n",
        "import re\n",
        "\n",
        "# ======= Load Cebab =======\n",
        "cebab = load_dataset(\"CEBaB/CEBaB\")\n",
        "\n",
        "cebab_train = pd.DataFrame(cebab['train_exclusive'])[[\n",
        "    'description','review_majority']].rename(columns={\n",
        "    'description': 'text',\n",
        "    'review_majority': 'label'\n",
        "})\n",
        "cebab_train = cebab_train[cebab_train['label'].apply(lambda x: x.isdigit())]\n",
        "cebab_train.to_csv('cebab_train.csv', index=False)\n",
        "\n",
        "cebab_test = pd.DataFrame(cebab['test'])[['description','review_majority']].rename(columns={\n",
        "    'description': 'text',\n",
        "    'review_majority': 'label'\n",
        "})\n",
        "cebab_test = cebab_test[cebab_test['label'].apply(lambda x: x.isdigit())]\n",
        "cebab_test.to_csv('cebab_test.csv', index=False)\n",
        "\n",
        "\n",
        "cebabConfig = {}\n",
        "cebabConfig['train'] = \"cebab_train.csv\"\n",
        "cebabConfig['test'] = \"cebab_test.csv\"\n",
        "cebabConfig['type'] = \"Regression\"\n",
        "cebabConfig['description'] = 'Restauraunt reviews from opentable'\n",
        "cebabConfig['class names'] = {\"1\":\"1 Star\",\n",
        "                              \"2\":\"2 Stars\",\n",
        "                              \"3\":\"3 Stars\",\n",
        "                              \"4\":\"4 Stars\",\n",
        "                              \"5\":\"5 Stars\"}\n",
        "configsDict['cebab'] = cebabConfig"
      ],
      "metadata": {
        "id": "bAj_tc0nR3od"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run Evaluation"
      ],
      "metadata": {
        "id": "bvWaahfvHw_q"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Run Specific Eval\n",
        "\n",
        "This cell train a TBM given a dataset that exists in the \"configsDict\" dictionary. As an example for how to load an item into this dictionary, see the cebab example above. The system currently outputs 4 items after training:\n",
        "- datasetName_test_df.csv - The testing dataset containing all the concept annotations\n",
        "- datasetName_train_df.csv - The training dataset containing all the concept annotations\n",
        "- datasetName_concepts.csv - A df of all the concepts included in the bottleneck\n",
        "- datasetName_concepts_unfiltered.csv - A df of all the concepts generated, including concepts that have not been included.\n",
        "\n",
        "Some important parameters for this section include\n",
        "- TRAIN_SIZE_LIMIT - Determines size of the training set for the TBM\n",
        "- TEST_SIZE_LIMIT  - Determines size of the test set for the TBM\n",
        "- EXAMPLE_LENGTH_LIMIT  - Determines max character length of examples in prompts\n",
        "- TRIAL_SIZE - Determines size of training set subset to be used during concept performance testing\n",
        "- N_ITERS - Number of iterations to perform\n",
        "- THRESHOLD - Relative performance increase threshold for including a concept in the bottleneck"
      ],
      "metadata": {
        "id": "5O5ewM3GH3Gj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title\n",
        "import os\n",
        "import json\n",
        "\n",
        "#EVAL PARAMETERS\n",
        "CONFIG_NAME = 'cebab'\n",
        "TRAIN_SIZE_LIMIT = 50\n",
        "TEST_SIZE_LIMIT = 50\n",
        "EXAMPLE_LENGTH_LIMIT = 500\n",
        "TRIAL_SIZE = 50\n",
        "N_ITERS = 5\n",
        "THRESHOLD = 0.01\n",
        "VERBOSE = True\n",
        "\n",
        "modelChoices = {}\n",
        "modelChoices['DEFAULT_GENERATION_MODEL'] = \"gpt-3.5-turbo\"\n",
        "modelChoices['DEFAULT_GENERATION_FORMAT'] = \"chat\"\n",
        "modelChoices['DEFAULT_MEASUREMENT_MODEL'] = \"gpt-3.5-turbo\"\n",
        "modelChoices['DEFAULT_MEASUREMENT_FORMAT'] = \"chat\"\n",
        "\n",
        "training_configs = TRAIN_SIZE_LIMIT,TEST_SIZE_LIMIT\n",
        "\n",
        "print(f\"\"\"Confirm the following parameters by typing \\\"yes\\\":\n",
        "Dataset: {CONFIG_NAME}\n",
        "Concepts to generate: {N_ITERS}\n",
        "Train size: {TRAIN_SIZE_LIMIT}\n",
        "Test size: {TEST_SIZE_LIMIT}\n",
        "Max Text length: {EXAMPLE_LENGTH_LIMIT}\n",
        "Trial size: {TRIAL_SIZE}\n",
        "Model choices: {modelChoices}\"\"\")\n",
        "if input(\"Confirm: \") == \"yes\":\n",
        "  clear_output()\n",
        "  config = configsDict[CONFIG_NAME]\n",
        "  train_fn = config['train']\n",
        "  test_fn = config['test']\n",
        "  task_type = config['type']\n",
        "  description = config['description']\n",
        "  label_guide = config['class names']\n",
        "\n",
        "  if task_type == 'Classification':\n",
        "    is_regression = False\n",
        "  elif task_type == 'Regression':\n",
        "    is_regression = True\n",
        "\n",
        "  concept_tree = SupervisedConceptSpace(train_fn,test_fn,is_regression,\n",
        "          training_configs,CONFIG_NAME,description,label_guide,\n",
        "          examplesMaxLength= EXAMPLE_LENGTH_LIMIT, modelChoices = modelChoices)\n",
        "\n",
        "\n",
        "  concept_tree.train(N_ITERS, TRIAL_SIZE, THRESHOLD, VERBOSE)\n",
        "  concept_trees[CONFIG_NAME] = concept_tree"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "ACY7DMmO1k7J",
        "outputId": "6aa1e287-3eb4-426a-c56b-9c637ae6cbdc"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "===== Concept Bottleck Model =====\n",
            "Current Concepts: []\n",
            "Total Generated: 1\n",
            "Total Accepted: 0\n",
            "Remaining iterations: 5\n",
            "==================================\n",
            "Generated Concept: {\n",
            "  \"Concept Name\": \"portion size\",\n",
            "  \"Concept Description\": \"Portion size refers to the amount of food served in a restaurant\n",
            "dish. It can vary from small to large and can impact the overall dining experience. Small portion sizes may leave customers unsatisfied, while large\n",
            "portion sizes may be overwhelming or lead to wastage. The concept of portion size can influence the rating given to a restaurant.\",\n",
            "  \"Concept\n",
            "Question\": \"What is the size of the portions in the restaurant dish?\",\n",
            "  \"Possible Responses\": [\n",
            "    \"small\",\n",
            "    \"medium\",\n",
            "    \"large\",\n",
            "\"uncertain\"\n",
            "  ],\n",
            "  \"Response Guide\": {\n",
            "    \"small\": \"The portion size of the dish is small, which is not sufficient to satisfy the customer.\",\n",
            "\"medium\": \"The portion size of the dish is moderate, neither small nor large.\",\n",
            "    \"large\": \"The portion size of the dish is large, which is generous\n",
            "and satisfying.\",\n",
            "    \"uncertain\": \"It is difficult to determine the portion size of the dish based on the provided text or it is not mentioned.\"\n",
            "  },\n",
            "\"Response Mapping\": {\n",
            "    \"small\": -1,\n",
            "    \"medium\": 0,\n",
            "    \"large\": 1,\n",
            "    \"uncertain\": 0\n",
            "  }\n",
            "}\n",
            "Repair any potential issues if any are detected. Do not repair any issues related to mutual exclusivity/collective exhaustion. Do not change the response set.\n",
            "{\"Errors\": \"None\",\"Fix\": \"None\",\"New Concept\": {\"Concept Name\": \"portion size\",\"Concept Description\": \"Portion size refers to the amount of food served in a restaurant dish. It can vary from small to large and can impact the overall dining experience. Small portion sizes may leave customers unsatisfied, while large portion sizes may be overwhelming or lead to wastage. The concept of portion size can influence the rating given to a restaurant.\",\"Concept Question\": \"What is the size of the portions in the restaurant dish?\",\"Possible Responses\": [\"small\", \"medium\", \"large\", \"uncertain\"],\"Response Guide\": {\"small\": \"The portion size of the dish is small, which is not sufficient to satisfy the customer.\",\"medium\": \"The portion size of the dish is moderate, neither small nor large.\",\"large\": \"The portion size of the dish is large, which is generous and satisfying.\",\"uncertain\": \"It is difficult to determine the portion size of the dish based on the provided text or it is not mentioned.\"},\"Response Mapping\": {\"small\": -1,\"medium\": 0,\"large\": 1,\"uncertain\": 0}}}\n",
            "=== System Response ===\n",
            "None\n",
            "{\n",
            "  \"Concept Name\": \"portion size\",\n",
            "  \"Concept Description\": \"Portion size refers to the amount of food served in a restaurant dish. It can vary from small to large and can impact the overall dining experience. Small portion sizes may leave customers unsatisfied, while large portion sizes may be overwhelming or lead to wastage. The concept of portion size can influence the rating given to a restaurant.\",\n",
            "  \"Concept Question\": \"What is the size of the portions in the restaurant dish?\",\n",
            "  \"Possible Responses\": [\n",
            "    \"small\",\n",
            "    \"medium\",\n",
            "    \"large\",\n",
            "    \"uncertain\"\n",
            "  ],\n",
            "  \"Response Guide\": {\n",
            "    \"small\": \"The portion size of the dish is small, which is not sufficient to satisfy the customer.\",\n",
            "    \"medium\": \"The portion size of the dish is moderate, neither small nor large.\",\n",
            "    \"large\": \"The portion size of the dish is large, which is generous and satisfying.\",\n",
            "    \"uncertain\": \"It is difficult to determine the portion size of the dish based on the provided text or it is not mentioned.\"\n",
            "  },\n",
            "  \"Response Mapping\": {\n",
            "    \"small\": -1,\n",
            "    \"medium\": 0,\n",
            "    \"large\": 1,\n",
            "    \"uncertain\": 0\n",
            "  }\n",
            "}\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 40%|████      | 20/50 [09:15<13:53, 27.79s/it]\n",
            "ERROR:root:Internal Python error in the inspect module.\n",
            "Below is the traceback from this internal error.\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"<ipython-input-7-ce613d8ab4b4>\", line 829, in measure_concept\n",
            "    responses = list(tqdm(executor.map(lambda p: completion(MODEL, TEMP, p, stopToken=\"###\", maxtokens=1000, format=\"chat\"), prompts), total=len(prompts)))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tqdm/std.py\", line 1182, in __iter__\n",
            "    for obj in iterable:\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/_base.py\", line 621, in result_iterator\n",
            "    yield _result_or_cancel(fs.pop())\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/_base.py\", line 319, in _result_or_cancel\n",
            "    return fut.result(timeout)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/_base.py\", line 453, in result\n",
            "    self._condition.wait(timeout)\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 320, in wait\n",
            "    waiter.acquire()\n",
            "KeyboardInterrupt\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\n",
            "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
            "  File \"<ipython-input-12-40dcfa4ea0c7>\", line 51, in <cell line: 32>\n",
            "    concept_tree.train(N_ITERS, TRIAL_SIZE, THRESHOLD, VERBOSE)\n",
            "  File \"<ipython-input-7-ce613d8ab4b4>\", line 1354, in train\n",
            "    self.iterate(trial_size, threshold, n_iters, verbose)\n",
            "  File \"<ipython-input-7-ce613d8ab4b4>\", line 1095, in iterate\n",
            "    if self.add_concept(new_concept, trial_size, threshold):\n",
            "  File \"<ipython-input-7-ce613d8ab4b4>\", line 919, in add_concept\n",
            "    self.label_dataset(trial_df, concept)\n",
            "  File \"<ipython-input-7-ce613d8ab4b4>\", line 868, in label_dataset\n",
            "    answers = self.measure_concept(subset_to_label[self.text_col], concept)\n",
            "  File \"<ipython-input-7-ce613d8ab4b4>\", line 828, in measure_concept\n",
            "    with ThreadPoolExecutor(max_workers=10) as executor:  # max_workers determines how many threads will be spawned\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/_base.py\", line 649, in __exit__\n",
            "    self.shutdown(wait=True)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 235, in shutdown\n",
            "    t.join()\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 1096, in join\n",
            "    self._wait_for_tstate_lock()\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 1116, in _wait_for_tstate_lock\n",
            "    if lock.acquire(block, timeout):\n",
            "KeyboardInterrupt\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/IPython/core/interactiveshell.py\", line 2099, in showtraceback\n",
            "    stb = value._render_traceback_()\n",
            "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/IPython/core/ultratb.py\", line 1101, in get_records\n",
            "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/IPython/core/ultratb.py\", line 248, in wrapped\n",
            "    return f(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/IPython/core/ultratb.py\", line 281, in _fixed_getinnerframes\n",
            "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
            "  File \"/usr/lib/python3.10/inspect.py\", line 1662, in getinnerframes\n",
            "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
            "  File \"/usr/lib/python3.10/inspect.py\", line 1624, in getframeinfo\n",
            "    lines, lnum = findsource(frame)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/IPython/core/ultratb.py\", line 170, in findsource\n",
            "    file = getsourcefile(object) or getfile(object)\n",
            "  File \"/usr/lib/python3.10/inspect.py\", line 829, in getsourcefile\n",
            "    module = getmodule(object, filename)\n",
            "  File \"/usr/lib/python3.10/inspect.py\", line 869, in getmodule\n",
            "    if ismodule(module) and hasattr(module, '__file__'):\n",
            "KeyboardInterrupt\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-7-ce613d8ab4b4>\u001b[0m in \u001b[0;36mmeasure_concept\u001b[0;34m(self, texts, concept, batching)\u001b[0m\n\u001b[1;32m    828\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mThreadPoolExecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# max_workers determines how many threads will be spawned\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 829\u001b[0;31m             \u001b[0mresponses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcompletion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mMODEL\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTEMP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstopToken\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"###\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxtokens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"chat\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprompts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    830\u001b[0m       \u001b[0mextracted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0manswer_extractor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconcept\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Response Mapping'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresponses\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1181\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1182\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1183\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult_iterator\u001b[0;34m()\u001b[0m\n\u001b[1;32m    620\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 621\u001b[0;31m                         \u001b[0;32myield\u001b[0m \u001b[0m_result_or_cancel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    622\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m_result_or_cancel\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    318\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfut\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    452\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 453\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    454\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    319\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 320\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    321\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "    \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n",
            "\u001b[0;32m<ipython-input-12-40dcfa4ea0c7>\u001b[0m in \u001b[0;36m<cell line: 32>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m   \u001b[0mconcept_tree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mN_ITERS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTRIAL_SIZE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTHRESHOLD\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVERBOSE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m   \u001b[0mconcept_trees\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mCONFIG_NAME\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconcept_tree\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-ce613d8ab4b4>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, n_iters, trial_size, threshold, verbose)\u001b[0m\n\u001b[1;32m   1353\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdashboard\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'remaining_iterations'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mn_iters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1354\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthreshold\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_iters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1355\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisplay_results\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-ce613d8ab4b4>\u001b[0m in \u001b[0;36miterate\u001b[0;34m(self, trial_size, threshold, remaining_iters, verbose)\u001b[0m\n\u001b[1;32m   1094\u001b[0m       \u001b[0;31m#If concept added, add to internal clf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1095\u001b[0;31m       \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_concept\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_concept\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthreshold\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1096\u001b[0m         \u001b[0mfeatureSet\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Concept Name'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcepts_list\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-ce613d8ab4b4>\u001b[0m in \u001b[0;36madd_concept\u001b[0;34m(self, concept, trial_size, threshold, messages)\u001b[0m\n\u001b[1;32m    918\u001b[0m         \u001b[0;31m# Label the trial_df\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 919\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconcept\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    920\u001b[0m         \u001b[0;31m# Update the corresponding rows in train_df\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-ce613d8ab4b4>\u001b[0m in \u001b[0;36mlabel_dataset\u001b[0;34m(self, dataset, concept)\u001b[0m\n\u001b[1;32m    867\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msubset_to_label\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 868\u001b[0;31m         \u001b[0manswers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmeasure_concept\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubset_to_label\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext_col\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconcept\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    869\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-ce613d8ab4b4>\u001b[0m in \u001b[0;36mmeasure_concept\u001b[0;34m(self, texts, concept, batching)\u001b[0m\n\u001b[1;32m    827\u001b[0m       \u001b[0mprompts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmeasure_concept_prompt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconcept\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtext\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtruncated_texts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m       \u001b[0;32mwith\u001b[0m \u001b[0mThreadPoolExecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# max_workers determines how many threads will be spawned\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    829\u001b[0m             \u001b[0mresponses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcompletion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mMODEL\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTEMP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstopToken\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"###\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxtokens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"chat\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprompts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, exc_type, exc_val, exc_tb)\u001b[0m\n\u001b[1;32m    648\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_tb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 649\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    650\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/thread.py\u001b[0m in \u001b[0;36mshutdown\u001b[0;34m(self, wait, cancel_futures)\u001b[0m\n\u001b[1;32m    234\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_threads\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m                 \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m     \u001b[0mshutdown\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_base\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/threading.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1095\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1096\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_tstate_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1097\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/threading.py\u001b[0m in \u001b[0;36m_wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1115\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1116\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1117\u001b[0m                 \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[1;32m   2098\u001b[0m                         \u001b[0;31m# in the engines. This should return a list of strings.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2099\u001b[0;31m                         \u001b[0mstb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_render_traceback_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2100\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'KeyboardInterrupt' object has no attribute '_render_traceback_'",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "    \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[1;32m   2099\u001b[0m                         \u001b[0mstb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_render_traceback_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2100\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2101\u001b[0;31m                         stb = self.InteractiveTB.structured_traceback(etype,\n\u001b[0m\u001b[1;32m   2102\u001b[0m                                             value, tb, tb_offset=tb_offset)\n\u001b[1;32m   2103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1365\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1366\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1367\u001b[0;31m         return FormattedTB.structured_traceback(\n\u001b[0m\u001b[1;32m   1368\u001b[0m             self, etype, value, tb, tb_offset, number_of_lines_of_context)\n\u001b[1;32m   1369\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1265\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose_modes\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1266\u001b[0m             \u001b[0;31m# Verbose modes need a full traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1267\u001b[0;31m             return VerboseTB.structured_traceback(\n\u001b[0m\u001b[1;32m   1268\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_offset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnumber_of_lines_of_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1269\u001b[0m             )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1140\u001b[0m         \u001b[0mchained_exc_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1141\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0mevalue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1142\u001b[0;31m             formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,\n\u001b[0m\u001b[1;32m   1143\u001b[0m                                                                      chained_exceptions_tb_offset)\n\u001b[1;32m   1144\u001b[0m             \u001b[0mexception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_parts_of_chained_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mformat_exception_as_a_whole\u001b[0;34m(self, etype, evalue, etb, number_of_lines_of_context, tb_offset)\u001b[0m\n\u001b[1;32m   1080\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1081\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1082\u001b[0;31m         \u001b[0mlast_unique\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecursion_repeat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_recursion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morig_etype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1083\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1084\u001b[0m         \u001b[0mframes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat_records\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlast_unique\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecursion_repeat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mfind_recursion\u001b[0;34m(etype, value, records)\u001b[0m\n\u001b[1;32m    380\u001b[0m     \u001b[0;31m# first frame (from in to out) that looks different.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_recursion_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 382\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    383\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m     \u001b[0;31m# Select filename, lineno, func_name to track frames with\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: object of type 'NoneType' has no len()"
          ]
        }
      ]
    }
  ]
}