agent_swarm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "504bc66a-d6b5-4644-8bb5-51f527d9436c",
   "metadata": {},
   "source": [
    "## GPT Agent Swarm\n",
    "This notebook uses the OpenAI API to create a group of Assistants to each create a different type of regression model on a shared set on data. The predictions from the testing data from each model are then compared by R^2 values as well as a \"ensemble\" approach which was the averaging of each model's prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "b38529c2-4e4a-40a5-abec-95ec767d8884",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: openai in c:\\users\\merwi\\anaconda3\\lib\\site-packages (1.3.3)\n",
      "Requirement already satisfied: anyio<4,>=3.5.0 in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from openai) (3.5.0)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from openai) (1.8.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from openai) (0.25.1)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from openai) (1.10.8)\n",
      "Requirement already satisfied: tqdm>4 in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from openai) (4.65.0)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.5 in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from openai) (4.7.1)\n",
      "Requirement already satisfied: idna>=2.8 in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from anyio<4,>=3.5.0->openai) (3.4)\n",
      "Requirement already satisfied: sniffio>=1.1 in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from anyio<4,>=3.5.0->openai) (1.2.0)\n",
      "Requirement already satisfied: certifi in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from httpx<1,>=0.23.0->openai) (2023.7.22)\n",
      "Requirement already satisfied: httpcore in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from httpx<1,>=0.23.0->openai) (1.0.2)\n",
      "Requirement already satisfied: colorama in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from tqdm>4->openai) (0.4.6)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\merwi\\anaconda3\\lib\\site-packages (from httpcore->httpx<1,>=0.23.0->openai) (0.14.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "e51de6e5-737d-48eb-a529-a23ab75f0920",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import openai\n",
    "import time\n",
    "import ipywidgets as widgets\n",
    "from IPython.display import display\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "from io import StringIO\n",
    "import io\n",
    "import json\n",
    "\n",
    "## Define Functions\n",
    "\n",
    "def read_and_save_file(first_file_id, file_name):    \n",
    "    # its binary, so read it and then make it a file like object\n",
    "    file_data = client.files.content(first_file_id)\n",
    "    file_data_bytes = file_data.read()\n",
    "    file_like_object = io.BytesIO(file_data_bytes)\n",
    "    #now read as csv to create df\n",
    "    returned_data = pd.read_csv(file_like_object)\n",
    "    returned_data.to_csv(file_name, index=False)\n",
    "    return returned_data\n",
    "    # file = read_and_save_file(first_file_id, \"analyst_output.csv\")\n",
    "    \n",
    "def files_from_messages(messages, asst_name):\n",
    "    first_thread_message = messages.data[0]  # Accessing the first ThreadMessage\n",
    "    message_ids = first_thread_message.file_ids\n",
    "    print(message_ids)\n",
    "    # Loop through each file ID and save the file with a sequential name\n",
    "    for i, file_id in enumerate(message_ids):\n",
    "        file_name = f\"{asst_name}_output_{i+1}.csv\"  # Generate a sequential file name\n",
    "        read_and_save_file(file_id, file_name)\n",
    "        print(f'saved {file_name}')    \n",
    "        \n",
    "def spin_up(target, base_instructions, file_id):\n",
    "    # create assistant\n",
    "    my_assistant = client.beta.assistants.create(\n",
    "        instructions=base_instructions,\n",
    "        name=\"agent\",\n",
    "        tools=[{\"type\": \"code_interpreter\"}],\n",
    "        model=\"gpt-4-1106-preview\", # gpt-4\n",
    "        file_ids=file_id)\n",
    "    message_string = \"Please execute your ACTIONS on the csv file, the target field is \" + target\n",
    "    # Create a Thread\n",
    "    thread = client.beta.threads.create()\n",
    "    # Add a Message to a Thread\n",
    "    message = client.beta.threads.messages.create(\n",
    "        thread_id=thread.id,\n",
    "        role=\"user\",\n",
    "        content= message_string)\n",
    "    # Run the Assistant\n",
    "    run = client.beta.threads.runs.create(\n",
    "        thread_id=thread.id,\n",
    "        assistant_id=my_assistant.id)\n",
    "    return my_assistant, thread, run \n",
    "    print('Finished creating Assistants')\n",
    "    #assistant, thread, run = spin_up(n, base_instructions, file_id)    \n",
    "    \n",
    "def catch_response(assistant, thread, run):\n",
    "    #time.sleep(240)  \n",
    "    # Retrieve the run status\n",
    "    run_status = client.beta.threads.runs.retrieve(\n",
    "        thread_id=thread.id,\n",
    "        run_id=run.id)\n",
    "    print('Checking for response...')\n",
    "    # If run is completed, get messages\n",
    "    if run_status.status == 'completed':\n",
    "        messages = client.beta.threads.messages.list(\n",
    "            thread_id=thread.id)\n",
    "        # Loop through messages and print content based on role\n",
    "        for msg in messages.data:\n",
    "            role = msg.role\n",
    "            try:\n",
    "                content = msg.content[0].text.value\n",
    "                print(f\"{role.capitalize()}: {content}\")\n",
    "                return messages, content\n",
    "            except AttributeError:\n",
    "                # This will execute if .text does not exist\n",
    "                print(f\"{role.capitalize()}: [Non-text content, possibly an image or other file type]\")\n",
    "    else:\n",
    "        print('no response yet')\n",
    "    #messages, content = catch_response(assistant, thread, run)   \n",
    "    \n",
    "def spin_down(my_assistant_id):\n",
    "    response = client.beta.assistants.delete(my_assistant_id)\n",
    "    print(response)  \n",
    "    #spin_down(my_assistant_id)\n",
    "\n",
    "def upload_csv(file_name):\n",
    "    response = client.files.create(\n",
    "        file=open(file_name, \"rb\"),\n",
    "        purpose=\"assistants\")\n",
    "    print(response)\n",
    "    file_id = response.id\n",
    "    return file_id\n",
    "\n",
    "def delete_all_assistant_files():\n",
    "    ''' Deletes all exising files uploaded to client using API key '''\n",
    "    # generate a files object\n",
    "    files_object = client.files.list()\n",
    "    # get a list comprehension\n",
    "    file_ids = [file.id for file in files_object.data]\n",
    "    print(f'Deleting {len(file_ids)} files.')\n",
    "    #delete them all\n",
    "    for file_id in file_ids:\n",
    "        client.files.delete(file_id)\n",
    "        print(f\"Deleted file with ID: {file_id}\")\n",
    "    print('Finished deleting all files')\n",
    "\n",
    "def create_dataframes_from_messages(messages, client):\n",
    "    loop_dfs = []\n",
    "    first_thread_message = messages.data[0]  # Accessing the first ThreadMessage\n",
    "    message_ids = first_thread_message.file_ids\n",
    "    # Loop through each file ID and create a DataFrame\n",
    "    for file_id in message_ids:\n",
    "        # Read the file content\n",
    "        file_data = client.files.content(file_id)\n",
    "        file_data_bytes = file_data.read()\n",
    "        file_like_object = io.BytesIO(file_data_bytes)\n",
    "        # Create a DataFrame from the file-like object and append\n",
    "        df = pd.read_csv(file_like_object)\n",
    "        loop_dfs.append(df)\n",
    "    return loop_dfs\n",
    "\n",
    "def consolidate_response_dfs(df_list):\n",
    "    # Extract 'actual_mpg' from the first DataFrame\n",
    "    actual_mpg = df_list[0][0][['row_id', 'actual_mpg']].drop_duplicates('row_id')\n",
    "    # create empty DataFrame for the predicted_mpg values\n",
    "    predicted_mpg_df = pd.DataFrame()\n",
    "    # Loop through each DataFrame in the list\n",
    "    for i, df_tuple in enumerate(df_list):\n",
    "        # Check if the tuple is not empty\n",
    "        if df_tuple:\n",
    "            df = df_tuple[0]\n",
    "            # Rename 'predicted_mpg' column to match which agent predicted it\n",
    "            df = df.rename(columns={'predicted_mpg': f'predicted_mpg{i+1}'})\n",
    "            # Select only the 'row_id' and the renamed 'predicted_mpg' column\n",
    "            df = df[['row_id', f'predicted_mpg{i+1}']]\n",
    "            # initialize predicted_mpg_df on the first iteration\n",
    "            if i == 0:\n",
    "                predicted_mpg_df = df\n",
    "            else:\n",
    "                # Join using 'row_id'\n",
    "                predicted_mpg_df = predicted_mpg_df.merge(df, on='row_id', how='outer')\n",
    "        else:\n",
    "            print(f'run {i} is empty')\n",
    "    # Join the 'actual_mpg' with the predicted_mpg_df\n",
    "    consolidated_df = actual_mpg.merge(predicted_mpg_df, on='row_id', how='outer')\n",
    "    # Calculate the average of the predictions and add it as a new column\n",
    "    prediction_columns = [col for col in consolidated_df.columns if col.startswith('predicted_mpg')]\n",
    "    consolidated_df['average_prediction'] = consolidated_df[prediction_columns].mean(axis=1)\n",
    "    consolidated_df = consolidated_df.dropna()\n",
    "    return consolidated_df\n",
    "\n",
    "def calculate_r2(df, predicted_column):\n",
    "    # Extract the actual and predicted values\n",
    "    actual = df['actual_mpg']\n",
    "    predicted = df[predicted_column]\n",
    "    # Calculate the R² score\n",
    "    r2 = r2_score(actual, predicted)\n",
    "    return r2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb40c4b1-bd88-4397-abeb-20cd98f32a49",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Initialize API Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "18c2b56e-12c0-427a-add2-d5c902bd0c24",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# set key and assistant ID\n",
    "OPENAI_API_KEY = '<your API key goes here>'\n",
    "\n",
    "# Instantiate the OpenAI client\n",
    "client = openai.OpenAI(api_key=OPENAI_API_KEY)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "792f7b12-9455-47cc-83ba-6ebbd1e5fc8d",
   "metadata": {},
   "source": [
    "## Upload Data Set and Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "81007d7a-1ced-484b-8318-73d34b0745d9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>mpg</th>\n",
       "      <th>cylinders</th>\n",
       "      <th>displacement</th>\n",
       "      <th>horsepower</th>\n",
       "      <th>weight</th>\n",
       "      <th>acceleration</th>\n",
       "      <th>model year</th>\n",
       "      <th>origin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8</td>\n",
       "      <td>307.0</td>\n",
       "      <td>130</td>\n",
       "      <td>3504</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>8</td>\n",
       "      <td>350.0</td>\n",
       "      <td>165</td>\n",
       "      <td>3693</td>\n",
       "      <td>11.5</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8</td>\n",
       "      <td>318.0</td>\n",
       "      <td>150</td>\n",
       "      <td>3436</td>\n",
       "      <td>11.0</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>16.0</td>\n",
       "      <td>8</td>\n",
       "      <td>304.0</td>\n",
       "      <td>150</td>\n",
       "      <td>3433</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>17.0</td>\n",
       "      <td>8</td>\n",
       "      <td>302.0</td>\n",
       "      <td>140</td>\n",
       "      <td>3449</td>\n",
       "      <td>10.5</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>393</th>\n",
       "      <td>393</td>\n",
       "      <td>27.0</td>\n",
       "      <td>4</td>\n",
       "      <td>140.0</td>\n",
       "      <td>86</td>\n",
       "      <td>2790</td>\n",
       "      <td>15.6</td>\n",
       "      <td>82</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>394</td>\n",
       "      <td>44.0</td>\n",
       "      <td>4</td>\n",
       "      <td>97.0</td>\n",
       "      <td>52</td>\n",
       "      <td>2130</td>\n",
       "      <td>24.6</td>\n",
       "      <td>82</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>395</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4</td>\n",
       "      <td>135.0</td>\n",
       "      <td>84</td>\n",
       "      <td>2295</td>\n",
       "      <td>11.6</td>\n",
       "      <td>82</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>396</td>\n",
       "      <td>28.0</td>\n",
       "      <td>4</td>\n",
       "      <td>120.0</td>\n",
       "      <td>79</td>\n",
       "      <td>2625</td>\n",
       "      <td>18.6</td>\n",
       "      <td>82</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>397</td>\n",
       "      <td>31.0</td>\n",
       "      <td>4</td>\n",
       "      <td>119.0</td>\n",
       "      <td>82</td>\n",
       "      <td>2720</td>\n",
       "      <td>19.4</td>\n",
       "      <td>82</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>398 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     row_id   mpg  cylinders  displacement horsepower  weight  acceleration  \\\n",
       "0         0  18.0          8         307.0        130    3504          12.0   \n",
       "1         1  15.0          8         350.0        165    3693          11.5   \n",
       "2         2  18.0          8         318.0        150    3436          11.0   \n",
       "3         3  16.0          8         304.0        150    3433          12.0   \n",
       "4         4  17.0          8         302.0        140    3449          10.5   \n",
       "..      ...   ...        ...           ...        ...     ...           ...   \n",
       "393     393  27.0          4         140.0         86    2790          15.6   \n",
       "394     394  44.0          4          97.0         52    2130          24.6   \n",
       "395     395  32.0          4         135.0         84    2295          11.6   \n",
       "396     396  28.0          4         120.0         79    2625          18.6   \n",
       "397     397  31.0          4         119.0         82    2720          19.4   \n",
       "\n",
       "     model year  origin  \n",
       "0            70       1  \n",
       "1            70       1  \n",
       "2            70       1  \n",
       "3            70       1  \n",
       "4            70       1  \n",
       "..          ...     ...  \n",
       "393          82       1  \n",
       "394          82       2  \n",
       "395          82       1  \n",
       "396          82       1  \n",
       "397          82       1  \n",
       "\n",
       "[398 rows x 9 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     row_id   mpg  cylinders  displacement horsepower  weight  acceleration  \\\n",
      "293      71  19.0          3          70.0         97    2330          13.5   \n",
      "294     106  12.0          8         350.0        180    4499          12.5   \n",
      "295     270  21.1          4         134.0         95    2515          14.8   \n",
      "296     348  37.7          4          89.0         62    2050          17.3   \n",
      "297     102  26.0          4          97.0         46    1950          21.0   \n",
      "\n",
      "     model year  origin  \n",
      "293          72       3  \n",
      "294          73       1  \n",
      "295          78       3  \n",
      "296          81       3  \n",
      "297          73       2  \n",
      "    row_id   mpg  cylinders  displacement horsepower  weight  acceleration  \\\n",
      "95     378  38.0          4         105.0         63    2125          14.7   \n",
      "96     371  29.0          4         135.0         84    2525          16.0   \n",
      "97     280  21.5          6         231.0        115    3245          15.4   \n",
      "98     323  27.9          4         156.0        105    2800          14.4   \n",
      "99      75  14.0          8         318.0        150    4077          14.0   \n",
      "\n",
      "    model year  origin  \n",
      "95          82       1  \n",
      "96          82       1  \n",
      "97          79       1  \n",
      "98          80       1  \n",
      "99          72       1  \n",
      "FileObject(id='file-OuQnwoGzOUgd9aVo6C19uWi4', bytes=10777, created_at=1701835836, filename='auto-mpg-train.csv', object='file', purpose='assistants', status='processed', status_details=None)\n",
      "FileObject(id='file-dwa2A18ABLcgLR7ix2jOCvPH', bytes=3666, created_at=1701835837, filename='auto-mpg-test.csv', object='file', purpose='assistants', status='processed', status_details=None)\n",
      "['file-OuQnwoGzOUgd9aVo6C19uWi4', 'file-dwa2A18ABLcgLR7ix2jOCvPH']\n",
      " Row count in training set: 298, row countin testing set: 100\n"
     ]
    }
   ],
   "source": [
    "# load and drop name column\n",
    "df = pd.read_csv('auto-mpg.csv')\n",
    "df = df.drop('car name', axis = 1)\n",
    "df = df.reset_index().rename(columns={'index': 'row_id'})\n",
    "display(df)\n",
    "\n",
    "# create training and testing files\n",
    "train_data, test_data = train_test_split(df, test_size=.25, random_state=42) \n",
    "train_data.to_csv('auto-mpg-train.csv', index=False)\n",
    "test_data.to_csv('auto-mpg-test.csv', index=False)\n",
    "\n",
    "# read them back in\n",
    "train_df = pd.read_csv('auto-mpg-train.csv')\n",
    "test_df = pd.read_csv('auto-mpg-test.csv')\n",
    "print(train_df.tail())\n",
    "print(test_df.tail())\n",
    "\n",
    "#upload both files to Assistants\n",
    "train_file_id = upload_csv('auto-mpg-train.csv')\n",
    "test_file_id = upload_csv('auto-mpg-test.csv')\n",
    "file_ids = [train_file_id, test_file_id]\n",
    "print(file_ids)\n",
    "\n",
    "print(f' Row count in training set: {len(train_df)}, row countin testing set: {len(test_df)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8feae26f-b47e-4e23-a2f8-61275ed58f0d",
   "metadata": {},
   "source": [
    "## Create Instruction Strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "c03be584-1ec7-411b-a20b-31b9193311a5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "base_instructions_1 = '''\n",
    "You are a data scientist who will build a predictive model with data from the provided training and testing csv files. \n",
    "When the user asks you to perform your ACTIONS, carry out the described ACTIONS on the provided files.\n",
    "Then continue with each of the steps listed below in your ACTIONS. The user will identify the target variable. \n",
    "\n",
    "ACTIONS:\n",
    "\n",
    "1. Train a ''' \n",
    "\n",
    "model_types = ['Linear Regression','Decision Tree Regression','Random Forest Regression','Support Vector Regression']\n",
    "\n",
    "base_instructions_2 = ''' model with the training data.\n",
    "2. Test the model using the testing data.\n",
    "3. Create a table with three columns, one for row_id value called 'row_id', one for the actual mpg values in the testing data called 'actual_mpg' and one for the predicted mpg values called 'predicted_mpg'  \n",
    "4. Prepare the table as a csv file for the user to download. \n",
    "\n",
    "DO NOT:\n",
    "1. Return any images. '''\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27c38f1c-bb4e-44a0-b2f0-2f0ad5f49495",
   "metadata": {},
   "source": [
    "## Create the Swarm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "8f680849-e0a7-45ce-816b-9c91c2939429",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating assistant Linear Regression\n",
      "Creating assistant Decision Tree Regression\n",
      "Creating assistant Random Forest Regression\n",
      "Creating assistant Support Vector Regression\n"
     ]
    }
   ],
   "source": [
    "# run a loop to create Agent swarm \n",
    "agents = []\n",
    "for i in model_types:\n",
    "    print(f'Creating assistant {i}')\n",
    "    instructions = base_instructions_1 + i + base_instructions_2\n",
    "    assistant, thread, run = spin_up(\"mpg\", instructions, file_ids) \n",
    "    agents.append((assistant, thread, run))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "041c88a0-dac3-4347-96a5-fc6682d79a0d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking for response...\n",
      "Assistant: The table with 'row_id', 'actual_mpg', and 'predicted_mpg' values has been saved to a CSV file. You can download it using the link below:\n",
      "\n",
      "[Download the predictions CSV file](sandbox:/mnt/data/mpg_predictions.csv)\n",
      "Checking for response...\n",
      "Assistant: The table with the actual and predicted `mpg` values has been saved as a CSV file. You can download it using the following link:\n",
      "\n",
      "[Download mpg_predictions.csv](sandbox:/mnt/data/mpg_predictions.csv)\n",
      "Checking for response...\n",
      "Assistant: The results table has been saved as a CSV file. You can download it using the following link:\n",
      "\n",
      "[Download the predicted mpg CSV file](sandbox:/mnt/data/mpg_predictions.csv)\n",
      "Checking for response...\n",
      "Assistant: The predictive model has been trained and tested. The table with actual and predicted mpg values has been prepared and saved to a csv file. You can download the results using the link below:\n",
      "\n",
      "[Download predicted_mpg_results.csv](sandbox:/mnt/data/predicted_mpg_results.csv)\n"
     ]
    }
   ],
   "source": [
    "# run a loop to catch the Agent responses\n",
    "time.sleep(240) \n",
    "agent_responses = []\n",
    "for assistant, thread, run in agents:\n",
    "    messages, content = catch_response(assistant, thread, run) \n",
    "    agent_responses.append((messages, content))\n",
    "    time.sleep(10) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "ccd974fb-f515-4143-86f9-46179b76b22e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#extract dataframes and compile for accuracy\n",
    "df_list = []\n",
    "for messages, content in agent_responses:\n",
    "    dataframes = create_dataframes_from_messages(messages, client)\n",
    "    df_list.append(dataframes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "1ea524b1-0190-48ff-ab1e-77add9e536f2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[    row_id  actual_mpg  predicted_mpg\n",
       "  0      198        33.0      32.952238\n",
       "  1      396        28.0      29.533315\n",
       "  2       33        19.0      21.123169\n",
       "  3      208        13.0      16.771163\n",
       "  4       93        14.0      12.557398\n",
       "  ..     ...         ...            ...\n",
       "  95     378        38.0      32.482212\n",
       "  96     371        29.0      30.093171\n",
       "  97     280        21.5      23.479354\n",
       "  98     323        27.9      26.616858\n",
       "  99      75        14.0      12.778561\n",
       "  \n",
       "  [100 rows x 3 columns]],\n",
       " [    row_id  actual_mpg  predicted_mpg\n",
       "  0      198        33.0           31.0\n",
       "  1      396        28.0           26.6\n",
       "  2       33        19.0           21.0\n",
       "  3      208        13.0           15.0\n",
       "  4       93        14.0           17.5\n",
       "  ..     ...         ...            ...\n",
       "  95     378        38.0           36.0\n",
       "  96     371        29.0           27.2\n",
       "  97     280        21.5           22.4\n",
       "  98     323        27.9           23.6\n",
       "  99      75        14.0           13.0\n",
       "  \n",
       "  [100 rows x 3 columns]],\n",
       " [    row_id  actual_mpg  predicted_mpg\n",
       "  0      198        33.0         30.321\n",
       "  1      396        28.0         29.974\n",
       "  2       33        19.0         20.113\n",
       "  3      208        13.0         14.977\n",
       "  4       93        14.0         14.565\n",
       "  ..     ...         ...            ...\n",
       "  95     378        38.0         36.674\n",
       "  96     371        29.0         29.365\n",
       "  97     280        21.5         21.336\n",
       "  98     323        27.9         26.488\n",
       "  99      75        14.0         14.540\n",
       "  \n",
       "  [100 rows x 3 columns]],\n",
       " [    row_id  actual_mpg  predicted_mpg\n",
       "  0      198        33.0      32.041218\n",
       "  1      396        28.0      28.737055\n",
       "  2       33        19.0      20.873603\n",
       "  3      208        13.0      15.330378\n",
       "  4       93        14.0      13.699613\n",
       "  ..     ...         ...            ...\n",
       "  95     378        38.0      32.043098\n",
       "  96     371        29.0      29.582122\n",
       "  97     280        21.5      20.944252\n",
       "  98     323        27.9      26.247537\n",
       "  99      75        14.0      13.829836\n",
       "  \n",
       "  [100 rows x 3 columns]]]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "9914f268-5869-40e5-96be-bfc547d55ed1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>actual_mpg</th>\n",
       "      <th>predicted_mpg1</th>\n",
       "      <th>predicted_mpg2</th>\n",
       "      <th>predicted_mpg3</th>\n",
       "      <th>predicted_mpg4</th>\n",
       "      <th>average_prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>198</td>\n",
       "      <td>33.0</td>\n",
       "      <td>32.952238</td>\n",
       "      <td>31.0</td>\n",
       "      <td>30.321</td>\n",
       "      <td>32.041218</td>\n",
       "      <td>31.578614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>396</td>\n",
       "      <td>28.0</td>\n",
       "      <td>29.533315</td>\n",
       "      <td>26.6</td>\n",
       "      <td>29.974</td>\n",
       "      <td>28.737055</td>\n",
       "      <td>28.711092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33</td>\n",
       "      <td>19.0</td>\n",
       "      <td>21.123169</td>\n",
       "      <td>21.0</td>\n",
       "      <td>20.113</td>\n",
       "      <td>20.873603</td>\n",
       "      <td>20.777443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>208</td>\n",
       "      <td>13.0</td>\n",
       "      <td>16.771163</td>\n",
       "      <td>15.0</td>\n",
       "      <td>14.977</td>\n",
       "      <td>15.330378</td>\n",
       "      <td>15.519635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>93</td>\n",
       "      <td>14.0</td>\n",
       "      <td>12.557398</td>\n",
       "      <td>17.5</td>\n",
       "      <td>14.565</td>\n",
       "      <td>13.699613</td>\n",
       "      <td>14.580503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>378</td>\n",
       "      <td>38.0</td>\n",
       "      <td>32.482212</td>\n",
       "      <td>36.0</td>\n",
       "      <td>36.674</td>\n",
       "      <td>32.043098</td>\n",
       "      <td>34.299828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>371</td>\n",
       "      <td>29.0</td>\n",
       "      <td>30.093171</td>\n",
       "      <td>27.2</td>\n",
       "      <td>29.365</td>\n",
       "      <td>29.582122</td>\n",
       "      <td>29.060073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>280</td>\n",
       "      <td>21.5</td>\n",
       "      <td>23.479354</td>\n",
       "      <td>22.4</td>\n",
       "      <td>21.336</td>\n",
       "      <td>20.944252</td>\n",
       "      <td>22.039901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>323</td>\n",
       "      <td>27.9</td>\n",
       "      <td>26.616858</td>\n",
       "      <td>23.6</td>\n",
       "      <td>26.488</td>\n",
       "      <td>26.247537</td>\n",
       "      <td>25.738099</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>75</td>\n",
       "      <td>14.0</td>\n",
       "      <td>12.778561</td>\n",
       "      <td>13.0</td>\n",
       "      <td>14.540</td>\n",
       "      <td>13.829836</td>\n",
       "      <td>13.537099</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    row_id  actual_mpg  predicted_mpg1  predicted_mpg2  predicted_mpg3  \\\n",
       "0      198        33.0       32.952238            31.0          30.321   \n",
       "1      396        28.0       29.533315            26.6          29.974   \n",
       "2       33        19.0       21.123169            21.0          20.113   \n",
       "3      208        13.0       16.771163            15.0          14.977   \n",
       "4       93        14.0       12.557398            17.5          14.565   \n",
       "..     ...         ...             ...             ...             ...   \n",
       "95     378        38.0       32.482212            36.0          36.674   \n",
       "96     371        29.0       30.093171            27.2          29.365   \n",
       "97     280        21.5       23.479354            22.4          21.336   \n",
       "98     323        27.9       26.616858            23.6          26.488   \n",
       "99      75        14.0       12.778561            13.0          14.540   \n",
       "\n",
       "    predicted_mpg4  average_prediction  \n",
       "0        32.041218           31.578614  \n",
       "1        28.737055           28.711092  \n",
       "2        20.873603           20.777443  \n",
       "3        15.330378           15.519635  \n",
       "4        13.699613           14.580503  \n",
       "..             ...                 ...  \n",
       "95       32.043098           34.299828  \n",
       "96       29.582122           29.060073  \n",
       "97       20.944252           22.039901  \n",
       "98       26.247537           25.738099  \n",
       "99       13.829836           13.537099  \n",
       "\n",
       "[100 rows x 7 columns]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "consolidated_df = consolidate_response_dfs(df_list)\n",
    "consolidated_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "9453db6a-5f2e-4aee-be6c-067433a82aab",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_types</th>\n",
       "      <th>R_scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regression</td>\n",
       "      <td>0.841901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Decision Tree Regression</td>\n",
       "      <td>0.757636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest Regression</td>\n",
       "      <td>0.889145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Support Vector Regression</td>\n",
       "      <td>0.873117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ensemble</td>\n",
       "      <td>0.887872</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model_types  R_scores\n",
       "0          Linear Regression  0.841901\n",
       "1   Decision Tree Regression  0.757636\n",
       "2   Random Forest Regression  0.889145\n",
       "3  Support Vector Regression  0.873117\n",
       "4                   Ensemble  0.887872"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_types.append('Ensemble')\n",
    "\n",
    "R_scores = []\n",
    "column_names = consolidated_df.columns.tolist()\n",
    "for name in column_names:\n",
    "    if name != 'row_id' and name != 'actual_mpg':\n",
    "        score = calculate_r2(consolidated_df, name)\n",
    "        R_scores.append(score)\n",
    "\n",
    "# Create dictionary with list names as keys\n",
    "data = {'model_types': model_types, 'R_scores': R_scores}\n",
    "\n",
    "# Creating the DataFrame\n",
    "models_and_scores_df = pd.DataFrame(data)    \n",
    "display(models_and_scores_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2194780-ff04-4758-936c-1448f351863a",
   "metadata": {},
   "source": [
    "## Clean Up Agents and Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "87c96945-83f2-4413-bc72-fbc5775f818e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AssistantDeleted(id='asst_UmmORqTu5THwaXBXp7TqZ4mu', deleted=True, object='assistant.deleted')\n",
      "None\n",
      "AssistantDeleted(id='asst_fNGcsPgzaPDMgROnRzwnbtKB', deleted=True, object='assistant.deleted')\n",
      "None\n",
      "AssistantDeleted(id='asst_wgxQFwA6d0KB4FgiaLYmzugq', deleted=True, object='assistant.deleted')\n",
      "None\n",
      "AssistantDeleted(id='asst_fidwhVdxZmnsUqOXhQf5N4pA', deleted=True, object='assistant.deleted')\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "for assistant, thread, run in agents:\n",
    "    assistant_id = assistant.id\n",
    "    response = spin_down(assistant_id)\n",
    "    print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "c9adedba-68cc-4293-9e97-b67464ce0c8a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting 2 files.\n",
      "Deleted file with ID: file-dwa2A18ABLcgLR7ix2jOCvPH\n",
      "Deleted file with ID: file-OuQnwoGzOUgd9aVo6C19uWi4\n",
      "Finished deleting all files\n"
     ]
    }
   ],
   "source": [
    "delete_all_assistant_files()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}