experiments/DeterministicResultsOpenAI/Deterministic Results in OpenAI (report).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "22441125-7eae-4d13-b8c9-97a8fe3704f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d590fa84-64bd-49a0-8dff-72e27d7f923f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv(\"../../../../settings/.env\", override=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8c00576a-6cf0-4b5c-9758-dc696e9e5131",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import pandas as pd\n",
    "\n",
    "def variability_from_most_common(all_values):#, reference, value\n",
    "\n",
    "    variability_matrix = []\n",
    "    for i, v in enumerate(all_values):\n",
    "        variability_matrix.append([])\n",
    "        for j, v2 in enumerate(all_values):\n",
    "            vectorizer = TfidfVectorizer().fit_transform([v, v2])\n",
    "            vectors = vectorizer.toarray()\n",
    "            csim = cosine_similarity(vectors)\n",
    "            variability_matrix[i].append(csim[0][1])\n",
    "\n",
    "    variability_matrix_df = pd.DataFrame(variability_matrix,columns=all_values)\n",
    "    \n",
    "    return variability_matrix_df.mean().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "f1b3d790-68e6-4c13-9c7c-93f34f952bee",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import Markdown, display\n",
    "\n",
    "def printmd(string):\n",
    "    display(Markdown(string))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5707ca13-b8ae-4f02-9979-bda500e68c8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(0, '../../../../05 Own Solutions/AIChampTools')\n",
    "\n",
    "import importlib\n",
    "\n",
    "from AIChampTools import AIChampTools, LLMUsage, PromptEngineeringExperiment\n",
    "prevent_output = importlib.reload(sys.modules['AIChampTools'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b858d31a-47d8-48e2-b575-613d4237bd85",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment = PromptEngineeringExperiment(\n",
    "    name=\"openai_seed_parameter\",\n",
    "    logs_folder=\"../../logs/\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad9803c0-3782-49aa-a476-262bf18dd9f1",
   "metadata": {},
   "source": [
    "# Experiment Report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddc73242-a628-4203-be9c-bab85c18ed23",
   "metadata": {},
   "source": [
    "In this experiment I was trying to see if seed parameter of OpenAI GPT API can give stable results and how stable they are compared to very low top_p."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b87f89fb-1630-4105-8902-8748502eb883",
   "metadata": {},
   "source": [
    "## Version 09"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a44896f3-4ca2-44ef-accd-84c8d4f75d2e",
   "metadata": {},
   "source": [
    "In this version, I have set a limit of 200 tokens for completions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "643b1259-9db7-4f00-ae9b-88b356531d93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "### Variation 1"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Messages template**:\n",
       "[\"[{'role': 'user', 'content': '{user_message}'}]\"]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**User Messages**:\n",
       "['{\\'user_message\\': \\'Describe how WW2 started.\\', \\'expected_answer\\': \\'\"\"\\'}']"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**LLM Params**:\n",
       "{\"model\": \"gpt-4-1106-preview\", \"max_tokens\": 200, \"seed\": 33, \"temperature\": 0, \"timeout\": 30}"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Completions in the experiment**:\n",
       "100"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Results variability**: 0.8988740587231018"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Variation 2"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Messages template**:\n",
       "[\"[{'role': 'user', 'content': '{user_message}'}]\"]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**User Messages**:\n",
       "['{\\'user_message\\': \\'Describe how WW2 started.\\', \\'expected_answer\\': \\'\"\"\\'}']"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**LLM Params**:\n",
       "{\"model\": \"gpt-4-1106-preview\", \"max_tokens\": 200, \"top_p\": 1e-06, \"temperature\": 0, \"timeout\": 30}"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Completions in the experiment**:\n",
       "100"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Results variability**: 0.8403985918161336"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Variation 3"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Messages template**:\n",
       "[\"[{'role': 'user', 'content': '{user_message}'}]\"]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**User Messages**:\n",
       "['{\\'user_message\\': \\'Describe how WW2 started.\\', \\'expected_answer\\': \\'\"\"\\'}']"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**LLM Params**:\n",
       "{\"model\": \"gpt-4-1106-preview\", \"max_tokens\": 200, \"temperature\": 0, \"timeout\": 30}"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Completions in the experiment**:\n",
       "100"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Results variability**: 0.8756134307092054"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ver09_results_nf = experiment.load_results(ver=\"09\")\n",
    "\n",
    "ver09_results_nf[\"llm_params\"] = ver09_results_nf[\"llm_params\"].apply(json.dumps)\n",
    "ver09_results_nf[\"messages_template\"] = ver09_results_nf[\"messages_template\"].apply(str)\n",
    "\n",
    "for i, llm_params in enumerate(ver09_results_nf[\"llm_params\"].unique()):\n",
    "    completions = ver09_results_nf[ver09_results_nf[\"llm_params\"]==llm_params]\n",
    "    \n",
    "    printmd(f'### Variation {i+1}')\n",
    "    printmd(f'**Messages template**:\\n{list(completions[\"messages_template\"].unique())}')\n",
    "    printmd(f'**User Messages**:\\n{completions[\"data\"].apply(str).unique()}')\n",
    "    printmd(f\"**LLM Params**:\\n{llm_params}\")\n",
    "    printmd(f'**Completions in the experiment**:\\n{len(completions)}')\n",
    "\n",
    "    vrblt = variability_from_most_common(list(completions[\"generation\"]))\n",
    "    printmd(f'**Results variability**: {vrblt}')\n",
    "    printmd(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6739e56-2516-46e2-b3d2-5fa0f8d975c7",
   "metadata": {},
   "source": [
    "## Version 10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18569b2a-b445-48e6-bd83-36190c7997f5",
   "metadata": {},
   "source": [
    "In this version, I have set a limit of 50 tokens for completions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "0c11c354-3d55-4ec5-ab99-b355b563681a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "### Variation 1"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Messages template**:\n",
       "[\"[{'role': 'user', 'content': '{user_message}'}]\"]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**User Messages**:\n",
       "['{\\'user_message\\': \\'Describe how WW2 started.\\', \\'expected_answer\\': \\'\"\"\\'}']"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**LLM Params**:\n",
       "{\"model\": \"gpt-4-1106-preview\", \"max_tokens\": 50, \"seed\": 33, \"temperature\": 0, \"timeout\": 30}"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Completions in the experiment**:\n",
       "100"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Results variability**: 0.9999999999999999"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Variation 2"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Messages template**:\n",
       "[\"[{'role': 'user', 'content': '{user_message}'}]\"]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**User Messages**:\n",
       "['{\\'user_message\\': \\'Describe how WW2 started.\\', \\'expected_answer\\': \\'\"\"\\'}']"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**LLM Params**:\n",
       "{\"model\": \"gpt-4-1106-preview\", \"max_tokens\": 50, \"top_p\": 1e-06, \"temperature\": 0, \"timeout\": 30}"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Completions in the experiment**:\n",
       "100"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Results variability**: 0.9019718039364597"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Variation 3"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Messages template**:\n",
       "[\"[{'role': 'user', 'content': '{user_message}'}]\"]"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**User Messages**:\n",
       "['{\\'user_message\\': \\'Describe how WW2 started.\\', \\'expected_answer\\': \\'\"\"\\'}']"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**LLM Params**:\n",
       "{\"model\": \"gpt-4-1106-preview\", \"max_tokens\": 50, \"temperature\": 0, \"timeout\": 30}"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Completions in the experiment**:\n",
       "100"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "**Results variability**: 0.9323350713117965"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ver10_results_nf = experiment.load_results(ver=\"10\")\n",
    "\n",
    "ver10_results_nf[\"llm_params\"] = ver10_results_nf[\"llm_params\"].apply(json.dumps)\n",
    "ver10_results_nf[\"messages_template\"] = ver10_results_nf[\"messages_template\"].apply(str)\n",
    "\n",
    "for i, llm_params in enumerate(ver10_results_nf[\"llm_params\"].unique()):\n",
    "    completions = ver10_results_nf[ver10_results_nf[\"llm_params\"]==llm_params]\n",
    "    \n",
    "    printmd(f'### Variation {i+1}')\n",
    "    printmd(f'**Messages template**:\\n{list(completions[\"messages_template\"].unique())}')\n",
    "    printmd(f'**User Messages**:\\n{completions[\"data\"].apply(str).unique()}')\n",
    "    printmd(f\"**LLM Params**:\\n{llm_params}\")\n",
    "    printmd(f'**Completions in the experiment**:\\n{len(completions)}')\n",
    "\n",
    "    vrblt = variability_from_most_common(list(completions[\"generation\"]))\n",
    "    printmd(f'**Results variability**: {vrblt}')\n",
    "    printmd(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92e424f3-7b5b-4592-a953-26d85a7fb1d0",
   "metadata": {},
   "source": [
    "## (Preliminary) Conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16dae244-3fae-4264-a3dd-fe4cc129bda2",
   "metadata": {},
   "source": [
    "More variations of the experiment (both in terms of models and input data) are required to make definitive conclusion.\n",
    "\n",
    "From the tests done so far we can see that `seed` parameter gives more stable results that low `top_p` and the longer the completion, the higher variability.\n",
    "\n",
    "Interestingly, adding very low `top_p` increased variability comparing to not having it."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}