evaluation/results/modelcomparision.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Comparisons Gemini / OpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install trulens_eval==0.19.1 llama_index>=0.9.15post2 openai\n",
    "!pip -q install langchain_experimental langchain_core\n",
    "!pip -q install google-generativeai==0.3.1\n",
    "!pip -q install google-ai-generativelanguage==0.4.0\n",
    "!pip -q install langchain-google-genai\n",
    "!pip install -q -U google-generativeai\n",
    "!pip3 install gemini"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1: Setup and Imports\n",
    "First, ensure you have all the necessary libraries and API keys set up. You've already shown the setup for OpenAI and Hugging Face, so I'll focus on the setup for Google's Vertex AI Gemini."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "from trulens_eval import Feedback, Tru, OpenAI, TruBasicApp, FeedbackMode\n",
    "from trulens_eval.feedback.provider.hugs import (\n",
    "    positive_sentiment,\n",
    "    language_match,\n",
    "    not_toxic,\n",
    "    pii_detection\n",
    ")\n",
    "import requests  # For sending requests to Vertex AI\n",
    "\n",
    "\n",
    "# Set up API keys\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"your_openai_api_key\"\n",
    "os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"] = \"path_to_your_google_credentials.json\"\n",
    "\n",
    "# Initialize OpenAI\n",
    "openai.api_key = os.environ[\"OPENAI_API_KEY\"]\n",
    "\n",
    "# Initialize TruLens\n",
    "tru = Tru()\n",
    "tru.reset_database()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2: Define Functions for Models\n",
    "Define functions for each model (GPT-3.5, GPT-4, and Gemini) that you want to compare."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gpt_model(prompt, model_name=\"gpt-3.5-turbo\"):\n",
    "    return openai.ChatCompletion.create(\n",
    "        model=model_name,\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a question and answer bot. Answer upbeat.\"},\n",
    "            {\"role\": \"user\", \"content\": prompt}\n",
    "        ]\n",
    "    )[\"choices\"][0][\"message\"][\"content\"]\n",
    "\n",
    "def gemini_model(prompt):\n",
    "    # Replace with the appropriate URL and project details for Vertex AI Gemini\n",
    "    url = \"https://us-central1-aiplatform.googleapis.com/v1/projects/YOUR_PROJECT_ID/locations/us-central1/publishers/google/models/gemini-pro:streamGenerateContent\"\n",
    "    headers = {\n",
    "        \"Authorization\": \"Bearer $(gcloud auth print-access-token)\",\n",
    "        \"Content-Type\": \"application/json; charset=utf-8\"\n",
    "    }\n",
    "    data = {\n",
    "        \"contents\": {\n",
    "            \"role\": \"user\",\n",
    "            \"parts\": {\"text\": prompt}\n",
    "        },\n",
    "        # Add other configurations as needed\n",
    "    }\n",
    "    response = requests.post(url, headers=headers, json=data)\n",
    "    return response.json()  # Parse and return the relevant part of the response\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3: Initialize Feedback Functions\n",
    "Set up the feedback functions as you have in your initial setup."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_language_match = Feedback(language_match, feedback_mode=FeedbackMode.DEFERRED).on_input_output()\n",
    "f_positive_sentiment = Feedback(positive_sentiment, feedback_mode=FeedbackMode.DEFERRED).on_output()\n",
    "f_not_toxic = Feedback(not_toxic, feedback_mode=FeedbackMode.DEFERRED).on_output()\n",
    "f_pii_detection = Feedback(pii_detection, feedback_mode=FeedbackMode.DEFERRED).on_output()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4: Instrument the Callables for Logging with TruLens\n",
    "Create TruBasicApp instances for each model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feedbacks=[\n",
    "    f_language_match, f_positive_sentiment, f_not_toxic, f_pii_detection\n",
    "]\n",
    "gpt35_recorder = TruBasicApp(lambda p: gpt_model(p, \"gpt-3.5-turbo\"), app_id=\"gpt-3.5-turbo\", feedbacks=feedbacks)\n",
    "gpt4_recorder = TruBasicApp(lambda p: gpt_model(p, \"gpt-4\"), app_id=\"gpt-4\", feedbacks=feedbacks)\n",
    "gemini_recorder = TruBasicApp(gemini_model, app_id=\"gemini-pro\", feedbacks=feedbacks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 5: Run Evaluations\n",
    "Run the evaluations for each model using your prompts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    # Get Prompt List here evaluation/baselineprompts\n",
    "\n",
    "with gpt35_recorder as recording:\n",
    "    for prompt in prompts:\n",
    "        gpt35_recorder.app(prompt)\n",
    "\n",
    "with gpt4_recorder as recording:\n",
    "    for prompt in prompts:\n",
    "        gpt4_recorder.app(prompt)\n",
    "\n",
    "with gemini_recorder as recording:\n",
    "    for prompt in prompts:\n",
    "        gemini_recorder.app(prompt)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 5: Explore Results\n",
    "Finally, use TruLens to explore the results of your evaluation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming you have TruLens setup\n",
    "tru.run_dashboard()"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}