llm_dad_joke_evaluation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sttkMBJusZKh"
      },
      "outputs": [],
      "source": [
        "!pip install --upgrade fireworks-ai openai"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import fireworks.client\n",
        "\n",
        "fireworks.client.api_key = \"YOUR_API_KEY\""
      ],
      "metadata": {
        "id": "dk7ix2YNsdoN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def firework(model, prompt):\n",
        "  completion = fireworks.client.Completion.create(\n",
        "    model=model,\n",
        "    prompt=prompt,\n",
        "    n=1,\n",
        "    max_tokens=150,\n",
        "    temperature=1.0,\n",
        "    top_p=0.9,\n",
        "  )\n",
        "  return completion"
      ],
      "metadata": {
        "id": "V82QIUVasdq0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import openai\n",
        "openai.api_key = \"sk-\"\n",
        "\n",
        "def chatGPT(model, prompt):\n",
        "      response = openai.ChatCompletion.create(\n",
        "        model=model, messages=[\n",
        "            {'role': 'user', 'content': prompt}\n",
        "        ],\n",
        "        n=1,\n",
        "        max_tokens=150,\n",
        "        top_p=0.9,\n",
        "        temperature=1,\n",
        "      )\n",
        "      return response"
      ],
      "metadata": {
        "id": "OB2zyIHXsduF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "oyaji_gyag = [\n",
        "    \"おやじギャグを言って\",\n",
        "    \"50歳のお父さんが高校生の娘に放ったおやじギャグは？\",\n",
        "    \"IT 企業のおやじさんが新卒社員に向けてはなったギャグは？\",\n",
        "    \"居酒屋で酔ったおやじさんがつぶやくおやじギャグは？\",\n",
        "    \"おやじギャグといえば「布団が\"\n",
        "]"
      ],
      "metadata": {
        "id": "MmX5CDDAsdwJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fireworks_models = [\n",
        "    \"accounts/fireworks/models/elyza-japanese-llama-2-7b-fast-instruct\",\n",
        "    \"accounts/cresta-ai/models/openorca-7b-fast\",\n",
        "    \"accounts/fireworks/models/mistral-7b-instruct-4k\",\n",
        "    \"accounts/fireworks/models/llama-v2-70b-chat\"\n",
        "]"
      ],
      "metadata": {
        "id": "Z-XJ40ezsdyW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "chatgpt_models = [\n",
        "    'gpt-3.5-turbo-0613',\n",
        "    'gpt-4-0613'\n",
        "]"
      ],
      "metadata": {
        "id": "AfqgdPq3slid"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import csv\n",
        "import datetime\n",
        "filename = \"oyaji_gyag.csv\"\n",
        "for model in fireworks_models:\n",
        "    for prompt in oyaji_gyag:\n",
        "        completion = firework(model, prompt)\n",
        "        answer = completion.choices[0].text\n",
        "        with open(filename, 'a') as f:\n",
        "          writer = csv.writer(f)\n",
        "          writer.writerow([model, prompt, answer, datetime.datetime.now().strftime('%Y-%m-%d')])\n",
        "for gpt_model in chatgpt_models:\n",
        "    for prompt in oyaji_gyag:\n",
        "        completion = chatGPT(gpt_model, prompt)\n",
        "        answer = completion[\"choices\"][0][\"message\"][\"content\"]\n",
        "        with open(filename, 'a') as f:\n",
        "          writer = csv.writer(f)\n",
        "          writer.writerow([gpt_model, prompt, answer, datetime.datetime.now().strftime('%Y-%m-%d')])\n"
      ],
      "metadata": {
        "id": "DThF0df7sllR"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}