langchain01.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNn/8Cx41TjtjVs+jX5fKhp",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/edersonmelo/prompt-engineer/blob/main/langchain01.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%env LANGCHAIN_API_KEY=<LANGCHAIN_API_KEY>\n",
        "%env OPENAI_API_KEY=<OPENAI_API_KEY>"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xjiE3QM6_-Gu",
        "outputId": "0145fa53-11d8-49ed-e285-019ecbfe5d3a"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "env: LANGCHAIN_API_KEY=<LANGCHAIN_API_KEY>\n",
            "env: OPENAI_API_KEY=<OPENAI_API_KEY>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install --upgrade --quiet  langchain langsmith langchainhub --quiet\n",
        "%pip install --upgrade --quiet  langchain-openai tiktoken pandas duckduckgo-search --quiet\n",
        "\n",
        "import os\n",
        "from uuid import uuid4\n",
        "\n",
        "unique_id = uuid4().hex[0:8]\n",
        "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
        "os.environ[\"LANGCHAIN_PROJECT\"] = f\"<LANGCHAIN_PROJECT>\"\n",
        "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
        "os.environ[\"LANGCHAIN_API_KEY\"] = \"<LANGCHAIN_API_KEY>\"  # Update to your API key\n",
        "\n",
        "\n",
        "# Used by the agent in this tutorial\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"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\""
      ],
      "metadata": {
        "id": "xcSz0RBROLei"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "LLKTl8j6Oyh1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install --quiet -U langchain\n",
        "%pip install --quiet -U langsmith\n",
        "%pip install --quiet -U openai"
      ],
      "metadata": {
        "id": "CCcAmRWOAJzi"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import langsmith\n",
        "import json\n",
        "\n",
        "client = langsmith.Client()\n",
        "\n",
        "def craft_messages(input, output) -> list[dict]:\n",
        "    out = json.dumps(output[\"clusters\"])\n",
        "    return [{\"role\": \"user\", \"content\": \"Extract triplets from the following sentence:\\n\\n\" + input[\"sentence\"]},\n",
        "            {\"role\": \"assistant\", \"content\": out}]"
      ],
      "metadata": {
        "id": "htFN_OawCp5d"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import itertools\n",
        "data = [\n",
        "    craft_messages(example.inputs, example.outputs) for example in itertools.islice(client.list_examples(dataset_name=\"Carb-IE-train\"), 50)\n",
        "    ]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "id": "yoylKBnwCvhU",
        "outputId": "9b74f7af-147a-44d9-c4d8-3b0936e98220"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'client' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-18-dfd2688d3817>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mitertools\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m data = [\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mcraft_messages\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexample\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexample\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mexample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mitertools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mislice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlist_examples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Carb-IE-train\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m50\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     ]\n",
            "\u001b[0;31mNameError\u001b[0m: name 'client' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import openai\n",
        "from io import BytesIO\n",
        "\n",
        "my_file = BytesIO()\n",
        "for m in data:\n",
        "    my_file.write((json.dumps({\"messages\": m}) + \"\\n\").encode('utf-8'))\n",
        "\n",
        "my_file.seek(0)\n",
        "training_file = openai.File.create(\n",
        "  file=my_file,\n",
        "  purpose='fine-tune'\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "id": "ElGWz37BLPV8",
        "outputId": "c346e40d-9dcc-47cb-9097-0762c3c1119a"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'data' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-65c786abd5d5>\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mmy_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBytesIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mm\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0mmy_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"messages\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"\\n\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'data' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import openai\n",
        "from io import BytesIO\n",
        "\n",
        "my_file = BytesIO()\n",
        "for m in data:\n",
        "    my_file.write((json.dumps({\"messages\": m}) + \"\\n\").encode('utf-8'))\n",
        "\n",
        "my_file.seek(0)\n",
        "training_file = openai.File.create(\n",
        "  file=my_file,\n",
        "  purpose='fine-tune'\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "id": "VIItGKmkDD6-",
        "outputId": "a296cc78-3589-4349-e7e2-566307f4937e"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'data' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-65c786abd5d5>\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mmy_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBytesIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mm\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0mmy_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"messages\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"\\n\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'data' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain import smith\n",
        "import json\n",
        "from typing import Any, Optional\n",
        "from langchain.evaluation import StringEvaluator\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain.prompts import ChatPromptTemplate\n",
        "from langchain.output_parsers import openai_functions\n",
        "\n",
        "eval_prompt = ChatPromptTemplate.from_messages(\n",
        "    [\n",
        "        (\"system\", \"You are an impartial grader tasked with measuring the accuracy of extracted entity relations.\"),\n",
        "        (\"human\", \"Please evaluate the following data:\\n\\n\"\n",
        "         \"<INPUT>\\n{input}</INPUT>\\n\"\n",
        "         \"<PREDICTED>\\n{prediction}</PREDICTED>\\n\"\n",
        "         \"<GROUND_TRUTH>\\n{reference}</GROUND_TRUTH>\\n\\n\"\n",
        "         \"Please save your reasoning and grading by calling the commit_grade function.\"\n",
        "         \" First, enumerate all factual discrepancies in the predicted triplets relative to the ground truth.\"\n",
        "         \" Finally, score the prediction on a scale out of 100, taking into account factuality and\"\n",
        "         \" correctness according to the ground truth.\"),\n",
        "\n",
        "    ]\n",
        ")\n",
        "\n",
        "commit_grade_schema = {\n",
        "    \"name\": \"commit_grade\",\n",
        "    \"description\": \"Commits a grade with reasoning.\",\n",
        "    \"parameters\": {\n",
        "        \"title\": \"commit_grade_parameters\",\n",
        "        \"description\": \"Parameters for the commit_grade function.\",\n",
        "        \"type\": \"object\",\n",
        "        \"properties\": {\n",
        "            \"mistakes\": {\n",
        "                \"title\": \"discrepancies\",\n",
        "                \"type\": \"string\",\n",
        "                \"description\": \"Any discrepencies between the predicted and ground truth.\"\n",
        "            },\n",
        "            \"reasoning\": {\n",
        "                \"title\": \"reasoning\",\n",
        "                \"type\": \"string\",\n",
        "                \"description\": \"The explanation or logic behind the final grade.\"\n",
        "            },\n",
        "            \"grade\": {\n",
        "                \"title\": \"grade\",\n",
        "                \"type\": \"number\",\n",
        "                \"description\": \"The numerical value representing the grade.\",\n",
        "                \"minimum\": 0,\n",
        "                \"maximum\": 100\n",
        "            }\n",
        "        },\n",
        "        \"required\": [\"reasoning\", \"grade\", \"mistakes\"],\n",
        "    }\n",
        "}\n",
        "\n",
        "def normalize_grade(func_args: str) -> dict:\n",
        "    args = json.loads(func_args)\n",
        "    return {\n",
        "        \"reasoning\": (args.get(\"reasoning\", \"\") + \"\\n\\n\" + args.get(\"discrepancies\", \"\")).strip(),\n",
        "        \"score\": args.get(\"grade\", 0) / 100,\n",
        "    }\n",
        "\n",
        "eval_chain = (\n",
        "    eval_prompt | ChatOpenAI(model=\"gpt-3\", temperature=0).bind(functions=[commit_grade_schema]) | openai_functions.OutputFunctionsParser() | normalize_grade\n",
        ")\n",
        "\n",
        "class EvaluateTriplets(StringEvaluator):\n",
        "    \"\"\"Evaluate the triplets of a predicted string.\"\"\"\n",
        "\n",
        "    @property\n",
        "    def requires_input(self) -> bool:\n",
        "        return True\n",
        "\n",
        "    @property\n",
        "    def requires_reference(self) -> bool:\n",
        "        return True\n",
        "\n",
        "    def _evaluate_strings(\n",
        "        self,\n",
        "        *,\n",
        "        prediction: str,\n",
        "        reference: Optional[str] = None,\n",
        "        input: Optional[str] = None,\n",
        "        **kwargs: Any,\n",
        "    ) -> dict:\n",
        "        callbacks = kwargs.pop(\"callbacks\", None)\n",
        "        return eval_chain.invoke(\n",
        "            {\"prediction\": prediction, \"reference\": reference, \"input\": input},\n",
        "            {\"callbacks\": callbacks},\n",
        "        )\n",
        "\n",
        "config = smith.RunEvalConfig(\n",
        "    custom_evaluators=[EvaluateTriplets()],\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 228
        },
        "id": "yUXp3fM_Ldwn",
        "outputId": "a7da5acb-cf31-483e-d0fe-aab8ae12f744"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "module 'langchain.output_parsers.openai_functions' has no attribute 'OutputFunctionsParser'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-10d5d5fadaf5>\u001b[0m in \u001b[0;36m<cell line: 62>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m eval_chain = (\n\u001b[0;32m---> 62\u001b[0;31m     \u001b[0meval_prompt\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0mChatOpenAI\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"gpt-3\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtemperature\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunctions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcommit_grade_schema\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0mopenai_functions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOutputFunctionsParser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0mnormalize_grade\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m )\n\u001b[1;32m     64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: module 'langchain.output_parsers.openai_functions' has no attribute 'OutputFunctionsParser'"
          ]
        }
      ]
    }
  ]
}