YouTube_Loader.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install langchain"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-aI5PbcTGRik",
        "outputId": "dcc5542c-c94b-40a1-f63d-895dd878f4e2"
      },
      "id": "-aI5PbcTGRik",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting langchain\n",
            "  Downloading langchain-0.0.176-py3-none-any.whl (873 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m873.5/873.5 kB\u001b[0m \u001b[31m23.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyYAML>=5.4.1 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.10)\n",
            "Collecting aiohttp<4.0.0,>=3.8.3 (from langchain)\n",
            "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m47.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0.0,>=4.0.0 (from langchain)\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting dataclasses-json<0.6.0,>=0.5.7 (from langchain)\n",
            "  Downloading dataclasses_json-0.5.7-py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.8.4)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.22.4)\n",
            "Collecting openapi-schema-pydantic<2.0,>=1.2 (from langchain)\n",
            "  Downloading openapi_schema_pydantic-1.2.4-py3-none-any.whl (90 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.0/90.0 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.10.7)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.27.1)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (2.0.12)\n",
            "Collecting multidict<7.0,>=4.5 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m25.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting marshmallow<4.0.0,>=3.3.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading marshmallow-3.19.0-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting marshmallow-enum<2.0.0,>=1.5.1 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading marshmallow_enum-1.5.1-py2.py3-none-any.whl (4.2 kB)\n",
            "Collecting typing-inspect>=0.4.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading typing_inspect-0.8.0-py3-none-any.whl (8.7 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<2,>=1->langchain) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.4)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (2.0.2)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.3.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (23.1)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Installing collected packages: mypy-extensions, multidict, marshmallow, frozenlist, async-timeout, yarl, typing-inspect, openapi-schema-pydantic, marshmallow-enum, aiosignal, dataclasses-json, aiohttp, langchain\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 dataclasses-json-0.5.7 frozenlist-1.3.3 langchain-0.0.176 marshmallow-3.19.0 marshmallow-enum-1.5.1 multidict-6.0.4 mypy-extensions-1.0.0 openapi-schema-pydantic-1.2.4 typing-inspect-0.8.0 yarl-1.9.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dd186bb6",
      "metadata": {
        "id": "dd186bb6"
      },
      "outputs": [],
      "source": [
        "from langchain.document_loaders import YoutubeLoader\n",
        "from langchain.llms import OpenAI\n",
        "from langchain.chains.summarize import load_summarize_chain"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2ba51e36",
      "metadata": {
        "hide_input": false,
        "id": "2ba51e36"
      },
      "outputs": [],
      "source": [
        "OPENAI_API_KEY = '...'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e567d2ef",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e567d2ef",
        "outputId": "c19a5adc-e253-4714-d2eb-568dc512b35f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting youtube-transcript-api\n",
            "  Downloading youtube_transcript_api-0.6.0-py3-none-any.whl (23 kB)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from youtube-transcript-api) (2.27.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->youtube-transcript-api) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->youtube-transcript-api) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->youtube-transcript-api) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->youtube-transcript-api) (3.4)\n",
            "Installing collected packages: youtube-transcript-api\n",
            "Successfully installed youtube-transcript-api-0.6.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pytube\n",
            "  Downloading pytube-15.0.0-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.6/57.6 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pytube\n",
            "Successfully installed pytube-15.0.0\n"
          ]
        }
      ],
      "source": [
        "!pip install youtube-transcript-api\n",
        "!pip install pytube"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "0b913708",
      "metadata": {
        "id": "0b913708"
      },
      "source": [
        "### 1. Simple Videos"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e30060ab",
      "metadata": {
        "id": "e30060ab"
      },
      "outputs": [],
      "source": [
        "loader = YoutubeLoader.from_youtube_url(\"https://www.youtube.com/watch?v=QsYGlZkevEg\", add_video_info=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ba6e2832",
      "metadata": {
        "id": "ba6e2832"
      },
      "outputs": [],
      "source": [
        "result = loader.load()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "52a99b6d",
      "metadata": {
        "collapsed": true,
        "id": "52a99b6d"
      },
      "outputs": [],
      "source": [
        "print (type(result))\n",
        "print (f\"Found video from {result[0].metadata['author']} that is {result[0].metadata['length']} seconds long\")\n",
        "print (\"\")\n",
        "print (result)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install openai"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3enO4xHOGrlF",
        "outputId": "df90da02-2abc-4969-f09a-df56de65e6b9"
      },
      "id": "3enO4xHOGrlF",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting openai\n",
            "  Downloading openai-0.27.7-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.0/72.0 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai) (2.27.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai) (4.65.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai) (3.8.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (3.4)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (1.3.1)\n",
            "Installing collected packages: openai\n",
            "Successfully installed openai-0.27.7\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0a675091",
      "metadata": {
        "id": "0a675091"
      },
      "outputs": [],
      "source": [
        "llm = OpenAI(temperature=0, openai_api_key=OPENAI_API_KEY)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "580bceb8",
      "metadata": {
        "id": "580bceb8"
      },
      "outputs": [],
      "source": [
        "chain = load_summarize_chain(llm, chain_type=\"stuff\", verbose=False)\n",
        "chain.run(result)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "f47acaa5",
      "metadata": {
        "id": "f47acaa5"
      },
      "source": [
        "### 2. Long Videos"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "985523d3",
      "metadata": {
        "id": "985523d3"
      },
      "outputs": [],
      "source": [
        "loader = YoutubeLoader.from_youtube_url(\"https://www.youtube.com/watch?v=6Ub7Z1AGIuk\", add_video_info=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d2c95abe",
      "metadata": {
        "id": "d2c95abe"
      },
      "outputs": [],
      "source": [
        "result = loader.load()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f7454d1c",
      "metadata": {
        "id": "f7454d1c"
      },
      "outputs": [],
      "source": [
        "chain = load_summarize_chain(llm, chain_type=\"stuff\", verbose=False)\n",
        "chain.run(result)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "92a9de4b",
      "metadata": {
        "id": "92a9de4b"
      },
      "source": [
        "Transcript/document is too long"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6d071a07",
      "metadata": {
        "id": "6d071a07"
      },
      "outputs": [],
      "source": [
        "from langchain.text_splitter import RecursiveCharacterTextSplitter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "36968fb9",
      "metadata": {
        "id": "36968fb9"
      },
      "outputs": [],
      "source": [
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cb6beb06",
      "metadata": {
        "id": "cb6beb06"
      },
      "outputs": [],
      "source": [
        "texts = text_splitter.split_documents(result)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b7889046",
      "metadata": {
        "id": "b7889046"
      },
      "outputs": [],
      "source": [
        "chain = load_summarize_chain(llm, chain_type=\"map_reduce\", verbose=True)\n",
        "chain.run(texts[:4])"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "7d5cf285",
      "metadata": {
        "id": "7d5cf285"
      },
      "source": [
        "### 3. Multiple Videos"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2dad798e",
      "metadata": {
        "id": "2dad798e"
      },
      "outputs": [],
      "source": [
        "youtube_url_list = [\"https://www.youtube.com/watch?v=AXq0QHUwmh8\", \"https://www.youtube.com/watch?v=EwHrjZxAT7g\"]\n",
        "\n",
        "texts = []\n",
        "\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=0)\n",
        "\n",
        "for url in youtube_url_list:\n",
        "    loader = YoutubeLoader.from_youtube_url(url, add_video_info=True)\n",
        "    result = loader.load()\n",
        "    \n",
        "    texts.extend(text_splitter.split_documents(result))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "862af735",
      "metadata": {
        "id": "862af735",
        "outputId": "67d2b9a3-de8e-4ab2-a7de-840aec05be7a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "' This article discusses the process of building a backyard golf green, including the cost of materials and the steps involved. Bella Ramsey and Pedro Pascal also discuss their experiences on the set of Game of Thrones and The Last of Us, including their first impressions of each other, what they took from set, and what item from the Game of Thrones world they would bring to The Last of Us. Bella talks about her most prized possession, a tear stick, and her best attribute in a survival scenario. Finally, they discuss their first jobs.'"
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "chain = load_summarize_chain(llm, chain_type=\"map_reduce\", verbose=False)\n",
        "chain.run(texts)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}