LangChain_Spark_AI.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNNVUmgiCMTLCjJChVIJ6yF",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sugarforever/LangChain-Tutorials/blob/main/LangChain_Spark_AI.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Connecting OpenAI with Apache Spark\n",
        "\n",
        "Introduction of [pyspark-ai](https://github.com/databrickslabs/pyspark-ai)\n",
        "\n",
        "Pyspark-AI takes English instructions and compile them into PySpark objects like DataFrames, to make Spark more user-friendly and accessible, allowing you to focus on extracting insights from your data."
      ],
      "metadata": {
        "id": "WCt1XDIYbJbi"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "yn5qTnakeBRW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "63ff54fc-a51c-4468-9cab-31c38c80c392"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/1.2 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.3/1.2 MB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m19.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m15.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.6/73.6 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m310.8/310.8 MB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.0/90.0 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.0/143.0 kB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.3/11.3 MB\u001b[0m \u001b[31m65.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m59.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m86.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m93.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m68.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.6/62.6 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m76.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.8/341.8 kB\u001b[0m \u001b[31m25.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "ipython 7.34.0 requires jedi>=0.16, which is not installed.\n",
            "google-colab 1.0.0 requires pandas==1.5.3, but you have pandas 2.0.3 which is incompatible.\n",
            "google-colab 1.0.0 requires requests==2.27.1, but you have requests 2.31.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "!pip install --quiet --upgrade langchain openai pyspark-ai pyspark"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ['OPENAI_API_KEY'] = 'your openai api key'"
      ],
      "metadata": {
        "id": "-EyoaM0HeOZ_"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "1. Initialize the Spark AI instance"
      ],
      "metadata": {
        "id": "TLKWpDYocfM9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.chat_models import ChatOpenAI\n",
        "from pyspark_ai import SparkAI\n",
        "\n",
        "# If 'gpt-4' is unavailable, use 'gpt-3.5-turbo' (might lower output quality)\n",
        "llm = ChatOpenAI(model_name='gpt-4', temperature=0)\n",
        "\n",
        "spark_ai = SparkAI(llm=llm, verbose=True)\n",
        "\n",
        "# Activate partial functions for Spark DataFrame\n",
        "spark_ai.activate()"
      ],
      "metadata": {
        "id": "iAUNKhTnedgg"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "2. Create a dataframe via a HTTP URL"
      ],
      "metadata": {
        "id": "9k0pycrlclEb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this case, we are fetching the share holders of Apple, one of the best performing stock in US market."
      ],
      "metadata": {
        "id": "kORq1qMKc2r-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "holders_dataframe = spark_ai.create_df(\"https://finance.yahoo.com/quote/AAPL/holders?p=AAPL\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J79qX_2ieiDX",
        "outputId": "a9788f52-a1bb-4c21-ac4d-1a4adf478873"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mParsing URL: https://finance.yahoo.com/quote/AAPL/holders?p=AAPL\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:Parsing URL: https://finance.yahoo.com/quote/AAPL/holders?p=AAPL\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mSQL query for the ingestion:\n",
            "\u001b[34mCREATE\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mOR\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mREPLACE\u001b[39;49;00m\u001b[37m \u001b[39;49;00mTEMP\u001b[37m \u001b[39;49;00m\u001b[34mVIEW\u001b[39;49;00m\u001b[37m \u001b[39;49;00mapple_stock_holders\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m*\u001b[37m \u001b[39;49;00m\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mVALUES\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Vanguard Group, Inc. (The)'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1309785362\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m8\u001b[39;49;00m.\u001b[34m33\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m254059068265\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Blackrock Inc.'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1035008939\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m6\u001b[39;49;00m.\u001b[34m58\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m200760685161\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Berkshire Hathaway, Inc'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m915560382\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m5\u001b[39;49;00m.\u001b[34m82\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m177591248414\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'State Street Corporation'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m576281774\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m3\u001b[39;49;00m.\u001b[34m66\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m111781376406\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'FMR, LLC'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m311437576\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m98\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m60409546996\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Geode Capital Management, LLC'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m285171112\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m81\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m55314640942\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Price (T.Rowe) Associates Inc'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m234017381\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m49\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m45392351678\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Morgan Stanley'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m200615893\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m28\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m38913465010\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Northern Trust Corporation'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m173130542\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m10\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m33582131443\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Norges Bank Investment Management'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m167374278\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Dec 30, 2022'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m06\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m32465588907\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00mv1(holder_name,\u001b[37m \u001b[39;49;00mshares,\u001b[37m \u001b[39;49;00mdate_reported,\u001b[37m \u001b[39;49;00mpercent_out,\u001b[37m \u001b[39;49;00mvalue)\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:SQL query for the ingestion:\n",
            "\u001b[34mCREATE\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mOR\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mREPLACE\u001b[39;49;00m\u001b[37m \u001b[39;49;00mTEMP\u001b[37m \u001b[39;49;00m\u001b[34mVIEW\u001b[39;49;00m\u001b[37m \u001b[39;49;00mapple_stock_holders\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m*\u001b[37m \u001b[39;49;00m\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mVALUES\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Vanguard Group, Inc. (The)'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1309785362\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m8\u001b[39;49;00m.\u001b[34m33\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m254059068265\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Blackrock Inc.'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1035008939\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m6\u001b[39;49;00m.\u001b[34m58\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m200760685161\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Berkshire Hathaway, Inc'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m915560382\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m5\u001b[39;49;00m.\u001b[34m82\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m177591248414\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'State Street Corporation'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m576281774\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m3\u001b[39;49;00m.\u001b[34m66\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m111781376406\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'FMR, LLC'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m311437576\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m98\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m60409546996\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Geode Capital Management, LLC'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m285171112\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m81\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m55314640942\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Price (T.Rowe) Associates Inc'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m234017381\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m49\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m45392351678\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Morgan Stanley'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m200615893\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m28\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m38913465010\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Northern Trust Corporation'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m173130542\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Mar 30, 2023'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m10\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m33582131443\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Norges Bank Investment Management'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m167374278\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'Dec 30, 2022'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m.\u001b[34m06\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m32465588907\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00mv1(holder_name,\u001b[37m \u001b[39;49;00mshares,\u001b[37m \u001b[39;49;00mdate_reported,\u001b[37m \u001b[39;49;00mpercent_out,\u001b[37m \u001b[39;49;00mvalue)\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mStoring data into temp view: apple_stock_holders\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:Storing data into temp view: apple_stock_holders\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "holders_dataframe.show(n=5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AJRyBYRaekJm",
        "outputId": "1baebc18-fe8a-42cf-a8a8-805b21ebd4b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+----------+-------------+-----------+------------+\n",
            "|         holder_name|    shares|date_reported|percent_out|       value|\n",
            "+--------------------+----------+-------------+-----------+------------+\n",
            "|Vanguard Group, I...|1309785362| Mar 30, 2023|       8.33|254059068265|\n",
            "|      Blackrock Inc.|1035008939| Mar 30, 2023|       6.58|200760685161|\n",
            "|Berkshire Hathawa...| 915560382| Mar 30, 2023|       5.82|177591248414|\n",
            "|State Street Corp...| 576281774| Mar 30, 2023|       3.66|111781376406|\n",
            "|            FMR, LLC| 311437576| Mar 30, 2023|       1.98| 60409546996|\n",
            "+--------------------+----------+-------------+-----------+------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "3. Plot"
      ],
      "metadata": {
        "id": "WbT_BWasdO_8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "holders_dataframe.ai.plot()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "tyWC-wpUemke",
        "outputId": "3a619b83-02d3-4dd3-d0b7-24cf8b7c903a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mHere is a Python code snippet that uses Plotly to visualize the data in the PySpark DataFrame `df`. This code assumes that you want to create a bar chart with `holder_name` on the x-axis and `shares` on the y-axis. \n",
            "\n",
            "\n",
            "```\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mgraph_objects\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mgo\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mpyspark\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36msql\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mfunctions\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m col\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert the Spark DataFrame to a Pandas DataFrame\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df = df.select(\u001b[33m\"\u001b[39;49;00m\u001b[33m*\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m).toPandas()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Create a bar chart\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig = go.Figure(data=go.Bar(x=pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mholder_name\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m], y=pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mshares\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m]))\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Add title and labels\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.update_layout(title=\u001b[33m'\u001b[39;49;00m\u001b[33mApple Stock Holders\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, xaxis_title=\u001b[33m'\u001b[39;49;00m\u001b[33mHolder Name\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, yaxis_title=\u001b[33m'\u001b[39;49;00m\u001b[33mShares\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Display the plot\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.show()\u001b[37m\u001b[39;49;00m\n",
            "```\n",
            "\n",
            "This code first converts the PySpark DataFrame to a Pandas DataFrame using the `toPandas()` method. Then, it creates a bar chart using Plotly's `go.Bar` function, with `holder_name` as the x-axis and `shares` as the y-axis. The `update_layout` method is used to add a title and labels to the plot. Finally, the `show` method is used to display the plot. \n",
            "\n",
            "Please note that the visualization type and the columns used for the x and y axes can be adjusted based on your specific needs.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:Here is a Python code snippet that uses Plotly to visualize the data in the PySpark DataFrame `df`. This code assumes that you want to create a bar chart with `holder_name` on the x-axis and `shares` on the y-axis. \n",
            "\n",
            "\n",
            "```\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mgraph_objects\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mgo\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mpyspark\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36msql\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mfunctions\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m col\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert the Spark DataFrame to a Pandas DataFrame\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df = df.select(\u001b[33m\"\u001b[39;49;00m\u001b[33m*\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m).toPandas()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Create a bar chart\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig = go.Figure(data=go.Bar(x=pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mholder_name\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m], y=pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mshares\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m]))\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Add title and labels\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.update_layout(title=\u001b[33m'\u001b[39;49;00m\u001b[33mApple Stock Holders\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, xaxis_title=\u001b[33m'\u001b[39;49;00m\u001b[33mHolder Name\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, yaxis_title=\u001b[33m'\u001b[39;49;00m\u001b[33mShares\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Display the plot\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.show()\u001b[37m\u001b[39;49;00m\n",
            "```\n",
            "\n",
            "This code first converts the PySpark DataFrame to a Pandas DataFrame using the `toPandas()` method. Then, it creates a bar chart using Plotly's `go.Bar` function, with `holder_name` as the x-axis and `shares` as the y-axis. The `update_layout` method is used to add a title and labels to the plot. Finally, the `show` method is used to display the plot. \n",
            "\n",
            "Please note that the visualization type and the columns used for the x and y axes can be adjusted based on your specific needs.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.18.2.min.js\"></script>                <div id=\"98ce94e1-edba-4fc9-b8e7-23e7ef9eed61\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"98ce94e1-edba-4fc9-b8e7-23e7ef9eed61\")) {                    Plotly.newPlot(                        \"98ce94e1-edba-4fc9-b8e7-23e7ef9eed61\",                        [{\"x\":[\"Vanguard Group, Inc. (The)\",\"Blackrock Inc.\",\"Berkshire Hathaway, Inc\",\"State Street Corporation\",\"FMR, LLC\",\"Geode Capital Management, LLC\",\"Price (T.Rowe) Associates Inc\",\"Morgan Stanley\",\"Northern Trust Corporation\",\"Norges Bank Investment Management\"],\"y\":[1309785362,1035008939,915560382,576281774,311437576,285171112,234017381,200615893,173130542,167374278],\"type\":\"bar\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"title\":{\"text\":\"Apple Stock Holders\"},\"xaxis\":{\"title\":{\"text\":\"Holder Name\"}},\"yaxis\":{\"title\":{\"text\":\"Shares\"}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('98ce94e1-edba-4fc9-b8e7-23e7ef9eed61');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "holders_dataframe.ai.plot(\"Pie chart for Apple's top holders, show their name and share percentages\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "V1TGiuSFeoQ3",
        "outputId": "b15218c7-084e-4531-fa2b-d63a5464b3ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mHere is a Python code snippet that uses PySpark and Plotly to visualize the result of `df` as a pie chart. This code assumes that the `percent_out` column represents the share percentages of each holder.\n",
            "\n",
            "\n",
            "```\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mgraph_objects\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mgo\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mpyspark\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36msql\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m SparkSession\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Assuming that SparkSession is already initialized\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "spark = SparkSession.builder.getOrCreate()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert the Spark DataFrame to a Pandas DataFrame\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df = df.toPandas()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Create a pie chart with Plotly\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig = go.Figure(data=[go.Pie(labels=pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mholder_name\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m], values=pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mpercent_out\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m], hole=\u001b[34m.3\u001b[39;49;00m)])\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Set the title of the chart\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.update_layout(title_text=\u001b[33m\"\u001b[39;49;00m\u001b[33mApple\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\u001b[33ms Top Holders Share Percentages\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Display the chart\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.show()\u001b[37m\u001b[39;49;00m\n",
            "```\n",
            "\n",
            "This code first converts the Spark DataFrame `df` to a Pandas DataFrame using the `toPandas()` method. Then, it creates a pie chart using Plotly's `go.Pie` function, with the holder names as labels and the share percentages as values. The `hole` parameter is set to 0.3 to create a donut-like pie chart. Finally, the chart is displayed using the `show()` method.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:Here is a Python code snippet that uses PySpark and Plotly to visualize the result of `df` as a pie chart. This code assumes that the `percent_out` column represents the share percentages of each holder.\n",
            "\n",
            "\n",
            "```\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mgraph_objects\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mgo\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mpyspark\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36msql\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m SparkSession\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Assuming that SparkSession is already initialized\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "spark = SparkSession.builder.getOrCreate()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert the Spark DataFrame to a Pandas DataFrame\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df = df.toPandas()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Create a pie chart with Plotly\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig = go.Figure(data=[go.Pie(labels=pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mholder_name\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m], values=pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mpercent_out\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m], hole=\u001b[34m.3\u001b[39;49;00m)])\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Set the title of the chart\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.update_layout(title_text=\u001b[33m\"\u001b[39;49;00m\u001b[33mApple\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\u001b[33ms Top Holders Share Percentages\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Display the chart\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.show()\u001b[37m\u001b[39;49;00m\n",
            "```\n",
            "\n",
            "This code first converts the Spark DataFrame `df` to a Pandas DataFrame using the `toPandas()` method. Then, it creates a pie chart using Plotly's `go.Pie` function, with the holder names as labels and the share percentages as values. The `hole` parameter is set to 0.3 to create a donut-like pie chart. Finally, the chart is displayed using the `show()` method.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.18.2.min.js\"></script>                <div id=\"207e1927-3209-40d8-816e-839732ce38ea\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"207e1927-3209-40d8-816e-839732ce38ea\")) {                    Plotly.newPlot(                        \"207e1927-3209-40d8-816e-839732ce38ea\",                        [{\"hole\":0.3,\"labels\":[\"Vanguard Group, Inc. (The)\",\"Blackrock Inc.\",\"Berkshire Hathaway, Inc\",\"State Street Corporation\",\"FMR, LLC\",\"Geode Capital Management, LLC\",\"Price (T.Rowe) Associates Inc\",\"Morgan Stanley\",\"Northern Trust Corporation\",\"Norges Bank Investment Management\"],\"values\":[8.33,6.58,5.82,3.66,1.98,1.81,1.49,1.28,1.1,1.06],\"type\":\"pie\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"title\":{\"text\":\"Apple's Top Holders Share Percentages\"}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('207e1927-3209-40d8-816e-839732ce38ea');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "top_holder_dataframe = holders_dataframe.ai.transform(\"name with the highest percentage, and its percentage\")\n",
        "top_holder_dataframe.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "InH3tPuqerT6",
        "outputId": "6bb3818b-122e-4242-8830-d29862d74108"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mSQL query for the transform:\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00mholder_name,\u001b[37m \u001b[39;49;00m\u001b[34mMAX\u001b[39;49;00m(percent_out)\u001b[37m \u001b[39;49;00m\u001b[34mas\u001b[39;49;00m\u001b[37m \u001b[39;49;00mmax_percent\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00mtemp_view_for_transform\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mGROUP\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mBY\u001b[39;49;00m\u001b[37m \u001b[39;49;00mholder_name\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mORDER\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mBY\u001b[39;49;00m\u001b[37m \u001b[39;49;00mmax_percent\u001b[37m \u001b[39;49;00m\u001b[34mDESC\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mLIMIT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:SQL query for the transform:\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00mholder_name,\u001b[37m \u001b[39;49;00m\u001b[34mMAX\u001b[39;49;00m(percent_out)\u001b[37m \u001b[39;49;00m\u001b[34mas\u001b[39;49;00m\u001b[37m \u001b[39;49;00mmax_percent\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00mtemp_view_for_transform\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mGROUP\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mBY\u001b[39;49;00m\u001b[37m \u001b[39;49;00mholder_name\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mORDER\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mBY\u001b[39;49;00m\u001b[37m \u001b[39;49;00mmax_percent\u001b[37m \u001b[39;49;00m\u001b[34mDESC\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mLIMIT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----------+\n",
            "|         holder_name|max_percent|\n",
            "+--------------------+-----------+\n",
            "|Vanguard Group, I...|       8.33|\n",
            "+--------------------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "4. Explain what the AI did"
      ],
      "metadata": {
        "id": "GX4yzzQSdRHz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "top_holder_dataframe.ai.explain()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "id": "BpCvHWg9ewAv",
        "outputId": "6c0bfaa0-05df-4fde-9288-9fdc83c732dd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'In summary, this dataframe is retrieving the holder name with the highest percentage of Apple stocks out of all holders. It presents the results sorted by the percentage of stocks in descending order and limits the result to the top holder.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "5. Verify the dataframe attributes by giving the expectatoin in natural language"
      ],
      "metadata": {
        "id": "4Ix8RQxUdUGr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "holders_dataframe.ai.verify(\"expect Apple's top holders have no more than 10% of shares\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vCF3_MiLezJ-",
        "outputId": "60d4d812-dd89-4b3d-c567-78c030f0d7f7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mGenerated code:\n",
            "\u001b[34mdef\u001b[39;49;00m \u001b[32mcheck_apple_top_holders\u001b[39;49;00m(df) -> \u001b[36mbool\u001b[39;49;00m:\u001b[37m\u001b[39;49;00m\n",
            "    \u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mpyspark\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36msql\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mfunctions\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m col\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "    \u001b[37m# Filter the DataFrame for rows where the holder_name is 'Apple'\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "    apple_df = df.filter(col(\u001b[33m'\u001b[39;49;00m\u001b[33mholder_name\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m) == \u001b[33m'\u001b[39;49;00m\u001b[33mApple\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "    \u001b[37m# Check if any row in the filtered DataFrame has a percent_out greater than 10\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "    \u001b[34mif\u001b[39;49;00m apple_df.filter(col(\u001b[33m'\u001b[39;49;00m\u001b[33mpercent_out\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m) > \u001b[34m10\u001b[39;49;00m).count() > \u001b[34m0\u001b[39;49;00m:\u001b[37m\u001b[39;49;00m\n",
            "        \u001b[34mreturn\u001b[39;49;00m \u001b[34mFalse\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "    \u001b[34melse\u001b[39;49;00m:\u001b[37m\u001b[39;49;00m\n",
            "        \u001b[34mreturn\u001b[39;49;00m \u001b[34mTrue\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "result = check_apple_top_holders(df)\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:Generated code:\n",
            "\u001b[34mdef\u001b[39;49;00m \u001b[32mcheck_apple_top_holders\u001b[39;49;00m(df) -> \u001b[36mbool\u001b[39;49;00m:\u001b[37m\u001b[39;49;00m\n",
            "    \u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mpyspark\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36msql\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mfunctions\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m col\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "    \u001b[37m# Filter the DataFrame for rows where the holder_name is 'Apple'\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "    apple_df = df.filter(col(\u001b[33m'\u001b[39;49;00m\u001b[33mholder_name\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m) == \u001b[33m'\u001b[39;49;00m\u001b[33mApple\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "    \u001b[37m# Check if any row in the filtered DataFrame has a percent_out greater than 10\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "    \u001b[34mif\u001b[39;49;00m apple_df.filter(col(\u001b[33m'\u001b[39;49;00m\u001b[33mpercent_out\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m) > \u001b[34m10\u001b[39;49;00m).count() > \u001b[34m0\u001b[39;49;00m:\u001b[37m\u001b[39;49;00m\n",
            "        \u001b[34mreturn\u001b[39;49;00m \u001b[34mFalse\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "    \u001b[34melse\u001b[39;49;00m:\u001b[37m\u001b[39;49;00m\n",
            "        \u001b[34mreturn\u001b[39;49;00m \u001b[34mTrue\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "result = check_apple_top_holders(df)\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0m\n",
            "Result: True\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:\n",
            "Result: True\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## More Exciting Use Cases\n",
        "\n",
        "Let's see more use cases in which we will explore how we can use natual language in data processing and analysis more elegantly with pyspark-ai."
      ],
      "metadata": {
        "id": "JuLgpzTGnqTi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = spark_ai.create_df(\"https://coinmarketcap.com/\", ['name', 'price', 'market_cap', 'circulating_supply'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QfS7PDK3hQMt",
        "outputId": "8acb86a1-61f5-42a8-d2cb-32b85f777ab4"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mParsing URL: https://coinmarketcap.com/\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:Parsing URL: https://coinmarketcap.com/\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mSQL query for the ingestion:\n",
            "\u001b[34mCREATE\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mOR\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mREPLACE\u001b[39;49;00m\u001b[37m \u001b[39;49;00mTEMP\u001b[37m \u001b[39;49;00m\u001b[34mVIEW\u001b[39;49;00m\u001b[37m \u001b[39;49;00mcryptocurrencies\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m*\u001b[37m \u001b[39;49;00m\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mVALUES\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Bitcoin'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$31,050.33'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$602.79B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'19,419,081 BTC'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Ethereum'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$1,957.54'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$235.29B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'120,219,234 ETH'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Tether'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$1.00'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$83.35B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'83,341,708,027 USDT'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'BNB'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$247.17'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$38.52B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'155,850,829 BNB'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'USD Coin'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$1.00'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$27.66B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'27,653,660,244 USDC'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'XRP'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$0.4886'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$25.53B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'52,254,289,650 XRP'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Cardano'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$0.2963'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$10.36B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'34,953,231,640 ADA'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Dogecoin'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$0.06809'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$9.53B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'140,024,896,384 DOGE'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Litecoin'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$106.61'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$7.81B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'73,282,214 LTC'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Solana'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$19.07'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$7.64B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'400,892,982 SOL'\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00mv1(name,\u001b[37m \u001b[39;49;00mprice,\u001b[37m \u001b[39;49;00mmarket_cap,\u001b[37m \u001b[39;49;00mcirculating_supply)\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:SQL query for the ingestion:\n",
            "\u001b[34mCREATE\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mOR\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mREPLACE\u001b[39;49;00m\u001b[37m \u001b[39;49;00mTEMP\u001b[37m \u001b[39;49;00m\u001b[34mVIEW\u001b[39;49;00m\u001b[37m \u001b[39;49;00mcryptocurrencies\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m*\u001b[37m \u001b[39;49;00m\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mVALUES\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Bitcoin'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$31,050.33'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$602.79B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'19,419,081 BTC'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Ethereum'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$1,957.54'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$235.29B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'120,219,234 ETH'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Tether'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$1.00'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$83.35B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'83,341,708,027 USDT'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'BNB'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$247.17'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$38.52B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'155,850,829 BNB'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'USD Coin'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$1.00'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$27.66B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'27,653,660,244 USDC'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'XRP'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$0.4886'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$25.53B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'52,254,289,650 XRP'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Cardano'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$0.2963'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$10.36B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'34,953,231,640 ADA'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Dogecoin'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$0.06809'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$9.53B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'140,024,896,384 DOGE'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Litecoin'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$106.61'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$7.81B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'73,282,214 LTC'\u001b[39;49;00m),\u001b[37m\u001b[39;49;00m\n",
            "(\u001b[33m'Solana'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$19.07'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'$7.64B'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m'400,892,982 SOL'\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00mv1(name,\u001b[37m \u001b[39;49;00mprice,\u001b[37m \u001b[39;49;00mmarket_cap,\u001b[37m \u001b[39;49;00mcirculating_supply)\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mStoring data into temp view: cryptocurrencies\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:Storing data into temp view: cryptocurrencies\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.show(n=5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Us6LczfjXLN",
        "outputId": "2e722665-061f-41b9-ef4f-3f8ac03a3fec"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+----------+----------+-------------------+\n",
            "|    name|     price|market_cap| circulating_supply|\n",
            "+--------+----------+----------+-------------------+\n",
            "| Bitcoin|$31,050.33|  $602.79B|     19,419,081 BTC|\n",
            "|Ethereum| $1,957.54|  $235.29B|    120,219,234 ETH|\n",
            "|  Tether|     $1.00|   $83.35B|83,341,708,027 USDT|\n",
            "|     BNB|   $247.17|   $38.52B|    155,850,829 BNB|\n",
            "|USD Coin|     $1.00|   $27.66B|27,653,660,244 USDC|\n",
            "+--------+----------+----------+-------------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.ai.plot('Bar chart with name and price')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "OH_NMjyY0MpS",
        "outputId": "7018823a-6ded-4110-b897-9a8f06c2d229"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mHere is a Python code snippet that uses Plotly to visualize the result of `df` as a bar chart with 'name' and 'price'. This code assumes that the 'price' column contains numerical values stored as strings.\n",
            "\n",
            "\n",
            "```\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mexpress\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpx\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mpandas\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpd\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert Spark DataFrame to Pandas DataFrame\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df = df.toPandas()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert price column to numeric\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mprice\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m] = pd.to_numeric(pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mprice\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m])\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Create a bar chart with 'name' and 'price'\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig = px.bar(pandas_df, x=\u001b[33m'\u001b[39;49;00m\u001b[33mname\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, y=\u001b[33m'\u001b[39;49;00m\u001b[33mprice\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Display the plot\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.show()\u001b[37m\u001b[39;49;00m\n",
            "```\n",
            "\n",
            "This code first converts the Spark DataFrame to a Pandas DataFrame using the `toPandas()` method. Then, it converts the 'price' column to numeric using the `pd.to_numeric()` function. This is necessary because Plotly requires numerical data for the y-axis of a bar chart. Finally, it creates a bar chart with 'name' on the x-axis and 'price' on the y-axis using the `px.bar()` function, and displays the plot using the `show()` method.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:Here is a Python code snippet that uses Plotly to visualize the result of `df` as a bar chart with 'name' and 'price'. This code assumes that the 'price' column contains numerical values stored as strings.\n",
            "\n",
            "\n",
            "```\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mexpress\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpx\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mpandas\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpd\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert Spark DataFrame to Pandas DataFrame\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df = df.toPandas()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert price column to numeric\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mprice\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m] = pd.to_numeric(pandas_df[\u001b[33m'\u001b[39;49;00m\u001b[33mprice\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m])\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Create a bar chart with 'name' and 'price'\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig = px.bar(pandas_df, x=\u001b[33m'\u001b[39;49;00m\u001b[33mname\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, y=\u001b[33m'\u001b[39;49;00m\u001b[33mprice\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Display the plot\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.show()\u001b[37m\u001b[39;49;00m\n",
            "```\n",
            "\n",
            "This code first converts the Spark DataFrame to a Pandas DataFrame using the `toPandas()` method. Then, it converts the 'price' column to numeric using the `pd.to_numeric()` function. This is necessary because Plotly requires numerical data for the y-axis of a bar chart. Finally, it creates a bar chart with 'name' on the x-axis and 'price' on the y-axis using the `px.bar()` function, and displays the plot using the `show()` method.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/_libs/lib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.maybe_convert_numeric\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: Unable to parse string \"$31,050.33\"",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-59-7f2eae941e7c>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mai\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Bar chart with name and price'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pyspark_ai/ai_utils.py\u001b[0m in \u001b[0;36mplot\u001b[0;34m(self, desc, cache)\u001b[0m\n\u001b[1;32m     62\u001b[0m                 \u001b[0mIf\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretrieves\u001b[0m \u001b[0mfresh\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mupdates\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \"\"\"\n\u001b[0;32m---> 64\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspark_ai\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdf_instance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdesc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mverify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdesc\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pyspark_ai/pyspark_ai.py\u001b[0m in \u001b[0;36mplot_df\u001b[0;34m(self, df, desc, cache)\u001b[0m\n\u001b[1;32m    376\u001b[0m         \u001b[0mcodeblocks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extract_code_blocks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    377\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mcode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcodeblocks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 378\u001b[0;31m             \u001b[0mexec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    379\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    380\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mverify_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdesc\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pyspark_ai/pyspark_ai.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/core/tools/numeric.py\u001b[0m in \u001b[0;36mto_numeric\u001b[0;34m(arg, errors, downcast, dtype_backend)\u001b[0m\n\u001b[1;32m    215\u001b[0m         \u001b[0mcoerce_numeric\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merrors\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\"ignore\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"raise\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m             values, new_mask = lib.maybe_convert_numeric(  # type: ignore[call-overload]  # noqa\n\u001b[0m\u001b[1;32m    218\u001b[0m                 \u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m                 \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/_libs/lib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.maybe_convert_numeric\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: Unable to parse string \"$31,050.33\" at position 0"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transformed_df = df.ai.transform(\n",
        "    \"\"\"\n",
        "    The price column is of string in the US currency format with comma separators, and denoted by US dollar sign.\n",
        "    You must process correctly with such format, and add a custom column 'price_float' that transforms the price column to float type\n",
        "    \"\"\"\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ELVvL8JOj5g_",
        "outputId": "e51a8660-3472-4ddd-bdfe-946fae585f64"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mSQL query for the transform:\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00m*,\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00m\u001b[34mCAST\u001b[39;49;00m(\u001b[34mREPLACE\u001b[39;49;00m(\u001b[34mREPLACE\u001b[39;49;00m(price,\u001b[37m \u001b[39;49;00m\u001b[33m'$'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m''\u001b[39;49;00m),\u001b[37m \u001b[39;49;00m\u001b[33m','\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m''\u001b[39;49;00m)\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[36mFLOAT\u001b[39;49;00m)\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00mprice_float\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00mtemp_view_for_transform\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:SQL query for the transform:\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00m*,\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00m\u001b[34mCAST\u001b[39;49;00m(\u001b[34mREPLACE\u001b[39;49;00m(\u001b[34mREPLACE\u001b[39;49;00m(price,\u001b[37m \u001b[39;49;00m\u001b[33m'$'\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m''\u001b[39;49;00m),\u001b[37m \u001b[39;49;00m\u001b[33m','\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m''\u001b[39;49;00m)\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[36mFLOAT\u001b[39;49;00m)\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00mprice_float\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00mtemp_view_for_transform\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transformed_df.show(n=5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wu8BOpXdw2MJ",
        "outputId": "0b41e566-58f2-48f8-c653-7912d30d4329"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+----------+----------+-------------------+-----------+\n",
            "|    name|     price|market_cap| circulating_supply|price_float|\n",
            "+--------+----------+----------+-------------------+-----------+\n",
            "| Bitcoin|$31,050.33|  $602.79B|     19,419,081 BTC|   31050.33|\n",
            "|Ethereum| $1,957.54|  $235.29B|    120,219,234 ETH|    1957.54|\n",
            "|  Tether|     $1.00|   $83.35B|83,341,708,027 USDT|        1.0|\n",
            "|     BNB|   $247.17|   $38.52B|    155,850,829 BNB|     247.17|\n",
            "|USD Coin|     $1.00|   $27.66B|27,653,660,244 USDC|        1.0|\n",
            "+--------+----------+----------+-------------------+-----------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transformed_df.ai.plot('Bar chart with name and float type price')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "zseWvqKZzySF",
        "outputId": "02e160e9-af73-47eb-c60a-4640136120cc"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mHere is the Python code to visualize the result of `df` using plotly. This code assumes that you have already created the `df` dataframe.\n",
            "\n",
            "\n",
            "```\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mexpress\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpx\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mpandas\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpd\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert Spark DataFrame to Pandas DataFrame\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df = df.toPandas()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Create a bar chart with name and float type price\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig = px.bar(pandas_df, x=\u001b[33m'\u001b[39;49;00m\u001b[33mname\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, y=\u001b[33m'\u001b[39;49;00m\u001b[33mprice_float\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, labels={\u001b[33m'\u001b[39;49;00m\u001b[33mx\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m:\u001b[33m'\u001b[39;49;00m\u001b[33mCryptocurrency\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, \u001b[33m'\u001b[39;49;00m\u001b[33my\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m:\u001b[33m'\u001b[39;49;00m\u001b[33mPrice\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m})\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Display the plot\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.show()\u001b[37m\u001b[39;49;00m\n",
            "```\n",
            "\n",
            "This code first converts the Spark DataFrame to a Pandas DataFrame. Then, it uses the `plotly.express` module to create a bar chart with the cryptocurrency names on the x-axis and the float type prices on the y-axis. Finally, it displays the plot.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:Here is the Python code to visualize the result of `df` using plotly. This code assumes that you have already created the `df` dataframe.\n",
            "\n",
            "\n",
            "```\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mexpress\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpx\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mpandas\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpd\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert Spark DataFrame to Pandas DataFrame\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df = df.toPandas()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Create a bar chart with name and float type price\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig = px.bar(pandas_df, x=\u001b[33m'\u001b[39;49;00m\u001b[33mname\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, y=\u001b[33m'\u001b[39;49;00m\u001b[33mprice_float\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, labels={\u001b[33m'\u001b[39;49;00m\u001b[33mx\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m:\u001b[33m'\u001b[39;49;00m\u001b[33mCryptocurrency\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, \u001b[33m'\u001b[39;49;00m\u001b[33my\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m:\u001b[33m'\u001b[39;49;00m\u001b[33mPrice\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m})\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Display the plot\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig.show()\u001b[37m\u001b[39;49;00m\n",
            "```\n",
            "\n",
            "This code first converts the Spark DataFrame to a Pandas DataFrame. Then, it uses the `plotly.express` module to create a bar chart with the cryptocurrency names on the x-axis and the float type prices on the y-axis. Finally, it displays the plot.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.18.2.min.js\"></script>                <div id=\"22646689-0598-4c84-ad7a-9ff6958b1e4d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"22646689-0598-4c84-ad7a-9ff6958b1e4d\")) {                    Plotly.newPlot(                        \"22646689-0598-4c84-ad7a-9ff6958b1e4d\",                        [{\"alignmentgroup\":\"True\",\"hovertemplate\":\"name=%{x}<br>price_float=%{y}<extra></extra>\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"pattern\":{\"shape\":\"\"}},\"name\":\"\",\"offsetgroup\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"textposition\":\"auto\",\"x\":[\"Bitcoin\",\"Ethereum\",\"Tether\",\"BNB\",\"USD Coin\",\"XRP\",\"Cardano\",\"Dogecoin\",\"Litecoin\",\"Solana\"],\"xaxis\":\"x\",\"y\":[31050.330078125,1957.5400390625,1.0,247.1699981689453,1.0,0.4885999858379364,0.2962999939918518,0.06808999925851822,106.61000061035156,19.06999969482422],\"yaxis\":\"y\",\"type\":\"bar\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"name\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"price_float\"}},\"legend\":{\"tracegroupgap\":0},\"margin\":{\"t\":60},\"barmode\":\"relative\"},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('22646689-0598-4c84-ad7a-9ff6958b1e4d');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transformed_df = transformed_df.ai.transform(\n",
        "    \"\"\"\n",
        "    The circulating_supply column is of string with comma separators, and trailing coin name symbol.\n",
        "    Add a custom column 'circulating_supply_long' that transforms the circulating_supply column to long type\n",
        "    \"\"\"\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9COXfXFDk5Yb",
        "outputId": "a6e054a1-26af-4aa4-bec7-34ffc6bfc818"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mSQL query for the transform:\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00m*,\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00m\u001b[34mCAST\u001b[39;49;00m(\u001b[34mREPLACE\u001b[39;49;00m(SUBSTRING_INDEX(circulating_supply,\u001b[37m \u001b[39;49;00m\u001b[33m' '\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m),\u001b[37m \u001b[39;49;00m\u001b[33m','\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m''\u001b[39;49;00m)\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00mLONG)\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00mcirculating_supply_long\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00mtemp_view_for_transform\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:SQL query for the transform:\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00m*,\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00m\u001b[34mCAST\u001b[39;49;00m(\u001b[34mREPLACE\u001b[39;49;00m(SUBSTRING_INDEX(circulating_supply,\u001b[37m \u001b[39;49;00m\u001b[33m' '\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[34m1\u001b[39;49;00m),\u001b[37m \u001b[39;49;00m\u001b[33m','\u001b[39;49;00m,\u001b[37m \u001b[39;49;00m\u001b[33m''\u001b[39;49;00m)\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00mLONG)\u001b[37m \u001b[39;49;00m\u001b[34mAS\u001b[39;49;00m\u001b[37m \u001b[39;49;00mcirculating_supply_long\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\n",
            "\u001b[37m    \u001b[39;49;00mtemp_view_for_transform\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transformed_df.show(n=5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Enbi6-k8kSfN",
        "outputId": "33629c62-a681-4ce8-8a0c-6c0c182ddd80"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+----------+----------+-------------------+-----------+-----------------------+\n",
            "|    name|     price|market_cap| circulating_supply|price_float|circulating_supply_long|\n",
            "+--------+----------+----------+-------------------+-----------+-----------------------+\n",
            "| Bitcoin|$31,050.33|  $602.79B|     19,419,081 BTC|   31050.33|               19419081|\n",
            "|Ethereum| $1,957.54|  $235.29B|    120,219,234 ETH|    1957.54|              120219234|\n",
            "|  Tether|     $1.00|   $83.35B|83,341,708,027 USDT|        1.0|            83341708027|\n",
            "|     BNB|   $247.17|   $38.52B|    155,850,829 BNB|     247.17|              155850829|\n",
            "|USD Coin|     $1.00|   $27.66B|27,653,660,244 USDC|        1.0|            27653660244|\n",
            "+--------+----------+----------+-------------------+-----------+-----------------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sorted_df = transformed_df.ai.transform('Sort by circulating_supply_long in a desc order')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X9-kY4Vvmlu8",
        "outputId": "32e7eed7-abe9-43cb-b840-c8a76017d3eb"
      },
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mSQL query for the transform:\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m*\u001b[37m \u001b[39;49;00m\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00mtemp_view_for_transform\u001b[37m \u001b[39;49;00m\u001b[34mORDER\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mBY\u001b[39;49;00m\u001b[37m \u001b[39;49;00mcirculating_supply_long\u001b[37m \u001b[39;49;00m\u001b[34mDESC\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:SQL query for the transform:\n",
            "\u001b[34mSELECT\u001b[39;49;00m\u001b[37m \u001b[39;49;00m*\u001b[37m \u001b[39;49;00m\u001b[34mFROM\u001b[39;49;00m\u001b[37m \u001b[39;49;00mtemp_view_for_transform\u001b[37m \u001b[39;49;00m\u001b[34mORDER\u001b[39;49;00m\u001b[37m \u001b[39;49;00m\u001b[34mBY\u001b[39;49;00m\u001b[37m \u001b[39;49;00mcirculating_supply_long\u001b[37m \u001b[39;49;00m\u001b[34mDESC\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sorted_df.show(n = 5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aIr-q-f5m0jm",
        "outputId": "75ef7c7e-a023-4968-a7da-a288b4d43dde"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------+----------+--------------------+-----------+-----------------------+\n",
            "|    name|   price|market_cap|  circulating_supply|price_float|circulating_supply_long|\n",
            "+--------+--------+----------+--------------------+-----------+-----------------------+\n",
            "|Dogecoin|$0.06809|    $9.53B|140,024,896,384 DOGE|    0.06809|           140024896384|\n",
            "|  Tether|   $1.00|   $83.35B| 83,341,708,027 USDT|        1.0|            83341708027|\n",
            "|     XRP| $0.4886|   $25.53B|  52,254,289,650 XRP|     0.4886|            52254289650|\n",
            "| Cardano| $0.2963|   $10.36B|  34,953,231,640 ADA|     0.2963|            34953231640|\n",
            "|USD Coin|   $1.00|   $27.66B| 27,653,660,244 USDC|        1.0|            27653660244|\n",
            "+--------+--------+----------+--------------------+-----------+-----------------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sorted_df.ai.plot('Bar chart with name and long type circulating supply value')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "_n6zDUDimqgC",
        "outputId": "b034c401-5338-41a9-8730-b63acd3b353e"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92mINFO: \u001b[0mHere is the Python code to visualize the result of `df` using plotly. This code will create a bar chart with the name of the cryptocurrency on the x-axis and the long type circulating supply value on the y-axis.\n",
            "\n",
            "\n",
            "```\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mexpress\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpx\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mio\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpio\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert Spark DataFrame to Pandas DataFrame\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df = df.toPandas()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Create a bar chart\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig = px.bar(pandas_df, x=\u001b[33m'\u001b[39;49;00m\u001b[33mname\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, y=\u001b[33m'\u001b[39;49;00m\u001b[33mcirculating_supply_long\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Display the plot\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pio.show(fig)\u001b[37m\u001b[39;49;00m\n",
            "```\n",
            "\n",
            "This code first converts the Spark DataFrame to a Pandas DataFrame using the `toPandas()` method. Then, it uses the `bar` function from the `plotly.express` module to create a bar chart. The x-axis is set to the 'name' column and the y-axis is set to the 'circulating_supply_long' column. Finally, the plot is displayed using the `show` function from the `plotly.io` module.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:spark_ai:Here is the Python code to visualize the result of `df` using plotly. This code will create a bar chart with the name of the cryptocurrency on the x-axis and the long type circulating supply value on the y-axis.\n",
            "\n",
            "\n",
            "```\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mexpress\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpx\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mplotly\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mio\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpio\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Convert Spark DataFrame to Pandas DataFrame\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pandas_df = df.toPandas()\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Create a bar chart\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "fig = px.bar(pandas_df, x=\u001b[33m'\u001b[39;49;00m\u001b[33mname\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, y=\u001b[33m'\u001b[39;49;00m\u001b[33mcirculating_supply_long\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m\u001b[39;49;00m\n",
            "\u001b[37m# Display the plot\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
            "pio.show(fig)\u001b[37m\u001b[39;49;00m\n",
            "```\n",
            "\n",
            "This code first converts the Spark DataFrame to a Pandas DataFrame using the `toPandas()` method. Then, it uses the `bar` function from the `plotly.express` module to create a bar chart. The x-axis is set to the 'name' column and the y-axis is set to the 'circulating_supply_long' column. Finally, the plot is displayed using the `show` function from the `plotly.io` module.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.18.2.min.js\"></script>                <div id=\"95206202-0999-42d3-bcbd-d9b3f523581f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"95206202-0999-42d3-bcbd-d9b3f523581f\")) {                    Plotly.newPlot(                        \"95206202-0999-42d3-bcbd-d9b3f523581f\",                        [{\"alignmentgroup\":\"True\",\"hovertemplate\":\"name=%{x}<br>circulating_supply_long=%{y}<extra></extra>\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"pattern\":{\"shape\":\"\"}},\"name\":\"\",\"offsetgroup\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"textposition\":\"auto\",\"x\":[\"Dogecoin\",\"Tether\",\"XRP\",\"Cardano\",\"USD Coin\",\"Solana\",\"BNB\",\"Ethereum\",\"Litecoin\",\"Bitcoin\"],\"xaxis\":\"x\",\"y\":[140024896384,83341708027,52254289650,34953231640,27653660244,400892982,155850829,120219234,73282214,19419081],\"yaxis\":\"y\",\"type\":\"bar\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"name\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"circulating_supply_long\"}},\"legend\":{\"tracegroupgap\":0},\"margin\":{\"t\":60},\"barmode\":\"relative\"},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('95206202-0999-42d3-bcbd-d9b3f523581f');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}