models/model_eval/visual_classifier_eval.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<td>\n",
        "   <a target=\"_blank\" href=\"https://www.clarifai.com/\" ><img src=\"https://upload.wikimedia.org/wikipedia/commons/b/bc/Clarifai_Logo_FC_Web.png\" width=256/></a>\n",
        "</td>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<td>\n",
        "<a href=\"https://colab.research.google.com/github/Clarifai/examples/blob/main/models/model_eval/visual_classifier_eval.ipynb\" target=\"_blank\"><img\n",
        "src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Colab\"></a>\n",
        "</td>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Model Evaluation For Visual Classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Getting Started"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Installation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "xGPEsIPcV3wC",
        "outputId": "e718f997-975e-455f-cc82-d7c7cdf95bc1"
      },
      "outputs": [],
      "source": [
        "!pip install clarifai"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gl0hz9qoVnd2"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "#Replace your PAT\n",
        "os.environ['CLARIFAI_PAT'] = \"PAT\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### For Colab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HmTpvh-tWcsi",
        "outputId": "a3752dcd-8904-47e8-b6a1-be5235f420d6"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/Clarifai/examples.git\n",
        "%cd /content/examples/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### For Local"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "git clone https://github.com/Clarifai/examples.git\n",
        "cd examples"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Setting Up App"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "zZZS9rT2V3SS",
        "outputId": "8db04e69-08e6-4633-b91f-cb21fbdf9adc"
      },
      "outputs": [],
      "source": [
        "from clarifai.client.user import User\n",
        "#replace your \"user_id\"\n",
        "client = User(user_id=\"user_id\")\n",
        "app = client.app(app_id=\"app_id\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Uploading Train Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "JAm_-0CVWTCo",
        "outputId": "3d429842-3319-408f-df13-28a0361ed7d3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:49:02 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.app:                                                        <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">app.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">311</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset created                                                             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                               <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"402e29d30bc9b2b99ef5ab90ef7a4d27\"</span>                                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                     <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:49:02\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.app:                                                        \u001b]8;id=343699;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\u001b\\\u001b[2mapp.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=335397;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\u001b\\\u001b[2m311\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset created                                                             \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                               \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                           \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"402e29d30bc9b2b99ef5ab90ef7a4d27\"\u001b[0m                                  \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                     \u001b[2m          \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.app:\n",
            "Dataset created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"402e29d30bc9b2b99ef5ab90ef7a4d27\"\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:49:02 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:  Getting dataset upload status<span style=\"color: #808000; text-decoration-color: #808000\">...</span>              <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#569\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">569</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:49:02\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:  Getting dataset upload status\u001b[33m...\u001b[0m              \u001b]8;id=508808;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=226590;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#569\u001b\\\u001b[2m569\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:Getting dataset upload status...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:                                                <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#110\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">110</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset Version created                                                 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                       <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"68dc61ec3f79de0e22ebc8cb248132c7\"</span>                              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:                                                \u001b]8;id=333;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=91527;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#110\u001b\\\u001b[2m110\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset Version created                                                 \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                           \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                       \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"68dc61ec3f79de0e22ebc8cb248132c7\"\u001b[0m                              \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                 \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:\n",
            "Dataset Version created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"68dc61ec3f79de0e22ebc8cb248132c7\"\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:49:02 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:  Crunching the dataset metrics. Please wait<span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">593</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:49:02\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:  Crunching the dataset metrics. Please wait\u001b[33m...\u001b[0m \u001b]8;id=319117;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=610141;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\u001b\\\u001b[2m593\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:Crunching the dataset metrics. Please wait...\n",
            "Uploading Dataset: 100%|██████████| 1/1 [00:25<00:00, 25.73s/it]\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:49:29 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:  Getting dataset upload status<span style=\"color: #808000; text-decoration-color: #808000\">...</span>              <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#569\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">569</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:49:29\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:  Getting dataset upload status\u001b[33m...\u001b[0m              \u001b]8;id=630646;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=71698;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#569\u001b\\\u001b[2m569\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:Getting dataset upload status...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:49:30 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:                                                <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#110\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">110</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset Version created                                                 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                       <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"818191a9d04709afc5b70b1f2ca001d4\"</span>                              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:49:30\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:                                                \u001b]8;id=362004;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=784258;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#110\u001b\\\u001b[2m110\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset Version created                                                 \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                           \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                       \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"818191a9d04709afc5b70b1f2ca001d4\"\u001b[0m                              \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                 \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:\n",
            "Dataset Version created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"818191a9d04709afc5b70b1f2ca001d4\"\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:49:30 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:  Crunching the dataset metrics. Please wait<span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">593</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:49:30\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:  Crunching the dataset metrics. Please wait\u001b[33m...\u001b[0m \u001b]8;id=256796;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=348203;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\u001b\\\u001b[2m593\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:Crunching the dataset metrics. Please wait...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080\">╭──────────────────────────────────────────── </span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\"> Dataset Upload Summary</span><span style=\"color: #000080; text-decoration-color: #000080\"> ────────────────────────────────────────────╮</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> <span style=\"color: #008080; text-decoration-color: #008080\">Inputs Progress:</span>      <span style=\"color: #729c1f; text-decoration-color: #729c1f\">━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━</span> <span style=\"color: #800080; text-decoration-color: #800080\">100%</span> <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> <span style=\"color: #008080; text-decoration-color: #008080\">Annotations Progress:</span> <span style=\"color: #729c1f; text-decoration-color: #729c1f\">━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━</span> <span style=\"color: #800080; text-decoration-color: #800080\">100%</span> <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━┓                                                     <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┃<span style=\"color: #000080; text-decoration-color: #000080\"> dataset_id     </span>┃<span style=\"color: #000080; text-decoration-color: #000080\"> user_id      </span>┃<span style=\"color: #000080; text-decoration-color: #000080\"> app_id                  </span>┃                                                     <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━┩                                                     <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> │ image_dataset3 │ 8tzpjy1a841y │ visual_classifier_eval2 │                                                     <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> └────────────────┴──────────────┴─────────────────────────┘                                                     <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">╭────────────────────────────────────────── </span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\"> Dataset Metrics Comparison</span><span style=\"color: #000080; text-decoration-color: #000080\"> ──────────────────────────────────────────╮</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> <span style=\"font-style: italic\">                     </span><span style=\"color: #008080; text-decoration-color: #008080; font-style: italic\">Local Dataset</span><span style=\"font-style: italic\">                                         </span><span style=\"color: #008080; text-decoration-color: #008080; font-style: italic\">Uploaded Dataset</span><span style=\"font-style: italic\">                    </span> <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┃<span style=\"color: #000080; text-decoration-color: #000080\"> Inputs Count </span>┃<span style=\"color: #000080; text-decoration-color: #000080\"> Annotations Count                    </span>┃┃<span style=\"color: #000080; text-decoration-color: #000080\"> Inputs Count </span>┃<span style=\"color: #000080; text-decoration-color: #000080\"> Annotations Count                     </span>┃ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> │ 20           │ {'concepts': 20, 'bboxes': 0,        ││ 20           │ {'concepts': 20, 'bboxes': 0,         │ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> │              │ 'polygons': 0}                       ││              │ 'polygons': 0}                        │ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> └──────────────┴──────────────────────────────────────┘└──────────────┴───────────────────────────────────────┘ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[34m╭─\u001b[0m\u001b[34m───────────────────────────────────────────\u001b[0m\u001b[34m \u001b[0m\u001b[1;34m Dataset Upload Summary\u001b[0m\u001b[34m \u001b[0m\u001b[34m───────────────────────────────────────────\u001b[0m\u001b[34m─╮\u001b[0m\n",
              "\u001b[34m│\u001b[0m \u001b[36mInputs Progress:\u001b[0m      \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m \u001b[36mAnnotations Progress:\u001b[0m \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━┓                                                     \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┃\u001b[34m \u001b[0m\u001b[34mdataset_id    \u001b[0m\u001b[34m \u001b[0m┃\u001b[34m \u001b[0m\u001b[34muser_id     \u001b[0m\u001b[34m \u001b[0m┃\u001b[34m \u001b[0m\u001b[34mapp_id                 \u001b[0m\u001b[34m \u001b[0m┃                                                     \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━┩                                                     \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m │ image_dataset3 │ 8tzpjy1a841y │ visual_classifier_eval2 │                                                     \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m └────────────────┴──────────────┴─────────────────────────┘                                                     \u001b[34m│\u001b[0m\n",
              "\u001b[34m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
              "\u001b[34m╭─\u001b[0m\u001b[34m─────────────────────────────────────────\u001b[0m\u001b[34m \u001b[0m\u001b[1;34m Dataset Metrics Comparison\u001b[0m\u001b[34m \u001b[0m\u001b[34m─────────────────────────────────────────\u001b[0m\u001b[34m─╮\u001b[0m\n",
              "\u001b[34m│\u001b[0m \u001b[3m                     \u001b[0m\u001b[3;36mLocal Dataset\u001b[0m\u001b[3m                     \u001b[0m\u001b[3m                    \u001b[0m\u001b[3;36mUploaded Dataset\u001b[0m\u001b[3m                    \u001b[0m \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┃\u001b[34m \u001b[0m\u001b[34mInputs Count\u001b[0m\u001b[34m \u001b[0m┃\u001b[34m \u001b[0m\u001b[34mAnnotations Count                   \u001b[0m\u001b[34m \u001b[0m┃┃\u001b[34m \u001b[0m\u001b[34mInputs Count\u001b[0m\u001b[34m \u001b[0m┃\u001b[34m \u001b[0m\u001b[34mAnnotations Count                    \u001b[0m\u001b[34m \u001b[0m┃ \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m │ 20           │ {'concepts': 20, 'bboxes': 0,        ││ 20           │ {'concepts': 20, 'bboxes': 0,         │ \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m │              │ 'polygons': 0}                       ││              │ 'polygons': 0}                        │ \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m └──────────────┴──────────────────────────────────────┘└──────────────┴───────────────────────────────────────┘ \u001b[34m│\u001b[0m\n",
              "\u001b[34m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from clarifai.datasets.upload.utils import load_module_dataloader\n",
        "\n",
        "\n",
        "# Construct the path to the dataset folder\n",
        "module_path = os.path.join(os.getcwd().split('/models/model_train')[0], 'datasets/upload/image_classification/food-101')\n",
        "\n",
        "# Load the dataloader module using the provided function from your module\n",
        "food101_dataloader = load_module_dataloader(module_path)\n",
        "\n",
        "# Create a Clarifai dataset with the specified dataset_id (\"image_dataset\")\n",
        "dataset = app.create_dataset(dataset_id=\"image_dataset\")\n",
        "\n",
        "# Upload the dataset using the provided dataloader and get the upload status\n",
        "dataset.upload_dataset(dataloader=food101_dataloader, get_upload_status=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Setup Trained Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "j_8AKL8ZWjxw",
        "outputId": "81ce5334-1eb3-45d2-8765-82a7398c7ffa"
      },
      "outputs": [],
      "source": [
        "MODEL_ID = \"model_classifier\"\n",
        "# Using a trained model from the app\n",
        "model = app.model(model_id=MODEL_ID)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Evaluating With Training Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "HE2fPrlDroQE",
        "outputId": "160de332-5882-41ca-a4d8-bab3d344ddf2"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:49:44 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.model:                                                    <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">model.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">676</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Model evaluation in progress. Kindly allow a few minutes for completion.  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Processing time may vary based on the model and dataset sizes.            <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:49:44\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.model:                                                    \u001b]8;id=465178;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\u001b\\\u001b[2mmodel.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=164539;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\u001b\\\u001b[2m676\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Model evaluation in progress. Kindly allow a few minutes for completion.  \u001b[2m            \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         Processing time may vary based on the model and dataset sizes.            \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.model:\n",
            "Model evaluation in progress. Kindly allow a few minutes for completion. Processing time may vary based on the model and dataset sizes.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[status {\n",
              "  code: MODEL_QUEUED_FOR_EVALUATION\n",
              "  description: \"Model is queued for evaluation.\"\n",
              "}\n",
              "id: \"seven\"\n",
              "user_id: \"8tzpjy1a841y\"\n",
              "app_id: \"visual_classifier_eval2\"\n",
              "]"
            ]
          },
          "execution_count": 51,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.evaluate(dataset_id='image_dataset',eval_id='one')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dGLnmuBQuGEy"
      },
      "outputs": [],
      "source": [
        "result=model.get_eval_by_id(eval_id=\"one\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r7MLIx1PuGCG",
        "outputId": "e9632158-591f-4643-c5c2-f8945c97ca5c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "macro_avg_roc_auc: 0.9200000166893005\n",
              "macro_std_roc_auc: 0.03399345278739929\n",
              "macro_avg_f1_score: 0.6682435274124146\n",
              "macro_std_f1_score: 0.08137183636426926\n",
              "macro_avg_precision: 0.53125\n",
              "macro_avg_recall: 0.949999988079071"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result.summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Uploading Test Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "d39ktpehcqpu",
        "outputId": "736e8d02-3926-4867-fd89-aa8b54b017b8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:45:24 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.app:                                                        <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">app.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">311</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset created                                                             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                               <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"fefb31f04c87d250d3b7b9bcdc642b79\"</span>                                  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                     <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">          </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:45:24\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.app:                                                        \u001b]8;id=351888;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py\u001b\\\u001b[2mapp.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=284078;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/app.py#311\u001b\\\u001b[2m311\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset created                                                             \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                               \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                           \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"fefb31f04c87d250d3b7b9bcdc642b79\"\u001b[0m                                  \u001b[2m          \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                     \u001b[2m          \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.app:\n",
            "Dataset created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"fefb31f04c87d250d3b7b9bcdc642b79\"\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:45:24 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:  Getting dataset upload status<span style=\"color: #808000; text-decoration-color: #808000\">...</span>              <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#569\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">569</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:45:24\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:  Getting dataset upload status\u001b[33m...\u001b[0m              \u001b]8;id=17815;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=622810;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#569\u001b\\\u001b[2m569\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:Getting dataset upload status...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:                                                <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#110\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">110</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset Version created                                                 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                       <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"893822d3b09dc01387f43c8d1596a388\"</span>                              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:                                                \u001b]8;id=179129;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=41833;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#110\u001b\\\u001b[2m110\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset Version created                                                 \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                           \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                       \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"893822d3b09dc01387f43c8d1596a388\"\u001b[0m                              \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                 \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:\n",
            "Dataset Version created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"893822d3b09dc01387f43c8d1596a388\"\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:45:24 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:  Crunching the dataset metrics. Please wait<span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">593</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:45:24\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:  Crunching the dataset metrics. Please wait\u001b[33m...\u001b[0m \u001b]8;id=37564;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=576878;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\u001b\\\u001b[2m593\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:Crunching the dataset metrics. Please wait...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:45:26 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:  Crunching the dataset metrics. Please wait<span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">593</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:45:26\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:  Crunching the dataset metrics. Please wait\u001b[33m...\u001b[0m \u001b]8;id=56744;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=310162;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\u001b\\\u001b[2m593\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:Crunching the dataset metrics. Please wait...\n",
            "Uploading Dataset: 100%|██████████| 1/1 [00:02<00:00,  2.64s/it]\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:45:31 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:  Getting dataset upload status<span style=\"color: #808000; text-decoration-color: #808000\">...</span>              <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#569\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">569</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:45:31\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:  Getting dataset upload status\u001b[33m...\u001b[0m              \u001b]8;id=553149;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=464079;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#569\u001b\\\u001b[2m569\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:Getting dataset upload status...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:                                                <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#110\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">110</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Dataset Version created                                                 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         code: SUCCESS                                                           <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         description: <span style=\"color: #008000; text-decoration-color: #008000\">\"Ok\"</span>                                                       <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         req_id: <span style=\"color: #008000; text-decoration-color: #008000\">\"ff254801c4f2211a09fb901aa18e0e21\"</span>                              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>                                                                                 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:                                                \u001b]8;id=549079;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=332413;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#110\u001b\\\u001b[2m110\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Dataset Version created                                                 \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         code: SUCCESS                                                           \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         description: \u001b[32m\"Ok\"\u001b[0m                                                       \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         req_id: \u001b[32m\"ff254801c4f2211a09fb901aa18e0e21\"\u001b[0m                              \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m                                                                                 \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:\n",
            "Dataset Version created\n",
            "code: SUCCESS\n",
            "description: \"Ok\"\n",
            "req_id: \"ff254801c4f2211a09fb901aa18e0e21\"\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:45:31 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:  Crunching the dataset metrics. Please wait<span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">593</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:45:31\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:  Crunching the dataset metrics. Please wait\u001b[33m...\u001b[0m \u001b]8;id=303259;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=318955;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\u001b\\\u001b[2m593\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:Crunching the dataset metrics. Please wait...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:45:33 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.dataset:  Crunching the dataset metrics. Please wait<span style=\"color: #808000; text-decoration-color: #808000\">...</span> <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">dataset.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">593</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:45:33\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.dataset:  Crunching the dataset metrics. Please wait\u001b[33m...\u001b[0m \u001b]8;id=510470;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py\u001b\\\u001b[2mdataset.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=865009;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/dataset.py#593\u001b\\\u001b[2m593\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.dataset:Crunching the dataset metrics. Please wait...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080\">╭──────────────────────────────────────────── </span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\"> Dataset Upload Summary</span><span style=\"color: #000080; text-decoration-color: #000080\"> ────────────────────────────────────────────╮</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> <span style=\"color: #008080; text-decoration-color: #008080\">Inputs Progress:</span>      <span style=\"color: #729c1f; text-decoration-color: #729c1f\">━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━</span> <span style=\"color: #800080; text-decoration-color: #800080\">100%</span> <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> <span style=\"color: #008080; text-decoration-color: #008080\">Annotations Progress:</span> <span style=\"color: #729c1f; text-decoration-color: #729c1f\">━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━</span> <span style=\"color: #800080; text-decoration-color: #800080\">100%</span> <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━┓                                                     <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┃<span style=\"color: #000080; text-decoration-color: #000080\"> dataset_id     </span>┃<span style=\"color: #000080; text-decoration-color: #000080\"> user_id      </span>┃<span style=\"color: #000080; text-decoration-color: #000080\"> app_id                  </span>┃                                                     <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━┩                                                     <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> │ image_dataset2 │ 8tzpjy1a841y │ visual_classifier_eval2 │                                                     <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> └────────────────┴──────────────┴─────────────────────────┘                                                     <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">╭────────────────────────────────────────── </span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\"> Dataset Metrics Comparison</span><span style=\"color: #000080; text-decoration-color: #000080\"> ──────────────────────────────────────────╮</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> <span style=\"font-style: italic\">                     </span><span style=\"color: #008080; text-decoration-color: #008080; font-style: italic\">Local Dataset</span><span style=\"font-style: italic\">                                         </span><span style=\"color: #008080; text-decoration-color: #008080; font-style: italic\">Uploaded Dataset</span><span style=\"font-style: italic\">                    </span> <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┃<span style=\"color: #000080; text-decoration-color: #000080\"> Inputs Count </span>┃<span style=\"color: #000080; text-decoration-color: #000080\"> Annotations Count                    </span>┃┃<span style=\"color: #000080; text-decoration-color: #000080\"> Inputs Count </span>┃<span style=\"color: #000080; text-decoration-color: #000080\"> Annotations Count                     </span>┃ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> ┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> │ 4            │ {'concepts': 4, 'bboxes': 0,         ││ 4            │ {'concepts': 4, 'bboxes': 0,          │ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> │              │ 'polygons': 0}                       ││              │ 'polygons': 0}                        │ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">│</span> └──────────────┴──────────────────────────────────────┘└──────────────┴───────────────────────────────────────┘ <span style=\"color: #000080; text-decoration-color: #000080\">│</span>\n",
              "<span style=\"color: #000080; text-decoration-color: #000080\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[34m╭─\u001b[0m\u001b[34m───────────────────────────────────────────\u001b[0m\u001b[34m \u001b[0m\u001b[1;34m Dataset Upload Summary\u001b[0m\u001b[34m \u001b[0m\u001b[34m───────────────────────────────────────────\u001b[0m\u001b[34m─╮\u001b[0m\n",
              "\u001b[34m│\u001b[0m \u001b[36mInputs Progress:\u001b[0m      \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m \u001b[36mAnnotations Progress:\u001b[0m \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━┓                                                     \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┃\u001b[34m \u001b[0m\u001b[34mdataset_id    \u001b[0m\u001b[34m \u001b[0m┃\u001b[34m \u001b[0m\u001b[34muser_id     \u001b[0m\u001b[34m \u001b[0m┃\u001b[34m \u001b[0m\u001b[34mapp_id                 \u001b[0m\u001b[34m \u001b[0m┃                                                     \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━┩                                                     \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m │ image_dataset2 │ 8tzpjy1a841y │ visual_classifier_eval2 │                                                     \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m └────────────────┴──────────────┴─────────────────────────┘                                                     \u001b[34m│\u001b[0m\n",
              "\u001b[34m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
              "\u001b[34m╭─\u001b[0m\u001b[34m─────────────────────────────────────────\u001b[0m\u001b[34m \u001b[0m\u001b[1;34m Dataset Metrics Comparison\u001b[0m\u001b[34m \u001b[0m\u001b[34m─────────────────────────────────────────\u001b[0m\u001b[34m─╮\u001b[0m\n",
              "\u001b[34m│\u001b[0m \u001b[3m                     \u001b[0m\u001b[3;36mLocal Dataset\u001b[0m\u001b[3m                     \u001b[0m\u001b[3m                    \u001b[0m\u001b[3;36mUploaded Dataset\u001b[0m\u001b[3m                    \u001b[0m \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┃\u001b[34m \u001b[0m\u001b[34mInputs Count\u001b[0m\u001b[34m \u001b[0m┃\u001b[34m \u001b[0m\u001b[34mAnnotations Count                   \u001b[0m\u001b[34m \u001b[0m┃┃\u001b[34m \u001b[0m\u001b[34mInputs Count\u001b[0m\u001b[34m \u001b[0m┃\u001b[34m \u001b[0m\u001b[34mAnnotations Count                    \u001b[0m\u001b[34m \u001b[0m┃ \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m ┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m │ 4            │ {'concepts': 4, 'bboxes': 0,         ││ 4            │ {'concepts': 4, 'bboxes': 0,          │ \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m │              │ 'polygons': 0}                       ││              │ 'polygons': 0}                        │ \u001b[34m│\u001b[0m\n",
              "\u001b[34m│\u001b[0m └──────────────┴──────────────────────────────────────┘└──────────────┴───────────────────────────────────────┘ \u001b[34m│\u001b[0m\n",
              "\u001b[34m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n",
              "\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Load the dataloader module using the provided function from your module\n",
        "# Give the correct path to dataset\n",
        "food101_dataloader = load_module_dataloader(\"/content/examples/datasets/upload/data/images_test\")\n",
        "\n",
        "# Create a Clarifai dataset with the specified dataset_id (\"image_dataset\")\n",
        "test_dataset = app.create_dataset(dataset_id=\"image_dataset2\")\n",
        "\n",
        "# Upload the dataset using the provided dataloader and get the upload status\n",
        "test_dataset.upload_dataset(dataloader=food101_dataloader, get_upload_status=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Evaluating With Testing Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "kfw86VKfgFL7",
        "outputId": "eb0eab97-53da-4d54-8466-342d23f63e5d"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:46:32 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.client.model:                                                    <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">model.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">676</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Model evaluation in progress. Kindly allow a few minutes for completion.  <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         Processing time may vary based on the model and dataset sizes.            <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">            </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:46:32\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.client.model:                                                    \u001b]8;id=799020;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py\u001b\\\u001b[2mmodel.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=878520;file:///usr/local/lib/python3.10/dist-packages/clarifai/client/model.py#676\u001b\\\u001b[2m676\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         Model evaluation in progress. Kindly allow a few minutes for completion.  \u001b[2m            \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         Processing time may vary based on the model and dataset sizes.            \u001b[2m            \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.client.model:\n",
            "Model evaluation in progress. Kindly allow a few minutes for completion. Processing time may vary based on the model and dataset sizes.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[status {\n",
              "  code: MODEL_QUEUED_FOR_EVALUATION\n",
              "  description: \"Model is queued for evaluation.\"\n",
              "}\n",
              "id: \"six\"\n",
              "user_id: \"8tzpjy1a841y\"\n",
              "app_id: \"visual_classifier_eval2\"\n",
              "]"
            ]
          },
          "execution_count": 40,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.evaluate(dataset_id='image_dataset2',eval_id='two')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "43LB0CBUgQPx"
      },
      "outputs": [],
      "source": [
        "result=model.get_eval_by_id(\"two\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mQ-DIrHMgQNd",
        "outputId": "4dc87fc9-9fc1-4c31-e19f-9fdb51fd1e16"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "macro_avg_roc_auc: 1.0\n",
              "macro_avg_f1_score: 0.7916666865348816\n",
              "macro_std_f1_score: 0.21650634706020355\n",
              "macro_avg_precision: 0.7083333134651184\n",
              "macro_avg_recall: 1.0"
            ]
          },
          "execution_count": 46,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result.summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Compare Results From Train & Test Datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "id": "FwyaDo4igQLB",
        "outputId": "3ddd54f2-bb9c-49cf-f526-c9af40e0eef0"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:50:39 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  Initializing models<span style=\"color: #808000; text-decoration-color: #808000\">...</span>                      <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#91\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">91</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:50:39\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  Initializing models\u001b[33m...\u001b[0m                      \u001b]8;id=231084;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=368119;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#91\u001b\\\u001b[2m91\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:Initializing models...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  *                                          <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#100\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">100</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         visual_classifier_eval2/model_classifier/48ed4                         <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">               </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  *                                          \u001b]8;id=649328;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=252374;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#100\u001b\\\u001b[2m100\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         visual_classifier_eval2/model_classifier/48ed4                         \u001b[2m               \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:* visual_classifier_eval2/model_classifier/48ed4\n"
          ]
        }
      ],
      "source": [
        "from clarifai.utils.evaluation import EvalResultCompare\n",
        "\n",
        "eval_result = EvalResultCompare(models=[model],datasets= [dataset,test_dataset])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "NiwLg9kVgQIh",
        "outputId": "c4f3552a-5159-4be4-ad47-473d1c21140e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:50:56 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation:  Running `detailed_summary`                 <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">__init__.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#123\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">123</span></a>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:50:56\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation:  Running `detailed_summary`                 \u001b]8;id=306859;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=187618;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/__init__.py#123\u001b\\\u001b[2m123\u001b[0m\u001b]8;;\u001b\\\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation:Running `detailed_summary`\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:50:56 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         visual_classifier_eval2/model_classifier/48ed4: retrieving              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: image_dataset3             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:50:56\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=401378;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=870633;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         visual_classifier_eval2/model_classifier/48ed4: retrieving              \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: image_dataset3             \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         visual_classifier_eval2/model_classifier/48ed4: retrieving              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: image_dataset3             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=392278;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=992902;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         visual_classifier_eval2/model_classifier/48ed4: retrieving              \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: image_dataset3             \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation.helpers:Model visual_classifier_eval2/model_classifier/48ed4: retrieving {'binary_metrics': True} metrics of dataset: image_dataset3\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:50:57 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         visual_classifier_eval2/model_classifier/48ed4: retrieving              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: image_dataset2             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:50:57\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=861959;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=332326;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         visual_classifier_eval2/model_classifier/48ed4: retrieving              \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: image_dataset2             \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">2024-04-04 09:50:57 </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO    </span> clarifai.utils.evaluation.helpers:  Model                               <a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">helpers.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">138</span></a>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         visual_classifier_eval2/model_classifier/48ed4: retrieving              <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'binary_metrics'</span>: <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">}</span> metrics of dataset: image_dataset2             <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">              </span>\n",
              "</pre>\n"
            ],
            "text/plain": [
              "\u001b[2;36m2024-04-04 09:50:57\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m clarifai.utils.evaluation.helpers:  Model                               \u001b]8;id=783787;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py\u001b\\\u001b[2mhelpers.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=194342;file:///usr/local/lib/python3.10/dist-packages/clarifai/utils/evaluation/helpers.py#138\u001b\\\u001b[2m138\u001b[0m\u001b]8;;\u001b\\\n",
              "\u001b[2;36m                    \u001b[0m         visual_classifier_eval2/model_classifier/48ed4: retrieving              \u001b[2m              \u001b[0m\n",
              "\u001b[2;36m                    \u001b[0m         \u001b[1m{\u001b[0m\u001b[32m'binary_metrics'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m metrics of dataset: image_dataset2             \u001b[2m              \u001b[0m\n"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:clarifai.utils.evaluation.helpers:Model visual_classifier_eval2/model_classifier/48ed4: retrieving {'binary_metrics': True} metrics of dataset: image_dataset2\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "(        Concept  Accuracy (ROC AUC)  Total Labeled  Total Predicted  \\\n",
              " 0      id-ramen               0.933              5               12   \n",
              " 0  id-prime_rib               0.960              5                5   \n",
              " 0  id-hamburger               0.920              5                8   \n",
              " 0   id-beignets               0.867              5               12   \n",
              " 0      id-ramen               1.000              1                3   \n",
              " 0  id-prime_rib               1.000              1                1   \n",
              " 0  id-hamburger               1.000              1                1   \n",
              " 0   id-beignets               1.000              1                1   \n",
              " \n",
              "    True Positives  False Negatives  False Positives  Recall  Precision  \\\n",
              " 0               5                0                7     1.0     0.4167   \n",
              " 0               4                1                1     0.8     0.6667   \n",
              " 0               5                0                3     1.0     0.6250   \n",
              " 0               5                0                7     1.0     0.4167   \n",
              " 0               1                0                2     1.0     0.3333   \n",
              " 0               1                0                0     1.0     1.0000   \n",
              " 0               1                0                0     1.0     1.0000   \n",
              " 0               1                0                0     1.0     0.5000   \n",
              " \n",
              "          F1         Dataset  \n",
              " 0  0.588269  image_dataset3  \n",
              " 0  0.727293  image_dataset3  \n",
              " 0  0.769231  image_dataset3  \n",
              " 0  0.588269  image_dataset3  \n",
              " 0  0.499962  image_dataset2  \n",
              " 0  1.000000  image_dataset2  \n",
              " 0  1.000000  image_dataset2  \n",
              " 0  0.666667  image_dataset2  ,\n",
              "             Total Concept  Accuracy (ROC AUC)  Total Labeled  Total Predicted  \\\n",
              " 0  Dataset:image_dataset3                0.92             20               37   \n",
              " 0  Dataset:image_dataset2                1.00              4                6   \n",
              " \n",
              "    True Positives  False Negatives  False Positives  Recall  Precision  \\\n",
              " 0              19                1               18    0.95   0.531275   \n",
              " 0               4                0                2    1.00   0.708325   \n",
              " \n",
              "          F1  \n",
              " 0  0.681455  \n",
              " 0  0.829263  )"
            ]
          },
          "execution_count": 59,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "eval_result.detailed_summary()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}