pl_compare_demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "%pip install pl-compare==0.3.6"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hLgpdsifsg0H",
        "outputId": "195d66e6-daab-4b77-9334-16b3672439a9"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pl-compare==0.3.6 in /usr/local/lib/python3.10/dist-packages (0.3.6)\n",
            "Requirement already satisfied: polars<0.20.0,>=0.19.8 in /usr/local/lib/python3.10/dist-packages (from pl-compare==0.3.6) (0.19.19)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import polars as pl\n",
        "from pl_compare import compare\n",
        "\n",
        "netflix_original = pl.read_csv(\"https://github.com/concur1/pl_compare/raw/main/example_data/netflix_titles.csv\")\n",
        "netflix_updated = pl.read_csv(\"https://github.com/concur1/pl_compare/raw/main/example_data/netflix_titles-corrupted.csv\")\n",
        "\n",
        "from pl_compare.compare import compare\n",
        "\n",
        "diff=compare(join_columns=['show_id'], base_df=netflix_original, compare_df=netflix_updated)\n"
      ],
      "metadata": {
        "id": "L-xfsyBvsZzh"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title A text report holding all the value differences\n",
        "\n",
        "diff.report()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "08dO8L0JFQD0",
        "outputId": "dff972f2-b3f5-4866-e782-499bb0e98804"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "--------------------------------------------------------------------------------\n",
              "COMPARISON REPORT\n",
              "--------------------------------------------------------------------------------\n",
              "\n",
              "SCHEMA DIFFERENCES:\n",
              "shape: (6, 2)\n",
              "┌─────────────────────────────────┬───────┐\n",
              "│ Statistic                       ┆ Count │\n",
              "│ ---                             ┆ ---   │\n",
              "│ str                             ┆ i64   │\n",
              "╞═════════════════════════════════╪═══════╡\n",
              "│ Columns in base                 ┆ 12    │\n",
              "│ Columns in compare              ┆ 10    │\n",
              "│ Columns in base and compare     ┆ 10    │\n",
              "│ Columns only in base            ┆ 2     │\n",
              "│ Columns only in compare         ┆ 0     │\n",
              "│ Columns with schema differences ┆ 1     │\n",
              "└─────────────────────────────────┴───────┘\n",
              "shape: (3, 3)\n",
              "┌───────────┬─────────────┬────────────────┐\n",
              "│ column    ┆ base_format ┆ compare_format │\n",
              "│ ---       ┆ ---         ┆ ---            │\n",
              "│ str       ┆ str         ┆ str            │\n",
              "╞═══════════╪═════════════╪════════════════╡\n",
              "│ rating    ┆ Utf8        ┆ null           │\n",
              "│ duration  ┆ Utf8        ┆ Int64          │\n",
              "│ listed_in ┆ Utf8        ┆ null           │\n",
              "└───────────┴─────────────┴────────────────┘\n",
              "--------------------------------------------------------------------------------\n",
              "\n",
              "ROW DIFFERENCES:\n",
              "shape: (5, 2)\n",
              "┌──────────────────────────┬───────┐\n",
              "│ Statistic                ┆ Count │\n",
              "│ ---                      ┆ ---   │\n",
              "│ str                      ┆ i64   │\n",
              "╞══════════════════════════╪═══════╡\n",
              "│ Rows in base             ┆ 8807  │\n",
              "│ Rows in compare          ┆ 6173  │\n",
              "│ Rows only in base        ┆ 2634  │\n",
              "│ Rows only in compare     ┆ 0     │\n",
              "│ Rows in base and compare ┆ 6173  │\n",
              "└──────────────────────────┴───────┘\n",
              "shape: (5, 3)\n",
              "┌─────────┬──────────┬──────────────┐\n",
              "│ show_id ┆ variable ┆ value        │\n",
              "│ ---     ┆ ---      ┆ ---          │\n",
              "│ str     ┆ str      ┆ str          │\n",
              "╞═════════╪══════════╪══════════════╡\n",
              "│ s2      ┆ status   ┆ in base only │\n",
              "│ s4      ┆ status   ┆ in base only │\n",
              "│ s5      ┆ status   ┆ in base only │\n",
              "│ s11     ┆ status   ┆ in base only │\n",
              "│ s15     ┆ status   ┆ in base only │\n",
              "└─────────┴──────────┴──────────────┘\n",
              "--------------------------------------------------------------------------------\n",
              "\n",
              "VALUE DIFFERENCES:\n",
              "shape: (9, 3)\n",
              "┌─────────────────────────┬───────┬────────────┐\n",
              "│ Value Differences       ┆ Count ┆ Percentage │\n",
              "│ ---                     ┆ ---   ┆ ---        │\n",
              "│ str                     ┆ i64   ┆ f64        │\n",
              "╞═════════════════════════╪═══════╪════════════╡\n",
              "│ Total Value Differences ┆ 6173  ┆ 12.5       │\n",
              "│ cast                    ┆ 0     ┆ 0.0        │\n",
              "│ country                 ┆ 0     ┆ 0.0        │\n",
              "│ date_added              ┆ 0     ┆ 0.0        │\n",
              "│ description             ┆ 0     ┆ 0.0        │\n",
              "│ director                ┆ 6173  ┆ 100.0      │\n",
              "│ release_year            ┆ 0     ┆ 0.0        │\n",
              "│ title                   ┆ 0     ┆ 0.0        │\n",
              "│ type                    ┆ 0     ┆ 0.0        │\n",
              "└─────────────────────────┴───────┴────────────┘\n",
              "shape: (5, 4)\n",
              "┌─────────┬──────────┬───────────────────────────────┬─────────┐\n",
              "│ show_id ┆ variable ┆ base                          ┆ compare │\n",
              "│ ---     ┆ ---      ┆ ---                           ┆ ---     │\n",
              "│ str     ┆ str      ┆ str                           ┆ str     │\n",
              "╞═════════╪══════════╪═══════════════════════════════╪═════════╡\n",
              "│ s1      ┆ director ┆ Kirsten Johnson               ┆ Me      │\n",
              "│ s3      ┆ director ┆ Julien Leclercq               ┆ Me      │\n",
              "│ s6      ┆ director ┆ Mike Flanagan                 ┆ Me      │\n",
              "│ s7      ┆ director ┆ Robert Cullen, José Luis Ucha ┆ Me      │\n",
              "│ s8      ┆ director ┆ Haile Gerima                  ┆ Me      │\n",
              "└─────────┴──────────┴───────────────────────────────┴─────────┘\n",
              "--------------------------------------------------------------------------------\n",
              "End of Report\n",
              "--------------------------------------------------------------------------------"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Booleans to indicate if there are schema, value and row differences\n",
        "\n",
        "print(\n",
        "    diff.is_schemas_equal(),\n",
        "    diff.is_values_equal(),\n",
        "    diff.is_rows_equal()\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fc4nmVvYCcgf",
        "outputId": "b08d3961-b19f-430a-a36c-ea094bbc7bd5"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "False False False\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title A summary of the schema differences\n",
        "\n",
        "print(diff.schemas_summary())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dbZ-wecHDF2L",
        "outputId": "114bf0bf-51db-49ad-d1e9-37ec05493709"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (6, 2)\n",
            "┌─────────────────────────────────┬───────┐\n",
            "│ Statistic                       ┆ Count │\n",
            "│ ---                             ┆ ---   │\n",
            "│ str                             ┆ i64   │\n",
            "╞═════════════════════════════════╪═══════╡\n",
            "│ Columns in base                 ┆ 12    │\n",
            "│ Columns in compare              ┆ 10    │\n",
            "│ Columns in base and compare     ┆ 10    │\n",
            "│ Columns only in base            ┆ 2     │\n",
            "│ Columns only in compare         ┆ 0     │\n",
            "│ Columns with schema differences ┆ 1     │\n",
            "└─────────────────────────────────┴───────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        " # @title Get a table with the schema differences\n",
        "\n",
        "print(diff.schemas_sample())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dgvR7V7OEKYA",
        "outputId": "3ad4b938-8e76-4795-9915-58a1a143a127"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (3, 3)\n",
            "┌───────────┬─────────────┬────────────────┐\n",
            "│ column    ┆ base_format ┆ compare_format │\n",
            "│ ---       ┆ ---         ┆ ---            │\n",
            "│ str       ┆ str         ┆ str            │\n",
            "╞═══════════╪═════════════╪════════════════╡\n",
            "│ rating    ┆ Utf8        ┆ null           │\n",
            "│ duration  ┆ Utf8        ┆ Int64          │\n",
            "│ listed_in ┆ Utf8        ┆ null           │\n",
            "└───────────┴─────────────┴────────────────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title A summary of the row differences\n",
        "\n",
        "print(diff.rows_summary())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zp8jCYZ-D5Jm",
        "outputId": "0da4d70e-263d-4ca6-ea12-a8b7480928fa"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (5, 2)\n",
            "┌──────────────────────────┬───────┐\n",
            "│ Statistic                ┆ Count │\n",
            "│ ---                      ┆ ---   │\n",
            "│ str                      ┆ i64   │\n",
            "╞══════════════════════════╪═══════╡\n",
            "│ Rows in base             ┆ 8807  │\n",
            "│ Rows in compare          ┆ 6173  │\n",
            "│ Rows only in base        ┆ 2634  │\n",
            "│ Rows only in compare     ┆ 0     │\n",
            "│ Rows in base and compare ┆ 6173  │\n",
            "└──────────────────────────┴───────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title A table contaning a sample of the row differences\n",
        "\n",
        "print(diff.rows_sample())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r_FKn_KIEBxD",
        "outputId": "1c0f4132-a59c-4992-81df-4ada46f9d298"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (5, 3)\n",
            "┌─────────┬──────────┬──────────────┐\n",
            "│ show_id ┆ variable ┆ value        │\n",
            "│ ---     ┆ ---      ┆ ---          │\n",
            "│ str     ┆ str      ┆ str          │\n",
            "╞═════════╪══════════╪══════════════╡\n",
            "│ s2      ┆ status   ┆ in base only │\n",
            "│ s4      ┆ status   ┆ in base only │\n",
            "│ s5      ┆ status   ┆ in base only │\n",
            "│ s11     ┆ status   ┆ in base only │\n",
            "│ s15     ┆ status   ┆ in base only │\n",
            "└─────────┴──────────┴──────────────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title A summary of the value differences\n",
        "\n",
        "\n",
        "print(diff.values_summary() )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_2M1dEbkD79T",
        "outputId": "edf2abd9-ed3b-4f8e-8576-3233a131c784"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (9, 3)\n",
            "┌─────────────────────────┬───────┬────────────┐\n",
            "│ Value Differences       ┆ Count ┆ Percentage │\n",
            "│ ---                     ┆ ---   ┆ ---        │\n",
            "│ str                     ┆ i64   ┆ f64        │\n",
            "╞═════════════════════════╪═══════╪════════════╡\n",
            "│ Total Value Differences ┆ 6173  ┆ 12.5       │\n",
            "│ cast                    ┆ 0     ┆ 0.0        │\n",
            "│ country                 ┆ 0     ┆ 0.0        │\n",
            "│ date_added              ┆ 0     ┆ 0.0        │\n",
            "│ description             ┆ 0     ┆ 0.0        │\n",
            "│ director                ┆ 6173  ┆ 100.0      │\n",
            "│ release_year            ┆ 0     ┆ 0.0        │\n",
            "│ title                   ┆ 0     ┆ 0.0        │\n",
            "│ type                    ┆ 0     ┆ 0.0        │\n",
            "└─────────────────────────┴───────┴────────────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title A table containt a sample of the value differences\n",
        "\n",
        "print(diff.values_sample())\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gTla7JpaEGeY",
        "outputId": "2715902d-9e5f-43aa-e433-ea8c36c3d4db"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (5, 4)\n",
            "┌─────────┬──────────┬───────────────────────────────┬─────────┐\n",
            "│ show_id ┆ variable ┆ base                          ┆ compare │\n",
            "│ ---     ┆ ---      ┆ ---                           ┆ ---     │\n",
            "│ str     ┆ str      ┆ str                           ┆ str     │\n",
            "╞═════════╪══════════╪═══════════════════════════════╪═════════╡\n",
            "│ s1      ┆ director ┆ Kirsten Johnson               ┆ Me      │\n",
            "│ s3      ┆ director ┆ Julien Leclercq               ┆ Me      │\n",
            "│ s6      ┆ director ┆ Mike Flanagan                 ┆ Me      │\n",
            "│ s7      ┆ director ┆ Robert Cullen, José Luis Ucha ┆ Me      │\n",
            "│ s8      ┆ director ┆ Haile Gerima                  ┆ Me      │\n",
            "└─────────┴──────────┴───────────────────────────────┴─────────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "vMT_rz85EHd5"
      },
      "execution_count": 41,
      "outputs": []
    }
  ]
}