tutorials/How_to_track_regression_testing_results.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "scJ3j0o6jeCV"
      },
      "outputs": [],
      "source": [
        "!pip install evidently[llm]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3apG-ua2lfsJ"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "from evidently.test_suite import TestSuite\n",
        "from evidently.descriptors import *\n",
        "from evidently.tests import *\n",
        "\n",
        "from evidently.ui.workspace.cloud import CloudWorkspace"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sfhqrIwcrWeK"
      },
      "source": [
        "Optional imports to create monitoring panels:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mqE66DeKlxC4"
      },
      "outputs": [],
      "source": [
        "from evidently.ui.dashboards import DashboardPanelPlot\n",
        "from evidently.ui.dashboards import DashboardPanelTestSuite\n",
        "from evidently.ui.dashboards import DashboardPanelTestSuiteCounter\n",
        "from evidently.ui.dashboards import TestSuitePanelType\n",
        "from evidently.ui.dashboards import ReportFilter\n",
        "from evidently.ui.dashboards import PanelValue\n",
        "from evidently.ui.dashboards import PlotType\n",
        "from evidently.ui.dashboards import CounterAgg\n",
        "from evidently.tests.base_test import TestStatus\n",
        "from evidently.renderers.html_widgets import WidgetSize"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F8NvKP6TnKlo"
      },
      "source": [
        "# Connect to Evidently Cloud"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sHuPy43ZnMGh"
      },
      "outputs": [],
      "source": [
        "ws = CloudWorkspace(token=\"YOUR_API_TOKEN\", url=\"https://app.evidently.cloud\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lSIxEjp7nNPW"
      },
      "source": [
        "Create a Project"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KUvOFoGSnM49"
      },
      "outputs": [],
      "source": [
        "project = ws.create_project(\"Regression testing example\", team_id=\"YOUR_TEAM_ID\")\n",
        "project.description = \"My project description\"\n",
        "project.save()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NKNreL_ZT_b9"
      },
      "source": [
        "# Golden dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6lyOYJzYFW3Z"
      },
      "source": [
        "Prepare a set of approved answers to compare against. Add a placeholder for the new response column (initially contains the duplicates of golden answers). This will be the **reference data**."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B3NILRSVjkvP"
      },
      "outputs": [],
      "source": [
        "data = [\n",
        "    [\"Hi there, how do I reset my password?\", \"To reset your password, click on 'Forgot Password' on the login page and follow the instructions sent to your registered email.\", \"To reset your password, click on 'Forgot Password' on the login page and follow the instructions sent to your registered email.\"],\n",
        "    [\"Can I change my email address on file?\", \"Yes, you can change your email address in the account settings under 'Profile Information'.\", \"Yes, you can change your email address in the account settings under 'Profile Information'.\"],\n",
        "    [\"Where can I view my purchase history?\", \"You can view your purchase history by going to 'Order History' in your account settings.\", \"You can view your purchase history by going to 'Order History' in your account settings.\"],\n",
        "    [\"What payment methods are accepted?\", \"We accept credit cards, debit cards, and PayPal.\", \"We accept credit cards, debit cards, and PayPal.\"],\n",
        "    [\"How do I contact customer support?\", \"You can contact customer support through the 'Help' section on our website or by emailing support@example.com email.\", \"You can contact customer support through the 'Help' section on our website or by emailing support@example.com email.\"],\n",
        "    [\"How do I update my billing information?\", \"You can update your billing information in the 'Billing' section of your account settings.\", \"You can update your billing information in the 'Billing' section of your account settings.\"],\n",
        "    [\"What is the return policy?\", \"Our return policy allows returns within 30 days of purchase with a receipt.\", \"Our return policy allows returns within 30 days of purchase with a receipt.\"],\n",
        "    [\"Can I track my order?\", \"Yes, you can track your order by logging into your account and going to 'Track Order'.\", \"Yes, you can track your order by logging into your account and going to 'Track Order'.\"],\n",
        "    [\"How do I apply a discount code?\", \"To apply a discount code, enter it at checkout in the 'Discount Code' field.\", \"To apply a discount code, enter it at checkout in the 'Discount Code' field.\"],\n",
        "    [\"Is there a mobile app available?\", \"Yes, our mobile app is available on both iOS and Android platforms.\", \"Yes, our mobile app is available on both iOS and Android platforms.\"]\n",
        "]\n",
        "\n",
        "columns = [\"question\", \"reference_response\", \"response\"]\n",
        "\n",
        "ref_data = pd.DataFrame(data, columns=columns)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PnyxddIuezCu"
      },
      "outputs": [],
      "source": [
        "pd.set_option('display.max_colwidth', None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yj2U_sZQjnZj"
      },
      "outputs": [],
      "source": [
        "ref_data.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gp1WgOyxGN-D"
      },
      "source": [
        "# Define the Test Suite"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FN0BqnilKqqz"
      },
      "outputs": [],
      "source": [
        "#test_suite = TestSuite(tests=[...])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9J9p9t8MGQfi"
      },
      "source": [
        "Define the test suite conditions you will be testing against:\n",
        "- If the **semantic similarity** between new and old responses is < 0.9, the test returns a warning.\n",
        "- If the **semantic similarity** between new and old responses is < 0.8, the test fails.\n",
        "- If the minimal **sentiment** is outside +/- 10% (default) from the reference range, the test fails.\n",
        "- If the mean **text length** is outside +/- 10% (default) from the reference range, the test returns a warning.\n",
        "- If the **text length** is outside the 0 to 250 symbols range, the test fails.\n",
        "- If responses **mention competitors**, the test fails."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2LpeQ62KGPTP"
      },
      "outputs": [],
      "source": [
        "test_suite = TestSuite(tests=[\n",
        "    TestColumnValueMin(\n",
        "        column_name=SemanticSimilarity(\n",
        "        display_name=\"Response Similarity\").\n",
        "        on([\"response\", \"reference_response\"]),\n",
        "        gte=0.9,\n",
        "        is_critical=False),\n",
        "    TestColumnValueMin(\n",
        "        column_name=SemanticSimilarity(\n",
        "        display_name=\"Response Similarity\").\n",
        "        on([\"response\", \"reference_response\"]),\n",
        "        gte=0.8),\n",
        "    TestColumnValueMin(\n",
        "        column_name=Sentiment(\n",
        "        display_name=\"Response Sentiment\").\n",
        "        on(\"response\")),\n",
        "    TestColumnValueMean(\n",
        "        column_name=TextLength(\n",
        "        display_name=\"Text Length\").\n",
        "        on(\"response\"),\n",
        "        is_critical=False),\n",
        "    TestShareOfOutRangeValues(\n",
        "        column_name=TextLength(\n",
        "        display_name=\"Text Length\").\n",
        "        on(\"response\"),\n",
        "        left=0, right=250,\n",
        "        eq=0),\n",
        "    TestCategoryCount(\n",
        "        column_name=Contains(\n",
        "            items=[\"AnotherFinCo\", \"YetAnotherFinCo\"],\n",
        "            display_name=\"Competitor Mentions\").\n",
        "        on(\"response\"),\n",
        "        category=True,\n",
        "        eq=0),\n",
        "])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q3Bw48oQUFRU"
      },
      "source": [
        "# Change prompt - 1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ToArMI1qjjuw"
      },
      "source": [
        "Generate a new set of answers. (Modify the prompt to be more conversational in style.)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FFpoK3jzT9Da"
      },
      "outputs": [],
      "source": [
        "#In real use case, call your LLM product API to get a new set of questions and add them to a \"response\" column.\n",
        "\n",
        "data = [\n",
        "    [\"Hi there, how do I reset my password?\", \"To reset your password, click on 'Forgot Password' on the login page and follow the instructions sent to your registered email.\", \"Hey! Just click on 'Forgot Password' on the login page and follow the steps in the email we send you.\"],\n",
        "    [\"Can I change my email address on file?\", \"Yes, you can change your email address in the account settings under 'Profile Information'.\", \"Of course! Go to 'Profile Information' in your account settings to update your email address.\"],\n",
        "    [\"Where can I view my purchase history?\", \"You can view your purchase history by going to 'Order History' in your account settings.\", \"Absolutely! Just go to 'Order History' in your account settings to see all your purchases.\"],\n",
        "    [\"What payment methods are accepted?\", \"We accept credit cards, debit cards, and PayPal.\", \"We take credit cards, debit cards, and PayPal.\"],\n",
        "    [\"How do I contact customer support?\", \"You can contact customer support through the 'Help' section on our website or by emailing support@example.com email.\", \"You can reach out to our support team via the 'Help' section on our website or by emailing support@example.com.\"],\n",
        "    [\"How do I update my billing information?\", \"You can update your billing information in the 'Billing' section of your account settings.\", \"You can easily update your billing info in the 'Billing' section of your account settings.\"],\n",
        "    [\"What is the return policy?\", \"Our return policy allows returns within 30 days of purchase with a receipt.\", \"You can return items within 30 days of purchase as long as you have the receipt.\"],\n",
        "    [\"Can I track my order?\", \"Yes, you can track your order by logging into your account and going to 'Track Order'.\", \"Of course! To track the order, log into your account and go to 'Track Order' to see where your package is.\"],\n",
        "    [\"How do I apply a discount code?\", \"To apply a discount code, enter it at checkout in the 'Discount Code' field.\", \"Simply enter your discount code at checkout in the 'Discount Code' box.\"],\n",
        "    [\"Is there a mobile app available?\", \"Yes, our mobile app is available on both iOS and Android platforms.\", \"Yes, we have a mobile app available on both iOS and Android.\"]\n",
        "]\n",
        "\n",
        "columns = [\"question\", \"reference_response\", \"response\"]\n",
        "\n",
        "cur_data_1 = pd.DataFrame(data, columns=columns)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QgsLYfZ8e2jv"
      },
      "outputs": [],
      "source": [
        "#To preview the dataset\n",
        "\n",
        "#cur_data_1.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GMp_TVI6nl0a"
      },
      "outputs": [],
      "source": [
        "test_suite.run(reference_data=ref_data, current_data=cur_data_1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZenoaPU5pO4j"
      },
      "outputs": [],
      "source": [
        "ws.add_test_suite(project.id, test_suite)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FsoAN18uqIT-"
      },
      "outputs": [],
      "source": [
        "#To view tests in Python\n",
        "\n",
        "#test_suite"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kqp0MKGvyxM-"
      },
      "source": [
        "# Create monitoring panels"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YuTR38C5SzZE"
      },
      "source": [
        "Add a counter panel to show the SUCCESS rate of the latest test run:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RD0IAyXzSyzT"
      },
      "outputs": [],
      "source": [
        "project.dashboard.add_panel(\n",
        "     DashboardPanelTestSuiteCounter(\n",
        "        title=\"Latest Test run\",\n",
        "        filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),\n",
        "        size=WidgetSize.FULL,\n",
        "        statuses=[TestStatus.SUCCESS],\n",
        "        agg=CounterAgg.LAST,\n",
        "    ),\n",
        "    tab=\"Tests\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P7u939wcymGL"
      },
      "source": [
        "Add a test monitoring panel to show all test results over time."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2yBc9eGMyllJ"
      },
      "outputs": [],
      "source": [
        "project.dashboard.add_panel(\n",
        "    DashboardPanelTestSuite(\n",
        "        title=\"Test results\",\n",
        "        filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),\n",
        "        size=WidgetSize.FULL,\n",
        "        panel_type=TestSuitePanelType.DETAILED,\n",
        "        time_agg=\"1min\",\n",
        "    ),\n",
        "    tab=\"Tests\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9sl5FcT6gvME"
      },
      "source": [
        "Add a panel with a bar chart to plot min, max, and mean semantic similarity."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2gU7fWj40_Ln"
      },
      "outputs": [],
      "source": [
        "project.dashboard.add_panel(\n",
        "    DashboardPanelPlot(\n",
        "        title=\"Semantic similarity\",\n",
        "        filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),\n",
        "        size=WidgetSize.HALF,\n",
        "        values=[\n",
        "            PanelValue(\n",
        "                metric_id=\"ColumnSummaryMetric\",\n",
        "                field_path='current_characteristics.min',\n",
        "                metric_args={\"column_name\" : SemanticSimilarity(display_name=\"Response Similarity\").on([\"response\", \"reference_response\"])},\n",
        "                legend='min',\n",
        "            ),\n",
        "            PanelValue(\n",
        "                metric_id=\"ColumnSummaryMetric\",\n",
        "                field_path='current_characteristics.mean',\n",
        "                metric_args={\"column_name\" : SemanticSimilarity(display_name=\"Response Similarity\").on([\"response\", \"reference_response\"])},\n",
        "                legend='mean',\n",
        "            ),\n",
        "            PanelValue(\n",
        "                metric_id=\"ColumnSummaryMetric\",\n",
        "                field_path='current_characteristics.max',\n",
        "                metric_args={\"column_name\" : SemanticSimilarity(display_name=\"Response Similarity\").on([\"response\", \"reference_response\"])},\n",
        "                legend='max',\n",
        "            ),\n",
        "        ],\n",
        "        plot_type=PlotType.BAR,\n",
        "    ),\n",
        "    tab=\"Tests\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PjveWhNog1GQ"
      },
      "source": [
        "Add a panel with a bar chart to plot min, max, and mean sentiment."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "edFuIywN2vsV"
      },
      "outputs": [],
      "source": [
        "project.dashboard.add_panel(\n",
        "    DashboardPanelPlot(\n",
        "        title=\"Sentiment\",\n",
        "        filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),\n",
        "        size=WidgetSize.HALF,\n",
        "        values=[\n",
        "            PanelValue(\n",
        "                metric_id=\"ColumnSummaryMetric\",\n",
        "                field_path='current_characteristics.min',\n",
        "                metric_args={\"column_name\" : Sentiment(display_name=\"Response Sentiment\").on(\"response\")},\n",
        "                legend='min',\n",
        "            ),\n",
        "            PanelValue(\n",
        "                metric_id=\"ColumnSummaryMetric\",\n",
        "                field_path='current_characteristics.mean',\n",
        "                metric_args={\"column_name\" : Sentiment(display_name=\"Response Sentiment\").on(\"response\")},\n",
        "                legend='mean',\n",
        "            ),\n",
        "            PanelValue(\n",
        "                metric_id=\"ColumnSummaryMetric\",\n",
        "                field_path='current_characteristics.max',\n",
        "                metric_args={\"column_name\" : Sentiment(display_name=\"Response Sentiment\").on(\"response\")},\n",
        "                legend='max',\n",
        "            ),\n",
        "        ],\n",
        "        plot_type=PlotType.BAR,\n",
        "    ),\n",
        "    tab=\"Tests\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BiF0cMj0g33G"
      },
      "source": [
        "Save results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RMplThLSg4lm"
      },
      "outputs": [],
      "source": [
        "project.save()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7qRLYG8qUSRh"
      },
      "source": [
        "# Change prompt - 2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SAD-nh5ZUWIa"
      },
      "source": [
        "Generate a new set of answers. (Modify the prompt to be more formal)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MB8tJhWNVIzt"
      },
      "outputs": [],
      "source": [
        "data = [\n",
        "    [\"Hi there, how do I reset my password?\", \"To reset your password, click on 'Forgot Password' on the login page and follow the instructions sent to your registered email.\", \"To reset your password, kindly click on the 'Forgot Password' link located on the login page and then follow the detailed instructions that will be sent to your registered email address. This should guide you through the process smoothly.\"],\n",
        "    [\"Can I change my email address on file?\", \"Yes, you can change your email address in the account settings under 'Profile Information'.\", \"Yes, you may update your email address by navigating to the account settings section and selecting 'Profile Information'. From there, you can easily make the necessary changes to your email address.\"],\n",
        "    [\"Where can I view my purchase history?\", \"You can view your purchase history by going to 'Order History' in your account settings.\", \"You can access your complete purchase history by logging into your account and visiting the 'Order History' section. This area contains detailed records of all your past transactions.\"],\n",
        "    [\"What payment methods are accepted?\", \"We accept credit cards, debit cards, and PayPal.\", \"We are pleased to accept a variety of payment methods, including credit cards, debit cards, and PayPal. These options provide flexibility and convenience for our valued customers.\"],\n",
        "    [\"How do I contact customer support?\", \"You can contact customer support through the 'Help' section on our website or by emailing support@example.com email.\", \"To get in touch with our customer support team, please visit the 'Help' section on our website. Alternatively, you can send an email to support@example.com, and we will be happy to assist you.\"],\n",
        "    [\"How do I update my billing information?\", \"You can update your billing information in the 'Billing' section of your account settings.\", \"Updating your billing information is straightforward. Simply log into your account, navigate to the 'Billing' section within your account settings, and make the necessary updates to your billing details.\"],\n",
        "    [\"What is the return policy?\", \"Our return policy allows returns within 30 days of purchase with a receipt.\", \"Our return policy permits returns within a 30-day period from the date of purchase, provided you have the receipt. This policy is designed to ensure your satisfaction with our products.\"],\n",
        "    [\"Can I track my order?\", \"Yes, you can track your order by logging into your account and going to 'Track Order'.\", \"Certainly! You can monitor the status of your order by logging into your account and selecting the 'Track Order' option. This will provide you with real-time updates on your shipment.\"],\n",
        "    [\"How do I apply a discount code?\", \"To apply a discount code, enter it at checkout in the 'Discount Code' field.\", \"To use a discount code, please enter it in the 'Discount Code' field during the checkout process. This will automatically apply the discount to your purchase.\"],\n",
        "    [\"Is there a mobile app available?\", \"Yes, our mobile app is available on both iOS and Android platforms.\", \"Yes, we offer a mobile application that is available for both iOS and Android platforms. You can download it from the respective app stores for a more convenient shopping experience.\"]\n",
        "]\n",
        "\n",
        "columns = [\"question\", \"reference_response\", \"response\"]\n",
        "\n",
        "cur_data_2 = pd.DataFrame(data, columns=columns)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "49fpY-zle50Q"
      },
      "outputs": [],
      "source": [
        "#To preview the dataset\n",
        "\n",
        "#cur_data_2.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xYBugmjgpY6K"
      },
      "outputs": [],
      "source": [
        "test_suite.run(reference_data=ref_data, current_data=cur_data_2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J7mS-hCOxQVv"
      },
      "outputs": [],
      "source": [
        "ws.add_test_suite(project.id, test_suite)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "drCuNLherTIx"
      },
      "outputs": [],
      "source": [
        "#To view tests in Python\n",
        "\n",
        "#test_suite"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Cx7-Z_-TwaCx"
      },
      "outputs": [],
      "source": [
        "test_suite.datasets()[1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cRAmUOetrp7G"
      },
      "outputs": [],
      "source": [
        "df = pd.DataFrame(test_suite.datasets()[1])\n",
        "\n",
        "filtered_df = df[df['Response Similarity'] < 0.9]\n",
        "\n",
        "filtered_df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P579GPfhVOAh"
      },
      "source": [
        "# Change prompt - 3"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZXqTlGvwVu7X"
      },
      "source": [
        "Generate a new set of answers. (Modify the prompt to add additional context when appropriate)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pYwpmGMoV1qS"
      },
      "outputs": [],
      "source": [
        "data = [\n",
        "    [\"Hi there, how do I reset my password?\", \"To reset your password, click on 'Forgot Password' on the login page and follow the instructions sent to your registered email.\", \"To reset your password, kindly click on the 'Forgot Password' link located on the login page. You will receive an email with detailed instructions. Please follow those steps carefully to successfully reset your password.\"],\n",
        "    [\"Can I change my email address on file?\", \"Yes, you can change your email address in the account settings under 'Profile Information'.\", \"Yes, you may update your email address by navigating to the account settings section and selecting 'Profile Information'. After updating, please ensure to save the changes to keep your account information current.\"],\n",
        "    [\"Where can I view my purchase history?\", \"You can view your purchase history by going to 'Order History' in your account settings.\", \"You can access your complete purchase history by logging into your account and visiting the 'Order History' section. This section contains detailed records of all your past transactions, including dates and amounts.\"],\n",
        "    [\"What payment methods are accepted?\", \"We accept credit cards, debit cards, and PayPal.\", \"We are pleased to accept a variety of payment methods, including credit cards, debit cards, and PayPal. These options are available to provide you with the utmost flexibility and convenience during the checkout process.\"],\n",
        "    [\"How do I contact customer support?\", \"You can contact customer support through the 'Help' section on our website or by emailing support@example.com email.\", \"To get in touch with our customer support team, please visit the 'Help' section on our website. Additionally, you can send an email to support@example.com, and we will respond to your query as soon as possible.\"],\n",
        "    [\"How do I update my billing information?\", \"You can update your billing information in the 'Billing' section of your account settings.\", \"Updating your billing information is straightforward. Simply log into your account, navigate to the 'Billing' section within your account settings, and make the necessary updates to your billing details. Please ensure to save the changes.\"],\n",
        "    [\"What is the return policy?\", \"Our return policy allows returns within 30 days of purchase with a receipt.\", \"Our return policy permits returns within a 30-day period from the date of purchase, provided you have the receipt. This policy is designed to ensure your satisfaction with our products, allowing you to shop with confidence.\"],\n",
        "    [\"Can I track my order?\", \"Yes, you can track your order by logging into your account and going to 'Track Order'.\", \"Certainly! You can monitor the status of your order by logging into your account and selecting the 'Track Order' option. This will provide you with real-time updates on your shipment, including the estimated delivery date.\"],\n",
        "    [\"How do I apply a discount code?\", \"To apply a discount code, enter it at checkout in the 'Discount Code' field.\", \"To use a discount code, please enter it in the 'Discount Code' field during the checkout process. This will automatically apply the discount to your purchase, providing you with the reduced price.\"],\n",
        "    [\"Is there a mobile app available?\", \"Yes, our mobile app is available on both iOS and Android platforms.\", \"Yes, we offer a mobile application that is available for both iOS and Android platforms. You can download it from the respective app stores for a more convenient shopping experience.\"]\n",
        "]\n",
        "\n",
        "columns = [\"question\", \"reference_response\", \"response\"]\n",
        "\n",
        "cur_data_3 = pd.DataFrame(data, columns=columns)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-Q2VJtJQe7UN"
      },
      "outputs": [],
      "source": [
        "#To preview the dataset\n",
        "\n",
        "#cur_data_3.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Pk1PKETXr4rw"
      },
      "outputs": [],
      "source": [
        "test_suite.run(reference_data=ref_data, current_data=cur_data_3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dTn5XqHkxOGN"
      },
      "outputs": [],
      "source": [
        "ws.add_test_suite(project.id, test_suite)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Pq31E0QYr7t6"
      },
      "outputs": [],
      "source": [
        "#To view tests in Python\n",
        "\n",
        "#test_suite"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MHVG3Y3lsjLB"
      },
      "outputs": [],
      "source": [
        "df = pd.DataFrame(test_suite.datasets()[1])\n",
        "\n",
        "filtered_df = df[df['Response Similarity'] < 0.9]\n",
        "\n",
        "filtered_df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EIiXzeAjWGLy"
      },
      "source": [
        "# Change prompt - 4"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZsS4P_nDwwgZ"
      },
      "source": [
        "Generate a new set of answers. (Modify the prompt to add examples).\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Cklpw-Z3WIxL"
      },
      "outputs": [],
      "source": [
        "data = [\n",
        "    [\"Hi there, how do I reset my password?\", \"To reset your password, click on 'Forgot Password' on the login page and follow the instructions sent to your registered email.\", \"To reset your password, please click on the 'Forgot Password' link on the login page. You will receive an email with specific instructions. For example, you might be asked to enter your registered email and create a new password.\"],\n",
        "    [\"Can I change my email address on file?\", \"Yes, you can change your email address in the account settings under 'Profile Information'.\", \"Yes, you can update your email address by going to the account settings and selecting 'Profile Information'. For instance, you could remove your old email and add a new one, ensuring to save the changes.\"],\n",
        "    [\"Where can I view my purchase history?\", \"You can view your purchase history by going to 'Order History' in your account settings.\", \"You can check your purchase history by logging into your account and navigating to the 'Order History' section. This section will display all your past orders, including the order dates and amounts paid.\"],\n",
        "    [\"What payment methods are accepted?\", \"We accept credit cards, debit cards, and PayPal.\", \"We accept various payment methods, such as credit cards, debit cards, and PayPal. This provides you with several options to choose from when making a purchase.\"],\n",
        "    [\"How do I contact customer support?\", \"You can contact customer support through the 'Help' section on our website or by emailing support@example.com email.\", \"To reach our customer support team, you can use the 'Help' section on our website. Additionally, you may email us at support@example.com, and we will address your concerns promptly.\"],\n",
        "    [\"How do I update my billing information?\", \"You can update your billing information in the 'Billing' section of your account settings.\", \"To update your billing information, log into your account, go to the 'Billing' section under account settings, and make the necessary updates. For example, you can change your credit card details or address.\"],\n",
        "    [\"What is the return policy?\", \"Our return policy allows returns within 30 days of purchase with a receipt.\", \"Our return policy allows you to return items within 30 days of purchase if you have the receipt. This ensures you can return products that do not meet your expectations or requirements.\"],\n",
        "    [\"Can I track my order?\", \"Yes, you can track your order by logging into your account and going to 'Track Order'.\", \"Yes, you can track your order by logging into your account and selecting the 'Track Order' option. This feature will provide you with updates on your order's status and estimated delivery date.\"],\n",
        "    [\"How do I apply a discount code?\", \"To apply a discount code, enter it at checkout in the 'Discount Code' field.\", \"To apply a discount code, enter it in the 'Discount Code' field during checkout. This will ensure the discount is applied to your total purchase amount, giving you the reduced price.\"],\n",
        "    [\"Is there a mobile app available?\", \"Yes, our mobile app is available on both iOS and Android platforms.\", \"Yes, our mobile app is available on both iOS and Android platforms. You can download it from the respective app stores for a more seamless and convenient shopping experience.\"]\n",
        "]\n",
        "\n",
        "columns = [\"question\", \"reference_response\", \"response\"]\n",
        "\n",
        "cur_data_4 = pd.DataFrame(data, columns=columns)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uuGti2mywoHa"
      },
      "outputs": [],
      "source": [
        "#To preview the dataset\n",
        "\n",
        "#cur_data_4.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RNWl8qSrscgo"
      },
      "outputs": [],
      "source": [
        "test_suite.run(reference_data=ref_data, current_data=cur_data_4)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O6Ye8zZexWF3"
      },
      "outputs": [],
      "source": [
        "ws.add_test_suite(project.id, test_suite)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z_P-wsuvvkRt"
      },
      "outputs": [],
      "source": [
        "#To view tests in Python\n",
        "\n",
        "#test_suite"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9vd7oSqqesZ7"
      },
      "source": [
        "# Change prompt - 5"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vzEhcq5iwcLu"
      },
      "source": [
        "Generate a new set of answers. (Modify the prompt to give shorter responses)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x98ps6SXend9"
      },
      "outputs": [],
      "source": [
        "data = [\n",
        "    [\"Hi there, how do I reset my password?\", \"To reset your password, click on 'Forgot Password' on the login page and follow the instructions sent to your registered email.\", \"Click 'Forgot Password' on the login page and follow the email instructions.\"],\n",
        "    [\"Can I change my email address on file?\", \"Yes, you can change your email address in the account settings under 'Profile Information'.\", \"Go to 'Profile Information' in account settings to change your email.\"],\n",
        "    [\"Where can I view my purchase history?\", \"You can view your purchase history by going to 'Order History' in your account settings.\", \"Check 'Order History' in your account settings.\"],\n",
        "    [\"What payment methods are accepted?\", \"We accept credit cards, debit cards, and PayPal.\", \"We accept credit cards, debit cards, and PayPal.\"],\n",
        "    [\"How do I contact customer support?\", \"You can contact customer support through the 'Help' section on our website or by emailing support@example.com email.\", \"Visit 'Help' on our website or email support@example.com.\"],\n",
        "    [\"How do I update my billing information?\", \"You can update your billing information in the 'Billing' section of your account settings.\", \"Update billing info in the 'Billing' section of account settings.\"],\n",
        "    [\"What is the return policy?\", \"Our return policy allows returns within 30 days of purchase with a receipt.\", \"Return items within 30 days with a receipt.\"],\n",
        "    [\"Can I track my order?\", \"Yes, you can track your order by logging into your account and going to 'Track Order'.\", \"Track your order in the 'Track Order' section of your account.\"],\n",
        "    [\"How do I apply a discount code?\", \"To apply a discount code, enter it at checkout in the 'Discount Code' field.\", \"Enter your discount code at checkout.\"],\n",
        "    [\"Is there a mobile app available?\", \"Yes, our mobile app is available on both iOS and Android platforms.\", \"Our app is available on iOS and Android.\"]\n",
        "]\n",
        "\n",
        "columns = [\"question\", \"reference_response\", \"response\"]\n",
        "\n",
        "cur_data_5 = pd.DataFrame(data, columns=columns)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#To preview the dataset\n",
        "\n",
        "#cur_data_5.head(10)"
      ],
      "metadata": {
        "id": "W0LFY9rVqMnM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "reo-8QWFwPES"
      },
      "outputs": [],
      "source": [
        "test_suite.run(reference_data=ref_data, current_data=cur_data_5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Qqlf57eHxTwn"
      },
      "outputs": [],
      "source": [
        "ws.add_test_suite(project.id, test_suite)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YuKpLgzKwRZ2"
      },
      "outputs": [],
      "source": [
        "#To view tests in Python\n",
        "\n",
        "#test_suite"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NHtSFpaJwXDY"
      },
      "outputs": [],
      "source": [
        "df = pd.DataFrame(test_suite.datasets()[1])\n",
        "\n",
        "filtered_df = df[df['Response Similarity'] < 0.9]\n",
        "\n",
        "filtered_df.head()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}