BERT_NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT_NLP.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "AcG8ZO1iQqn5",
        "fZ3ut8XCatI8",
        "ONl2oh28HSIL",
        "j_fMj1SFzvWH",
        "At7st3UmL1v5"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AcG8ZO1iQqn5"
      },
      "source": [
        "##BERT Installations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c0BSQJNH-HF1",
        "outputId": "265879b6-5ab6-43e8-9b8b-012901a2be67",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#Install the server and client via pip. They can be installed separately or even on different machines\n",
        "!pip install bert-serving-client"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting bert-serving-client\n",
            "  Downloading https://files.pythonhosted.org/packages/1f/09/aae1405378a848b2e87769ad89a43d6d71978c4e15534ca48e82e723a72f/bert_serving_client-1.10.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: pyzmq>=17.1.0 in /usr/local/lib/python3.6/dist-packages (from bert-serving-client) (19.0.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from bert-serving-client) (1.18.5)\n",
            "Installing collected packages: bert-serving-client\n",
            "Successfully installed bert-serving-client-1.10.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pCgft3wa-SmF",
        "outputId": "6f066c15-4f98-4244-d6c5-8b9a8d80d95e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip install -U bert-serving-server[http]"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting bert-serving-server[http]\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b0/bd/cab677bbd0c5fb08b72e468371d2bca6ed9507785739b4656b0b5470d90b/bert_serving_server-1.10.0-py3-none-any.whl (61kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 5.6MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from bert-serving-server[http]) (1.18.5)\n",
            "Requirement already satisfied, skipping upgrade: pyzmq>=17.1.0 in /usr/local/lib/python3.6/dist-packages (from bert-serving-server[http]) (19.0.2)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from bert-serving-server[http]) (1.15.0)\n",
            "Collecting GPUtil>=1.3.0\n",
            "  Downloading https://files.pythonhosted.org/packages/ed/0e/5c61eedde9f6c87713e89d794f01e378cfd9565847d4576fa627d758c554/GPUtil-1.4.0.tar.gz\n",
            "Requirement already satisfied, skipping upgrade: termcolor>=1.1 in /usr/local/lib/python3.6/dist-packages (from bert-serving-server[http]) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: bert-serving-client; extra == \"http\" in /usr/local/lib/python3.6/dist-packages (from bert-serving-server[http]) (1.10.0)\n",
            "Collecting flask-cors; extra == \"http\"\n",
            "  Downloading https://files.pythonhosted.org/packages/69/7f/d0aeaaafb5c3c76c8d2141dbe2d4f6dca5d6c31872d4e5349768c1958abc/Flask_Cors-3.0.9-py2.py3-none-any.whl\n",
            "Collecting flask-compress; extra == \"http\"\n",
            "  Downloading https://files.pythonhosted.org/packages/eb/33/7bcfc1d240bf4cf701cc742716a3af95b9df5b26d605559ea029b6ffa04f/Flask-Compress-1.8.0.tar.gz\n",
            "Collecting flask-json; extra == \"http\"\n",
            "  Downloading https://files.pythonhosted.org/packages/6f/2d/4c21d98b11f3a206fabbdd965b53a2ca3ee9fab7646c93cf36c060e8f1a4/Flask_JSON-0.3.4-py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: flask; extra == \"http\" in /usr/local/lib/python3.6/dist-packages (from bert-serving-server[http]) (1.1.2)\n",
            "Collecting brotli\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b4/d3/7c98f05b7b9103e2f3a112ba42f269c798155b3e5404fb80bb8f823aaebe/Brotli-1.0.9-cp36-cp36m-manylinux1_x86_64.whl (357kB)\n",
            "\u001b[K     |████████████████████████████████| 358kB 9.5MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: click>=5.1 in /usr/local/lib/python3.6/dist-packages (from flask; extra == \"http\"->bert-serving-server[http]) (7.1.2)\n",
            "Requirement already satisfied, skipping upgrade: itsdangerous>=0.24 in /usr/local/lib/python3.6/dist-packages (from flask; extra == \"http\"->bert-serving-server[http]) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: Jinja2>=2.10.1 in /usr/local/lib/python3.6/dist-packages (from flask; extra == \"http\"->bert-serving-server[http]) (2.11.2)\n",
            "Requirement already satisfied, skipping upgrade: Werkzeug>=0.15 in /usr/local/lib/python3.6/dist-packages (from flask; extra == \"http\"->bert-serving-server[http]) (1.0.1)\n",
            "Requirement already satisfied, skipping upgrade: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.10.1->flask; extra == \"http\"->bert-serving-server[http]) (1.1.1)\n",
            "Building wheels for collected packages: GPUtil, flask-compress\n",
            "  Building wheel for GPUtil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for GPUtil: filename=GPUtil-1.4.0-cp36-none-any.whl size=7411 sha256=1e7aa2b27ff577cb1afa2a506af559f503bca5cab51cc4a6a11847ac3d1aead6\n",
            "  Stored in directory: /root/.cache/pip/wheels/3d/77/07/80562de4bb0786e5ea186911a2c831fdd0018bda69beab71fd\n",
            "  Building wheel for flask-compress (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for flask-compress: filename=Flask_Compress-1.8.0-cp36-none-any.whl size=7215 sha256=158114214afe4bf6bccd67fdf38522ccf27ee524dc50c7e987119c91f7f5086c\n",
            "  Stored in directory: /root/.cache/pip/wheels/f9/aa/e9/c4b763ea17eb0662124f67924cbfb9ee0efa83e28f6131baec\n",
            "Successfully built GPUtil flask-compress\n",
            "Installing collected packages: GPUtil, flask-cors, brotli, flask-compress, flask-json, bert-serving-server\n",
            "Successfully installed GPUtil-1.4.0 bert-serving-server-1.10.0 brotli-1.0.9 flask-compress-1.8.0 flask-cors-3.0.9 flask-json-0.3.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vfnsfHMT-Vxo",
        "outputId": "7538f653-9848-4667-9cbe-6e9a4bdaa0d6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#Download a model listed below, then uncompress the zip file into some folder, say /tmp/english_L-12_H-768_A-12/\n",
        "!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-11-16 02:44:17--  https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.195.128, 74.125.142.128, 74.125.20.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.195.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 407727028 (389M) [application/zip]\n",
            "Saving to: ‘uncased_L-12_H-768_A-12.zip’\n",
            "\n",
            "uncased_L-12_H-768_ 100%[===================>] 388.84M   161MB/s    in 2.4s    \n",
            "\n",
            "2020-11-16 02:44:19 (161 MB/s) - ‘uncased_L-12_H-768_A-12.zip’ saved [407727028/407727028]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T5bmHw1d-a5T",
        "outputId": "38cf99b3-25d3-4723-bac0-af1bff3e03ff",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!unzip uncased_L-12_H-768_A-12.zip"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  uncased_L-12_H-768_A-12.zip\n",
            "   creating: uncased_L-12_H-768_A-12/\n",
            "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.meta  \n",
            "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.data-00000-of-00001  \n",
            "  inflating: uncased_L-12_H-768_A-12/vocab.txt  \n",
            "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.index  \n",
            "  inflating: uncased_L-12_H-768_A-12/bert_config.json  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gzHVaghv-d1f",
        "outputId": "532688f6-d705-49c0-9678-50e012fcc10d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "%tensorflow_version 1.x"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OnPK-j6I-hPT"
      },
      "source": [
        "!nohup bert-serving-start -model_dir=./uncased_L-12_H-768_A-12 > out.file 2>&1 -max_seq_len=512 &"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "By0E9Wee-kCC"
      },
      "source": [
        "from bert_serving.client import BertClient\n",
        "bc = BertClient()"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w9ndjIKmGgZT"
      },
      "source": [
        "#Dataset1-Big(85282 rows)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fZ3ut8XCatI8"
      },
      "source": [
        "##Import Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b2ri6RQELPui",
        "outputId": "b6159505-5aad-414e-d8a9-71deca73c908",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        }
      },
      "source": [
        "import pandas as pd;\n",
        "\n",
        "#Loading dataset and rename columns\n",
        "\n",
        "df=pd.read_excel(\"healthtapQAs_v1.xlsx\");\n",
        "df.columns=[\"questions\",\"answers\"];\n",
        "\n",
        "df"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>answers</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>zirconium dental implants. how common is it us...</td>\n",
              "      <td>a majority of the dental implants placed are t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>zoloft/ sertaline cause heart racing can't sit...</td>\n",
              "      <td>check with your doc could be side effect of th...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>zoloft (sertraline) side effects? does it make...</td>\n",
              "      <td>it can be related to either weight gain or los...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>yellow skin around wound from cat bite. why?</td>\n",
              "      <td>this may be the secondary result of a resolvin...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>yeast infection and open sores major swelling ...</td>\n",
              "      <td>bathing in a bath with baking soda may help de...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85277</th>\n",
              "      <td>what is the best: a conventional angiography o...</td>\n",
              "      <td>for small vessels conventional angiography is ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85278</th>\n",
              "      <td>what is unique about taking medicine every 24 ...</td>\n",
              "      <td>try taking one half tab every day. but 10mg si...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85279</th>\n",
              "      <td>what treat should be of following sypmtoms pai...</td>\n",
              "      <td>your description of the pain sounds neuropathi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85280</th>\n",
              "      <td>can you explain incomplete right bundle branch...</td>\n",
              "      <td>the righty and left ventricles are induced to ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85281</th>\n",
              "      <td>can someone with a pacemaker receive radiation...</td>\n",
              "      <td>it is not safe to be in direct field of radiat...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>85282 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               questions                                            answers\n",
              "0      zirconium dental implants. how common is it us...  a majority of the dental implants placed are t...\n",
              "1      zoloft/ sertaline cause heart racing can't sit...  check with your doc could be side effect of th...\n",
              "2      zoloft (sertraline) side effects? does it make...  it can be related to either weight gain or los...\n",
              "3           yellow skin around wound from cat bite. why?  this may be the secondary result of a resolvin...\n",
              "4      yeast infection and open sores major swelling ...  bathing in a bath with baking soda may help de...\n",
              "...                                                  ...                                                ...\n",
              "85277  what is the best: a conventional angiography o...  for small vessels conventional angiography is ...\n",
              "85278  what is unique about taking medicine every 24 ...  try taking one half tab every day. but 10mg si...\n",
              "85279  what treat should be of following sypmtoms pai...  your description of the pain sounds neuropathi...\n",
              "85280  can you explain incomplete right bundle branch...  the righty and left ventricles are induced to ...\n",
              "85281  can someone with a pacemaker receive radiation...  it is not safe to be in direct field of radiat...\n",
              "\n",
              "[85282 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jx8GakBta1ba"
      },
      "source": [
        "##Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CKYjAjKIMNuD"
      },
      "source": [
        "import re\n",
        "import gensim \n",
        "from gensim.parsing.preprocessing import remove_stopwords\n",
        "\n",
        "#keeping only words in the sentence and removing punctuations.\n",
        "\n",
        "def clean_sentence(sentence, stopwords=False):\n",
        "    \n",
        "    sentence = sentence.lower().strip()\n",
        "    sentence = re.sub(r'[^a-z0-9\\s]', '', sentence)\n",
        "    \n",
        "    if stopwords:\n",
        "         sentence = remove_stopwords(sentence)\n",
        "    \n",
        "    \n",
        "    return sentence\n",
        "                    \n",
        "def get_cleaned_sentences(df,stopwords=False):    \n",
        "    sents=df[[\"questions\"]];\n",
        "    cleaned_sentences=[]\n",
        "\n",
        "    for index,row in df.iterrows():\n",
        "        #print(index,row)\n",
        "        cleaned=clean_sentence(row[\"questions\"],stopwords);\n",
        "        cleaned_sentences.append(cleaned);\n",
        "    return cleaned_sentences;\n",
        "\n",
        "cleaned_sentences=get_cleaned_sentences(df,stopwords=True)\n",
        "#print(cleaned_sentences);\n",
        "\n",
        "\n",
        "cleaned_sentences_with_stopwords=get_cleaned_sentences(df,stopwords=False)\n",
        "#print(cleaned_sentences_with_stopwords[0]);"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W60k9PEUckz_",
        "outputId": "e8d92046-fe26-48d6-af90-42176a6ed427",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(cleaned_sentences_with_stopwords[0]);"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "zirconium dental implants how common is it used now is there any advantages or benefits over titanium implants cons  pros please thanks\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lmbctU8J-H9P"
      },
      "source": [
        "##BERT Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xJWUucKE_V2L"
      },
      "source": [
        "import sklearn\n",
        "\n",
        "#finding cosine similarity from vectors\n",
        "from sklearn.metrics.pairwise import cosine_similarity;\n",
        "def retrieveAndPrintFAQAnswer(question_embedding,sentence_embeddings,FAQdf,sentences):\n",
        "    max_sim=-1;\n",
        "    index_sim=-1;\n",
        "    for index,faq_embedding in enumerate(sentence_embeddings):\n",
        "        sim=cosine_similarity(faq_embedding,question_embedding)[0][0];\n",
        "        #print(index, sim, sentences[index])\n",
        "        if sim>max_sim:\n",
        "            max_sim=sim;\n",
        "            index_sim=index;\n",
        "    print(\"\\n\")\n",
        "    print(\"Question: \",question)\n",
        "    print(\"\\n\");\n",
        "    print(\"Retrieved Question: \",FAQdf.iloc[index_sim,0]) \n",
        "    print(\"Retrieved Answer: \",FAQdf.iloc[index_sim,1]) "
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kCEgHmFl_XNz"
      },
      "source": [
        "#giving input question\n",
        "question_orig=\"How to keep good health ?\"\n",
        "#cleaning the question\n",
        "question=clean_sentence(question_orig,stopwords=False);"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q3eKl23h-nJN",
        "outputId": "c7c8a121-abf7-45fd-9f23-5dd1d6e7135f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "cleaned_sentences=get_cleaned_sentences(df,stopwords=False)\n",
        "#creating word embeddings\n",
        "import time\n",
        "m1 = time.time()\n",
        "sent_bertphrase_embeddings=[];\n",
        "\n",
        "for sent in cleaned_sentences:\n",
        "    sent_bertphrase_embeddings.append(bc.encode([sent]));\n",
        "m2 = time.time()\n",
        "print(\"Question Embedding time for Huge dataset\",m2 - m1)\n",
        "question_embedding=bc.encode([question]);\n",
        "\n",
        "retrieveAndPrintFAQAnswer(question_embedding,sent_bertphrase_embeddings,df, cleaned_sentences);"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Question Embedding time for Huge dataset 3334.6015825271606\n",
            "\n",
            "\n",
            "Question:  how to keep good health \n",
            "\n",
            "\n",
            "Retrieved Question:  how to eat healthy and get fit?\n",
            "Retrieved Answer:  portion control (use salad plate instead of dinner plate); 4-6 mini-meals/day; avoid fast food & sugary & diet drinks. lean meats plenty of fruits & veggies (choose low glycemic ones) low-fat dairy eggs good fats (olive/canola) avoid starches (bread/rice /potatoes/pasta—unless whole grain); exercise 30 min/day (walking/pool exercises); can add weights/ machines with guidance to prevent injury.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ONl2oh28HSIL"
      },
      "source": [
        "#Implementing Chatbot using BERT Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aNaXq6GJHWSt",
        "outputId": "92e4c789-b56c-40f9-e12d-ed50fdb71e5f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import sklearn\n",
        "from sklearn.metrics.pairwise import cosine_similarity;\n",
        "import numpy\n",
        "import time\n",
        "\n",
        "print(\"Welcome to the Question Answering System\")      \n",
        "\n",
        "while True:\n",
        "  print(\"Question:\") \n",
        "  \n",
        "  question_orig=input()\n",
        "  \n",
        "  if (question_orig.lower()=='quit'):\n",
        "    break\n",
        "  measure1 = time.time()\n",
        "  question=clean_sentence(question_orig,stopwords=False);\n",
        "  \n",
        "\n",
        "  question_embedding=bc.encode([question]);  \n",
        "  def retrieveAndPrintFAQAnswer(question_embedding,sentence_embeddings,FAQdf,sentences):\n",
        "    max_sim=-1;\n",
        "    index_sim=-1;\n",
        "    for index,faq_embedding in enumerate(sentence_embeddings):\n",
        "      sim=cosine_similarity(faq_embedding,question_embedding)[0][0]\n",
        "      #print(index, sim, sentences[index])\n",
        "      if sim>max_sim:\n",
        "        max_sim=sim;\n",
        "        index_sim=index;\n",
        "    #print(\"\\n\")\n",
        "    #print(\"Question: \",question)\n",
        "    #print(\"\\n\");\n",
        "    print(\"Retrieved: \",FAQdf.iloc[index_sim,0]) \n",
        "    #print(FAQdf.iloc[index_sim,1]) \n",
        "    print(\"Answer:\",FAQdf.iloc[index_sim,1])\n",
        "    print(\"----------------------\")\n",
        "    measure2 = time.time()\n",
        "    print(\"Response Time\",measure2 - measure1)\n",
        "    print(\"----------------------\")\n",
        "    print(\"\\n\");\n",
        "    print(\"Enter Quit to exit: \")\n",
        "  retrieveAndPrintFAQAnswer(question_embedding,sent_bertphrase_embeddings,df, cleaned_sentences);\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "print(\"----------------------\")\n",
        "print(\"Thank You\")\n",
        "print(\"----------------------\")"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Welcome to the Question Answering System\n",
            "Question:\n",
            "How to take care of health?\n",
            "Retrieved:  how to avoid obesity?\n",
            "Answer: stay active and eat right. also make sure you get sleep and stay clear of excess stress.\n",
            "----------------------\n",
            "Response Time 14.445129871368408\n",
            "----------------------\n",
            "\n",
            "\n",
            "Enter Quit to exit: \n",
            "Question:\n",
            "Give some tips for healthy lifestyle?\n",
            "Retrieved:  give advise on weight loss programme?\n",
            "Answer: weight loss is possible. it takes a regular change in habits. adopting a low glycemic diet is helpful for many. regular activity daily and vigorous activity two times per week. sleep and manage stress. some supplements and programs can be helpful. <link>\n",
            "----------------------\n",
            "Response Time 14.472314357757568\n",
            "----------------------\n",
            "\n",
            "\n",
            "Enter Quit to exit: \n",
            "Question:\n",
            "Quit\n",
            "----------------------\n",
            "Thank You\n",
            "----------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "joKDCycKLZmD"
      },
      "source": [
        "#Dataset2-Easy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j_fMj1SFzvWH"
      },
      "source": [
        "##Import Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QUhQrEmFzvWI",
        "outputId": "309f95eb-606b-4db5-ef52-4549a911d42f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "import pandas as pd;\n",
        "\n",
        "#Loading dataset and renaming columns\n",
        "\n",
        "df=pd.read_csv(\"FAQ_standard.csv\");\n",
        "df.columns=[\"questions\",\"answers\",\"Label\"];\n",
        "\n",
        "df"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>answers</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>Job hunting experience involves networking to ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>There are many kinds of roles, data scientist,...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>Hiring is going to slow down. First in small c...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>The following are some skills employers usuall...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>For more senior roles: People typically look f...</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>Here are some tasks in the typical day of a da...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>Yes. In many data science interviews (ML Scien...</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>The following three are the basic building blo...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>Usually the data science interview has a subse...</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>Needs to be reasonably proficient. Again, a da...</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ... Label\n",
              "0   What does the job hunting experience look like ?  ...     1\n",
              "1  Any insights you can offer about the DS job ma...  ...     2\n",
              "2  What's the impact of Covid on hiring for DS ro...  ...     3\n",
              "3  What skills and qualities do employers look fo...  ...     4\n",
              "4       Do employers look for an advanced ML degree?  ...     5\n",
              "5  How does a typical day of a data scientist loo...  ...     6\n",
              "6  Is preparation of algorithms and data structur...  ...     7\n",
              "7  What is the mathematical background required t...  ...     8\n",
              "8  What are the various rounds in a data scientis...  ...     9\n",
              "9  What level of proficiency is needed for a data...  ...    10\n",
              "\n",
              "[10 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4R88GCS-0ReG",
        "outputId": "e43a1af9-e918-45b9-e5d3-faf860c3bd7e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "import pandas as pd;\n",
        "\n",
        "#Load dataset and examine dataset, rename columns to questions and answers\n",
        "\n",
        "df_test=pd.read_excel(\"FAQ_test_Easy.xlsx\")\n",
        "df_test.columns=[\"questions\",\"ActualCode\",\"PredictedCode\",\"Prediction?\",\"ActualQuestion\",\"PredictedQuestion\"]\n",
        "\n",
        "df_test"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>ActualCode</th>\n",
              "      <th>PredictedCode</th>\n",
              "      <th>Prediction?</th>\n",
              "      <th>ActualQuestion</th>\n",
              "      <th>PredictedQuestion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How is the job searching experience nowadays?</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>How are the DS jobs on the market, please?</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How is DS hiring impacted due to Covid?</td>\n",
              "      <td>3</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>In a data scientist what qualifications and at...</td>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Are employers looking for an advanced ML degree?</td>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>On a typical day what does a data scientist do?</td>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Do we need to study algorithms for a data scie...</td>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>What is the mathematical background needed to ...</td>\n",
              "      <td>8</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>How many rounds of interview are in a data sc...</td>\n",
              "      <td>9</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>How proficient a data scientist has to be in c...</td>\n",
              "      <td>10</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ...  PredictedQuestion\n",
              "0      How is the job searching experience nowadays?  ...                NaN\n",
              "1         How are the DS jobs on the market, please?  ...                NaN\n",
              "2            How is DS hiring impacted due to Covid?  ...                NaN\n",
              "3  In a data scientist what qualifications and at...  ...                NaN\n",
              "4   Are employers looking for an advanced ML degree?  ...                NaN\n",
              "5    On a typical day what does a data scientist do?  ...                NaN\n",
              "6  Do we need to study algorithms for a data scie...  ...                NaN\n",
              "7  What is the mathematical background needed to ...  ...                NaN\n",
              "8   How many rounds of interview are in a data sc...  ...                NaN\n",
              "9  How proficient a data scientist has to be in c...  ...                NaN\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "As-8_00HzvWQ"
      },
      "source": [
        "##BERT Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qF26zdNAzvWV",
        "outputId": "6e0c8da1-6826-45ba-e271-720bb20cc04a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#cleaning test data questions\n",
        "cleaned_sentences_test=get_cleaned_sentences(df_test,stopwords=False)\n",
        "print(cleaned_sentences_test);"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['how is the job searching experience nowadays', 'how are the ds jobs on the market please', 'how is ds hiring impacted due to covid', 'in a data scientist what qualifications and attributes do employers look for', 'are employers looking for an advanced ml degree', 'on a typical day what does a data scientist do', 'do we need to study algorithms for a data science interview', 'what is the mathematical background needed to be a data scientist ', 'how many rounds of interview are in a data science interview ', 'how proficient a data scientist has to be in coding']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vAc58kXNOjRF"
      },
      "source": [
        "#cleaning original data questions\n",
        "cleaned_sentences=get_cleaned_sentences(df,stopwords=False)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WxUZrX2ehDEi",
        "outputId": "28dd9201-81fc-4cbb-9622-cf257d988c91",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(cleaned_sentences)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['what does the job hunting experience look like ', 'any insights you can offer about the ds job market ', 'whats the impact of covid on hiring for ds roles', 'what skills and qualities do employers look for in a data scientist', 'do employers look for an advanced ml degree', 'how does a typical day of a data scientist look like', 'is preparation of algorithms and data structures needed for a data science interview ', 'what is the mathematical background required to be a data scientist ', 'what are the various rounds in a data scientist interview ', 'what level of proficiency is needed for a data scientist in coding']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mcAo9vJT80_Y"
      },
      "source": [
        "#Embedding original question dataset\n",
        "\n",
        "sent_bertphrase_embeddings=[];\n",
        "\n",
        "for sent in cleaned_sentences:\n",
        "    sent_bertphrase_embeddings.append(bc.encode([sent]));"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LwCPa1iM92k2"
      },
      "source": [
        "#Embeding test question dataset\n",
        "\n",
        "sent_bertphrase_embeddings_test=[];\n",
        "for sent_test in cleaned_sentences_test:\n",
        "    sent_bertphrase_embeddings_test.append(bc.encode([sent_test]));\n"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "czRpjKFj_35p",
        "outputId": "870c8422-a652-4a9f-809a-ca924dc038c5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import sklearn\n",
        "from sklearn.metrics.pairwise import cosine_similarity;\n",
        "def retrieveAndPrintFAQAnswer(question_embeddings_test1,sentence_embeddings1,FAQdf1,sentences1,FAQdf_test1,sentences_test1):\n",
        "  for index_test,faq_embedding_test in enumerate(question_embeddings_test1):\n",
        "    max_sim=-1;\n",
        "    index_sim=-1;\n",
        "    for index,faq_embedding in enumerate(sentence_embeddings1):\n",
        "        \n",
        "      sim=cosine_similarity(faq_embedding,faq_embedding_test)[0][0];\n",
        "      #print(index, sim, sentences[index])\n",
        "      if sim>max_sim:\n",
        "        max_sim=sim;\n",
        "        index_sim=index;\n",
        "\n",
        "    FAQdf_test1.iloc[index_test,2]=FAQdf1.iloc[index_sim,2]\n",
        "    FAQdf_test1.iloc[index_test,5]=FAQdf1.iloc[index_sim,0]\n",
        "    print(\"\\n\")\n",
        "    print(\"Question: \",sentences_test1[index_test])\n",
        "    print(\"\\n\");\n",
        "    print(\"Retrieved: \",FAQdf1.iloc[index_sim,0]) \n",
        "    print(\"Answer:\",FAQdf1.iloc[index_sim,1])        \n",
        "  df_test=FAQdf_test1 \n",
        "retrieveAndPrintFAQAnswer(sent_bertphrase_embeddings_test,sent_bertphrase_embeddings,df,cleaned_sentences,df_test,cleaned_sentences_test);"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Question:  how is the job searching experience nowadays\n",
            "\n",
            "\n",
            "Retrieved:  What does the job hunting experience look like ?\n",
            "Answer: Job hunting experience involves networking to get in touch with the right people in various companies, applying to lots of jobs through various channels, preparing for interviews – while interviews are uncertain it is necessary to prepare well what you CAN prepare, smart scheduling of interviews to get the best job and salary you can.\n",
            "\n",
            "\n",
            "Question:  how are the ds jobs on the market please\n",
            "\n",
            "\n",
            "Retrieved:  What does the job hunting experience look like ?\n",
            "Answer: Job hunting experience involves networking to get in touch with the right people in various companies, applying to lots of jobs through various channels, preparing for interviews – while interviews are uncertain it is necessary to prepare well what you CAN prepare, smart scheduling of interviews to get the best job and salary you can.\n",
            "\n",
            "\n",
            "Question:  how is ds hiring impacted due to covid\n",
            "\n",
            "\n",
            "Retrieved:  What's the impact of Covid on hiring for DS roles?\n",
            "Answer: Hiring is going to slow down. First in small companies then eventually in enterprises in a couple of months. But it is likely to pick up in a few months. Hence, if you are looking to transition, this is a good time to ramp up skills and learn new things that might usually take some time.\n",
            "\n",
            "\n",
            "Question:  in a data scientist what qualifications and attributes do employers look for\n",
            "\n",
            "\n",
            "Retrieved:  What skills and qualities do employers look for in a data scientist?\n",
            "Answer: The following are some skills employers usually look for: \n",
            "\n",
            "(1) Should be good at coding \n",
            "(2) Should have good problem solving and analysis skills \n",
            "(3) Should be good with stats and building testing and deploying models. \n",
            "\n",
            "A data scientist is a software developer ++ with stats and modeling skills to build and deploy models and make inferences from data.\n",
            "\n",
            "\n",
            "Question:  are employers looking for an advanced ml degree\n",
            "\n",
            "\n",
            "Retrieved:  Do employers look for an advanced ML degree?\n",
            "Answer: For more senior roles: People typically look for practical experience for several years or a specialized degree with some experience. Employers typically want folks who have seen various kinds of ML problems and solved them, who can come up with the right solution for a new problem when they are looking for a more senior role.   \n",
            "\n",
            "For a beginner role: a degree usually adds some level of credibility, but if one has a good portfolio with good projects, it is usually an accepted substitute. Employers do hope to see if the candidate is comfortable with basic concepts in ML along with being able to write code.\n",
            "\n",
            "\n",
            "Question:  on a typical day what does a data scientist do\n",
            "\n",
            "\n",
            "Retrieved:  How does a typical day of a data scientist look like?\n",
            "Answer: Here are some tasks in the typical day of a data scientist:\n",
            "\n",
            "Make a plan for the day\n",
            "Look at data, what clean up is required, figure out what models can be built\n",
            "Talk to various stakeholders about what modeling is possible and help them narrow down to something useful for the business\n",
            "Build models, test and debug (takes a long time)\n",
            "Parameter tuning – test tons and tons of parameters (takes a long time)\n",
            "Come up with prod architecture to get deployment ready\n",
            "Write ML pipeline for production ready modeles – deploy them\n",
            "Wait for long time till we have a significant sample to see if they are working\n",
            "Analyze and see whether the models are working as expected, have any impact\n",
            "Come up with improvements/ corrections based on prod feedback and prepare for next iteration.\n",
            "Meeting with team members / daily sprints / bug triages based on production feedback – Interaction with ML Manager, Product Manager, Developers, Data engineers\n",
            "\n",
            "\n",
            "Question:  do we need to study algorithms for a data science interview\n",
            "\n",
            "\n",
            "Retrieved:  Is preparation of algorithms and data structures needed for a data science interview ?\n",
            "Answer: Yes. In many data science interviews (ML Scientist, data scientist, ML Engineer, Data engineer) there is a coding round and an algorithms round. So preparing algorithms and data structures is necessary. Some of the Analyst interviews, such as a data analyst and a business analyst might not have an algorithms and data structures round.\n",
            "\n",
            "\n",
            "Question:  what is the mathematical background needed to be a data scientist \n",
            "\n",
            "\n",
            "Retrieved:  What is the mathematical background required to be a data scientist ?\n",
            "Answer: The following three are the basic building blocks in terms of data science math background: Linear Algebra, Probability and Statistics and Calculus and optimization.\n",
            "\n",
            "\n",
            "Question:  how many rounds of interview are in a data science interview \n",
            "\n",
            "\n",
            "Retrieved:  What are the various rounds in a data scientist interview ?\n",
            "Answer: Usually the data science interview has a subset of these rounds.\n",
            "\n",
            "Resume deep dive\n",
            "ML Concepts\n",
            "ML Scenario and Problem solving\n",
            "Algorithms and data structures\n",
            "Coding\n",
            "Behavioural\n",
            "\n",
            "Sometimes, some of these rounds might be combined, for instance there might not be two separate rounds for coding and algorithms. Similarly there might be a single round for ML concepts and scenario based problem solving.\n",
            "\n",
            "\n",
            "Question:  how proficient a data scientist has to be in coding\n",
            "\n",
            "\n",
            "Retrieved:  What level of proficiency is needed for a data scientist in coding?\n",
            "Answer: Needs to be reasonably proficient. Again, a data scientist is a developer ++. Usually expected to have the basic skills a developer has including understanding algorithms and data structures and being able to write clean, understable, efficient, well documented code.\n",
            "\n",
            "In many cases, the data scientist is expected to write production ready code and be able to understand the deployment process. In some cases, the coding could be for decision support.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J-Y59iBfAQcn",
        "outputId": "427ebf78-faf9-46b7-8da1-c4bf3699f7e9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 511
        }
      },
      "source": [
        "df_test"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>ActualCode</th>\n",
              "      <th>PredictedCode</th>\n",
              "      <th>Prediction?</th>\n",
              "      <th>ActualQuestion</th>\n",
              "      <th>PredictedQuestion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How is the job searching experience nowadays?</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>How are the DS jobs on the market, please?</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How is DS hiring impacted due to Covid?</td>\n",
              "      <td>3</td>\n",
              "      <td>3.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>In a data scientist what qualifications and at...</td>\n",
              "      <td>4</td>\n",
              "      <td>4.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Are employers looking for an advanced ML degree?</td>\n",
              "      <td>5</td>\n",
              "      <td>5.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>On a typical day what does a data scientist do?</td>\n",
              "      <td>6</td>\n",
              "      <td>6.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Do we need to study algorithms for a data scie...</td>\n",
              "      <td>7</td>\n",
              "      <td>7.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>What is the mathematical background needed to ...</td>\n",
              "      <td>8</td>\n",
              "      <td>8.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>How many rounds of interview are in a data sc...</td>\n",
              "      <td>9</td>\n",
              "      <td>9.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>How proficient a data scientist has to be in c...</td>\n",
              "      <td>10</td>\n",
              "      <td>10.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ...                                  PredictedQuestion\n",
              "0      How is the job searching experience nowadays?  ...   What does the job hunting experience look like ?\n",
              "1         How are the DS jobs on the market, please?  ...   What does the job hunting experience look like ?\n",
              "2            How is DS hiring impacted due to Covid?  ...  What's the impact of Covid on hiring for DS ro...\n",
              "3  In a data scientist what qualifications and at...  ...  What skills and qualities do employers look fo...\n",
              "4   Are employers looking for an advanced ML degree?  ...       Do employers look for an advanced ML degree?\n",
              "5    On a typical day what does a data scientist do?  ...  How does a typical day of a data scientist loo...\n",
              "6  Do we need to study algorithms for a data scie...  ...  Is preparation of algorithms and data structur...\n",
              "7  What is the mathematical background needed to ...  ...  What is the mathematical background required t...\n",
              "8   How many rounds of interview are in a data sc...  ...  What are the various rounds in a data scientis...\n",
              "9  How proficient a data scientist has to be in c...  ...  What level of proficiency is needed for a data...\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I0d5jdxhBPsv",
        "outputId": "334c7bd5-e6da-4dc3-afd5-9c657bae5019",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 511
        }
      },
      "source": [
        "for i in range(df_test.shape[0]):\n",
        " if df_test.iloc[i,1]!=df_test.iloc[i,2]:\n",
        "   df_test.iloc[i,3]='InCorrect'\n",
        " else:\n",
        "   df_test.iloc[i,3]='correct'\n",
        "df_test"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>ActualCode</th>\n",
              "      <th>PredictedCode</th>\n",
              "      <th>Prediction?</th>\n",
              "      <th>ActualQuestion</th>\n",
              "      <th>PredictedQuestion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How is the job searching experience nowadays?</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>How are the DS jobs on the market, please?</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "      <td>InCorrect</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How is DS hiring impacted due to Covid?</td>\n",
              "      <td>3</td>\n",
              "      <td>3.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>In a data scientist what qualifications and at...</td>\n",
              "      <td>4</td>\n",
              "      <td>4.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Are employers looking for an advanced ML degree?</td>\n",
              "      <td>5</td>\n",
              "      <td>5.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>On a typical day what does a data scientist do?</td>\n",
              "      <td>6</td>\n",
              "      <td>6.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Do we need to study algorithms for a data scie...</td>\n",
              "      <td>7</td>\n",
              "      <td>7.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>What is the mathematical background needed to ...</td>\n",
              "      <td>8</td>\n",
              "      <td>8.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>How many rounds of interview are in a data sc...</td>\n",
              "      <td>9</td>\n",
              "      <td>9.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>How proficient a data scientist has to be in c...</td>\n",
              "      <td>10</td>\n",
              "      <td>10.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ...                                  PredictedQuestion\n",
              "0      How is the job searching experience nowadays?  ...   What does the job hunting experience look like ?\n",
              "1         How are the DS jobs on the market, please?  ...   What does the job hunting experience look like ?\n",
              "2            How is DS hiring impacted due to Covid?  ...  What's the impact of Covid on hiring for DS ro...\n",
              "3  In a data scientist what qualifications and at...  ...  What skills and qualities do employers look fo...\n",
              "4   Are employers looking for an advanced ML degree?  ...       Do employers look for an advanced ML degree?\n",
              "5    On a typical day what does a data scientist do?  ...  How does a typical day of a data scientist loo...\n",
              "6  Do we need to study algorithms for a data scie...  ...  Is preparation of algorithms and data structur...\n",
              "7  What is the mathematical background needed to ...  ...  What is the mathematical background required t...\n",
              "8   How many rounds of interview are in a data sc...  ...  What are the various rounds in a data scientis...\n",
              "9  How proficient a data scientist has to be in c...  ...  What level of proficiency is needed for a data...\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XYmWMFJ6BfV9"
      },
      "source": [
        "list_data_answers=df_test.ActualCode.values.tolist()\n",
        "list_data_FoundCode=df_test.PredictedCode.values.tolist()"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X4RAC3NaBh_Z",
        "outputId": "b058b5f1-483e-4908-c6ea-64d2d56e1477",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#printing confusion matrix\n",
        "from sklearn.metrics import multilabel_confusion_matrix\n",
        "from sklearn import metrics\n",
        "multilabel_confusion_matrix(list_data_answers, list_data_FoundCode)\n",
        "#tn, fp, fn, tp = confusion_matrix([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1.0, 2.0, 2.0, 10.0, 4.0, 1.0, 2.0, 1.0, 2.0, 2.0]).ravel()\n",
        "#print(metrics.classification_report(list_data_answers, list_data_FoundCode))"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[[8, 1],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [1, 0]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G9eRPsLpBkTp"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from mlxtend.evaluate import confusion_matrix\n",
        "\n",
        "\n",
        "cm = confusion_matrix(list_data_answers, list_data_FoundCode, binary=False)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zrAkUo-TBmSF",
        "outputId": "852d943e-bed5-4df3-89da-e3702ad51d00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from mlxtend.evaluate import confusion_matrix\n",
        "from mlxtend.plotting import plot_confusion_matrix\n",
        "\n",
        "import pprint\n",
        "#printing confusion matrix\n",
        "fig, ax = plot_confusion_matrix(conf_mat=cm)\n",
        "plt.show()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQMAAAEGCAYAAABhHPB4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAT9UlEQVR4nO3de7RcdXnG8e9DYiqBCNTEdpFAE+4GlxI5WBTrUqBUBMG6tILaFlHTIgp4R6uiVRdaqNWqVRC8rIqgBKxKEazWawohh0SuAY1cE2JJUDELrSHh7R/zOzIk58yZnJnf2fMmz2etLM7M7PPs9+wV3uw9s3/vUURgZrZD0wWY2WBwMzAzwM3AzAo3AzMD3AzMrJjadAHtNHXH0LQZfc9d8OQ9+55pltHdd9/FunXrNNprg9UMps3gD/b/q77nLl7yib5nmmV02J8OjfmaLxPMDHAzMLPCzcDMADcDMyvcDMwMSNYMPn3WK7j7O2czfOk7+579rauv4qkH7s+BB+zDOf/0oYHNdG69zO09t2ozkPR8SbdLWinpzF7z/v0b13L8qZ/sR2mPsWnTJs447VS+9o1vsvzGW7n0kotZceutA5fp3Hy1Zsqt1gwkTQE+CRwNzAdOlDS/l8zFy37GLx78TT/Ke4yl113H3nvvw7y99mLatGm89GUncMU3vjZwmc7NV2um3JpnBs8AVkbEHRGxAbgEOL7i/ibsvvtWM2fOHr9/PHv2HFavXj1wmc6tl+ncus1gNnBv2+NV5bnHkLRQ0rCk4dj424rlmFknjb+BGBHnR8RQRAxp6o6N1LD77rNZterRvrV69Spmz96ibzWe6dx6mc6t2wxWA3u0PZ5Tnhs4Q4ccwsqVP+WuO+9kw4YNXPrlSzjm2OMGLtO5+WrNlFtzodJSYF9J82g1gROAl/cS+IWzT+LPDt6XmbvuzMqr3s/7P30lX/iPa3oudOrUqfzLxz7BC4/5CzZt2sTfnnQy8w88cOAynZuv1ky5qjkQVdILgI8CU4DPRsQHO22/w/QnRY1Vi79c6lWLZtBatXj99cOTv4Q5Iq4Erqy5DzPrj8bfQDSzweBmYGaAm4GZFW4GZga4GZhZ4WZgZoCbgZkVbgZmBrgZmFnhZmBmgJuBmRVuBmYGJGsGno7s3JqZ23tuqmbg6cjOzVZrptxUzcDTkZ2brdZMuamaQS2e4JsrN1OtmXLdDMwMcDMAPME3W26mWjPluhngCb7ZcjPVmim36gzEfvN0ZOdmqzVTbtXpyFvL05HN6uo0HdmXCWYGuBmYWeFmYGaAm4GZFW4GZga4GZhZMVD3GSx48p4sXpLnY8DdDnl93zP9Mag1xWcGZga4GZhZ4WZgZoCbgZkVbgZmBrgZmFmRrhlkmDI7otY050zHoFZuplqz5FZrBpL2kPRdSbdKukXS6b1mZpkyO6LGNOdsx8DTkfPk1jwz2Ai8OSLmA4cCp0qa30tglimzI2pMc852DDwdOU9utWYQEWsiYln5ej2wAuhp8FuWKbM1ZTsGno6cJ3dS3jOQNBdYACwZ5bWFkoYlDa9dt3YyyjGzUVRvBpJ2Bi4DzoiIX2/+ekScHxFDETE0a+asjllZpszWlO0YeDpyntyqzUDS42g1gosi4vJe87JMma0p2zHwdOQ8udVWLUoScCGwIiI+0o/MLFNmR9SY5pztGHg6cp7catORJT0b+CFwE/BIefqdEXHlWN9z8MFDsXjJcJV6avASZsum03TkamcGEfEjYNSdmtngSXcHopnV4WZgZoCbgZkVbgZmBgzYQNRsarzzX+MTCvCnFDY+nxmYGeBmYGaFm4GZAW4GZla4GZgZ4GZgZkW6ZpBhsGTNTPCg1VqZ23tuqmaQZbBkzVrBg1Yz1ZopN1UzyDJYsmat4EGrmWrNlJuqGWQZLFkrsyYfW+emagZmVk+qZpBlsGStzJp8bJ2bqhlkGSxZs9ZafGydm2rVYpbBkjVrBQ9azVRrptxqA1EnIttA1Bq8hNlq6jQQNdVlgpnV42ZgZoCbgZkVbgZmBrgZmFnhZmBmQLL7DLYHtT4C9EeWNp4xm4Gk9cDITQgjn0tG+Toi4gmVazOzSTRmM4iIGZNZiJk1q6v3DCQ9W9KrytczJc2rW5aZTbZxm4Gks4C3A+8oT00DvlizKDObfN2cGfwlcBzwEEBE3Af4EsJsG9NNM9gQrdVMASBpp7olmVkTumkGX5F0HrCrpNcC3wY+U7essWWYMlszs1ZurYnL4GObJXfcZhAR5wKLgMuA/YD3RMTHu92BpCmSlku6YuJltmSZMputVqgzcRl8bDPldnsH4k3AD4EflK+3xunAiq38nlFlmTKbrVaoM3EZfGwz5XbzacJrgOuAFwMvAa6VdHI34ZLmAMcAF0y4wjZZpszWyqyZW4uPbZ7cbm5HfiuwICIeAJD0ROB/gM928b0fBd5Gh08fJC0EFgLsseeeXUSaWQ3dXCY8AKxve7y+PNeRpGOB+yPi+k7bRcT5ETEUEUOzZs7qmJllymytzJq5tfjY5skdsxlIepOkNwErgSWS3ltuQLoW+EkX2YcBx0m6C7gEOFxSTzcrZZkym63Wmnxs8+R2ukwYObX/Wfkzoqt3KCLiHZS7FiU9F3hLRLxyAjX+XpYps9lqhToTl2vVm+3YZsmdlOnIbc3g2E7beTpyPV7CbNB5OvK4byBKmkXrTcADgcePPB8Rh3dbQER8D/het9ub2eTr5g3Ei4DbgHnA+4C7gKUVazKzBnTTDJ4YERcCD0fE9yPiZKDrswIzy6Gb+wweLv9dI+kY4D7gD+uVZGZN6KYZfEDSLsCbgY8DTwDeWLUqM5t04zaDiBhZYPQg8Ly65ZhZUzoNRP04jw5E3UJEnFalIqvCU5dtPJ3ODPyBv9l2pNN05C9MZiFm1iz/RiUzA9wMzKxwMzAzoLtJR/tJ+o6km8vjp0p6V/3SzGwydXNm8BlaS5EfBoiIG4ETahbVSYYpszUzs+XWmrqc6Rhkye2mGUyPiOs2e25jT3udoCxTZrPVWjO3xtTlbMcgS243zWCdpL159JeovARYM+E99iDLlNlstdbMrTF1OdsxyJLbTTM4FTgPOEDSauAM4JQJ77EHWabM1srMmFtDtmOQJbebtQl3AEeWX6u2Q0SsH+97zCyfbiYdvWezxwBExD9WqmlMWabM1srMmFtDtmOQJbeby4SH2v5sAo4G5k54jz3IMmU2W601c2vIdgyy5HZzmfDP7Y8lnQtcPeE99iDLlNlstdbMrTF1OdsxyJK71dORJe0GLI2IfSa81zF4OnI+XsKcS6/TkW/i0bkGU4BZwKS/X2BmdXUz9qz9dx1sBP43Ihq56cjM6unYDCRNAa6OiAMmqR4za0jHTxMiYhNwuyT/emSzbVw3lwm7AbdIuo7Wx4sARMRgfu5kZhPSTTN4d/UqzKxx3TSDF0TE29ufkPRh4Pt1SrJMPHV529HNHYh/PspzR/e7EDNrVqffm3AK8DpgL0k3tr00A1hcuzAzm1ydLhO+BHwTOBs4s+359RHxi6pVmdmk6/R7Ex6k9SvVTpy8csysKZ6ObGaAm4GZFemaQYYpszUzndviqcv9z63aDCTtKmmRpNskrZD0zF7yskyZzVZrxlxPXW5mOnIvPgZcVRY6PQ1Y0UtYlimz2WrNmOupy81MR54QSbsAzwEuBIiIDRHxq14ys0yZrZXp3LqyHYN+59Y8M5gHrAU+J2m5pAvKhOXHkLRQ0rCk4bXr1lYsx8w6qdkMpgJPBz4VEQtorXg8c/ONIuL8iBiKiKFZM2d1DMwyZbZWpnPrynYMmpiOPFGrgFURsaQ8XkSrOUxYlimz2WrNmFtDtmMw6dORJyoifi7pXkn7R8TtwBFAT2+hZpkym63WjLmeujwA05G3Klw6CLgAmAbcAbwqIn451vaejmwjvIS5jp6mI/ciIn4MDNXch5n1R7o7EM2sDjcDMwPcDMyscDMwM8DNwMyKqp8mmE1UpqnL28rHlT4zMDPAzcDMCjcDMwPcDMyscDMwMyBhM8gwWLJmpnPrZcL2PWg1VTPIMlgyW63ZcmvVCtv3oNVUzSDLYMlstWbLrVUrbN+DVlM1gyyDJWtlOrdeZk1Zjm2qZmBm9aRqBlkGS9bKdG69zJqyHNtUzSDLYMlstWbLzTRkFfIc21QLlbIMlsxWa7bcWrXC9j1otepA1K3lgahW2/a+arHTQNRUlwlmVo+bgZkBbgZmVrgZmBngZmBmhZuBmQHJ7jMw61WNjwG3ld8L6TMDMwPcDMyscDMwM8DNwMwKNwMzA9wMzKxI1wwyTJmtmencepm1cmtNXIZE05ElvVHSLZJulnSxpMf3kpdlymy2WrPlZqoV6kxchkTTkSXNBk4DhiLiKcAU4IReMrNMmc1Wa7bcTLVCnYnLkG868lRgR0lTgenAfb2EZZkyWyvTufUya+bWkmY6ckSsBs4F7gHWAA9GxLc2307SQknDkobXrltbqxwzG0fNy4TdgOOBecDuwE6SXrn5dhFxfkQMRcTQrJmzOmZmmTJbK9O59TJr5taSaTrykcCdEbE2Ih4GLgee1Utglimz2WrNlpup1poyTUe+BzhU0nTgt8ARQE/TTrNMmc1Wa7bcTLVCnYnLNeqtOh1Z0vuAlwEbgeXAayLid2Nt7+nIllGmJcydpiNXnWcQEWcBZ9Xch5n1R7o7EM2sDjcDMwPcDMyscDMwM8DNwMwKT0c261GtKcY1PrL83e33jPmazwzMDHAzMLPCzcDMADcDMyvcDMwMcDMwsyJdM8g0FTdTrdlyM9VaK7ffU5dTNYNMU3Ez1ZotN1OtNXP7PXU5VTPINBU3U63ZcjPVWjO331OXUzWDTFNxM9WaLTdTrTVz+y1VMzCzelI1g0xTcTPVmi03U601c/stVTPINBU3U63ZcjPVWjO331KtWsw0FTdTrdlyM9VaM7ffU5erTkfeWp6ObPaoOkuYv8Ijv7l/1OnIqS4TzKweNwMzA9wMzKxwMzAzwM3AzAo3AzMDBuyjRUlrgbu72HQmsK5yOf2Uqd5MtUKuegeh1j+JiFmjvTBQzaBbkoYjYqjpOrqVqd5MtUKuege9Vl8mmBngZmBmRdZmcH7TBWylTPVmqhVy1TvQtaZ8z8DM+i/rmYGZ9ZmbgZkBCZuBpOdLul3SSklnNl3PWCTtIem7km6VdIuk05uuqRuSpkhaLumKpmvpRNKukhZJuk3SCknPbLqmTiS9sfw9uFnSxZIe33RNm0vVDCRNAT4JHA3MB06UNL/Zqsa0EXhzRMwHDgVOHeBa250OrGi6iC58DLgqIg4AnsYA1yxpNnAaMBQRTwGmACc0W9WWUjUD4BnAyoi4IyI2AJcAxzdc06giYk1ELCtfr6f1l3XwBt+1kTQHOAa4oOlaOpG0C/Ac4EKAiNgQEb9qtqpxTQV2lDQVmA7c13A9W8jWDGYD97Y9XsWA/w8GIGkusABY0mwl4/oo8DbgkaYLGcc8YC3wuXJJc4GknZouaiwRsRo4F7gHWAM8GBHfaraqLWVrBulI2hm4DDgjIn7ddD1jkXQscH9EXN90LV2YCjwd+FRELAAeAgb5/aPdaJ3BzgN2B3aS9Mpmq9pStmawGtij7fGc8txAkvQ4Wo3gooi4vOl6xnEYcJyku2hdfh0u6YvNljSmVcCqiBg501pEqzkMqiOBOyNibUQ8DFwOPKvhmraQrRksBfaVNE/SNFpvwny94ZpGJUm0rmlXRMRHmq5nPBHxjoiYExFzaR3X/46IgfvXCyAifg7cK2n/8tQRQO+/vLCee4BDJU0vfy+OYADf8Ew1Kj0iNkp6PXA1rXdkPxsRtzRc1lgOA/4auEnSj8tz74yIKxusaVvyBuCi8o/CHcCrGq5nTBGxRNIiYBmtT5mWM4C3Jvt2ZDMD8l0mmFklbgZmBrgZmFnhZmBmgJuBmRVuBtspSc8dWZko6bhOK0DLCsHXTWAf75X0lm6f32ybz0t6yVbsa66km7e2RnuUm8E2pqzs3CoR8fWI+FCHTXYFtroZWC5uBkmUf/luk3RRWb+/SNL08tpdkj4saRnwUklHSbpG0jJJl5b1ESOzIG4r2724LfskSZ8oX/+RpK9KuqH8eRbwIWBvST+WdE7Z7q2Slkq6UdL72rL+QdJPJP0I2J9xSHptyblB0mUjP1NxpKThknds2X6KpHPa9v13vR5ba3EzyGV/4N8i4snAr3nsv9YPRMTTgW8D7wKOLI+HgTeVYRqfAV4IHAz88Rj7+Ffg+xHxNFr3+99CaxHQzyLioIh4q6SjgH1pLSk/CDhY0nMkHUzrVuaDgBcAh3TxM10eEYeU/a0AXt322tyyj2OAT5ef4dW0Vv0dUvJfK2leF/uxcaS6Hdm4NyIWl6+/SGtgxrnl8ZfLfw+lNfhlces2eKYB1wAH0Fos81OAsghp4Sj7OBz4G4CI2AQ8WFbdtTuq/FleHu9MqznMAL4aEb8p++hm3chTJH2A1qXIzrRuNR/xlYh4BPippDvKz3AU8NS29xN2Kfv+SRf7sg7cDHLZ/N7x9scPlf8K+K+IOLF9Q0kH9bEOAWdHxHmb7eOMCWR9HnhRRNwg6STguW2vjfbzCnhDRLQ3jZGZEdYDXybksmfbrL+XAz8aZZtrgcMk7QMgaSdJ+wG3AXMl7V22O3GU7wX4DnBK+d4pZarQelr/6o+4Gji57b2I2ZKeBPwAeJGkHSXNoHVJMp4ZwJqy3PsVm732Ukk7lJr3Am4v+z6lbI+k/QZ5sEkmbga53E5rluIKYDfgU5tvEBFrgZOAiyXdSLlEiIj/o3VZ8J/lDcT7x9jH6cDzJN0EXA/Mj4gHaF123CzpnDKl50vANWW7RcCMMubty8ANwDdpLTkfz7tpTYBaTKthtbsHuK5k/X35GS6gtVx5Wfko8Tx8htsXXrWYRDkNvqIM1DTrO58ZmBngMwMzK3xmYGaAm4GZFW4GZga4GZhZ4WZgZgD8PyVFmiDs2ps0AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "13LA4OWABoHB",
        "outputId": "bb588d90-9bf7-48d0-ba36-4b6f3805520d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(metrics.classification_report(list_data_answers, list_data_FoundCode))"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.50      1.00      0.67         1\n",
            "           2       0.00      0.00      0.00         1\n",
            "           3       1.00      1.00      1.00         1\n",
            "           4       1.00      1.00      1.00         1\n",
            "           5       1.00      1.00      1.00         1\n",
            "           6       1.00      1.00      1.00         1\n",
            "           7       1.00      1.00      1.00         1\n",
            "           8       1.00      1.00      1.00         1\n",
            "           9       1.00      1.00      1.00         1\n",
            "          10       1.00      1.00      1.00         1\n",
            "\n",
            "    accuracy                           0.90        10\n",
            "   macro avg       0.85      0.90      0.87        10\n",
            "weighted avg       0.85      0.90      0.87        10\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eAzR_uh9Cb5P"
      },
      "source": [
        "#Dataset3-Medium"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nmZC5IN5Cb5R"
      },
      "source": [
        "##Import Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiu8p5a9Cb5T",
        "outputId": "6838b398-83eb-4a33-be81-aaeb83050268",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        }
      },
      "source": [
        "import pandas as pd;\n",
        "\n",
        "#Loading dataset and renaming columns\n",
        "\n",
        "df=pd.read_csv(\"FAQ_standard.csv\");\n",
        "df.columns=[\"questions\",\"answers\",\"Label\"];\n",
        "\n",
        "df"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>answers</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>Job hunting experience involves networking to ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>There are many kinds of roles, data scientist,...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>Hiring is going to slow down. First in small c...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>The following are some skills employers usuall...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>For more senior roles: People typically look f...</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>Here are some tasks in the typical day of a da...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>Yes. In many data science interviews (ML Scien...</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>The following three are the basic building blo...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>Usually the data science interview has a subse...</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>Needs to be reasonably proficient. Again, a da...</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ... Label\n",
              "0   What does the job hunting experience look like ?  ...     1\n",
              "1  Any insights you can offer about the DS job ma...  ...     2\n",
              "2  What's the impact of Covid on hiring for DS ro...  ...     3\n",
              "3  What skills and qualities do employers look fo...  ...     4\n",
              "4       Do employers look for an advanced ML degree?  ...     5\n",
              "5  How does a typical day of a data scientist loo...  ...     6\n",
              "6  Is preparation of algorithms and data structur...  ...     7\n",
              "7  What is the mathematical background required t...  ...     8\n",
              "8  What are the various rounds in a data scientis...  ...     9\n",
              "9  What level of proficiency is needed for a data...  ...    10\n",
              "\n",
              "[10 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ULsXb7IwCb5c",
        "outputId": "61f5ebab-b972-4213-d0e2-5ba502baf58f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 377
        }
      },
      "source": [
        "import pandas as pd;\n",
        "\n",
        "#Load dataset and examine dataset, rename columns to questions and answers\n",
        "\n",
        "df_test=pd.read_excel(\"FAQ_test_Medium.xlsx\")\n",
        "df_test.columns=[\"questions\",\"ActualCode\",\"PredictedCode\",\"Prediction?\",\"ActualQuestion\",\"PredictedQuestion\"]\n",
        "\n",
        "df_test"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>ActualCode</th>\n",
              "      <th>PredictedCode</th>\n",
              "      <th>Prediction?</th>\n",
              "      <th>ActualQuestion</th>\n",
              "      <th>PredictedQuestion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How does job searching experience would look?</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Please provide how DS jobs are in the market?</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How has the Covid impacted DS hiring?</td>\n",
              "      <td>3</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>What skills do employers want from data scienc...</td>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Is advanced degree in ML required?</td>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>What data scientist do in a typical day?</td>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>For an interview in data science are we requir...</td>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>8</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>How many rounds of interview are required to c...</td>\n",
              "      <td>9</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>How much proficient a data scientist should be...</td>\n",
              "      <td>10</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ...  PredictedQuestion\n",
              "0      How does job searching experience would look?  ...                NaN\n",
              "1      Please provide how DS jobs are in the market?  ...                NaN\n",
              "2              How has the Covid impacted DS hiring?  ...                NaN\n",
              "3  What skills do employers want from data scienc...  ...                NaN\n",
              "4                 Is advanced degree in ML required?  ...                NaN\n",
              "5           What data scientist do in a typical day?  ...                NaN\n",
              "6  For an interview in data science are we requir...  ...                NaN\n",
              "7  What is the mathematical background required t...  ...                NaN\n",
              "8  How many rounds of interview are required to c...  ...                NaN\n",
              "9  How much proficient a data scientist should be...  ...                NaN\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Q_5foxVCb5k"
      },
      "source": [
        "##BERT Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ryeW5QrLCb5l",
        "outputId": "dd002bbc-d548-41ba-efa5-fef27efabf19",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#cleaning test data questions\n",
        "cleaned_sentences_test=get_cleaned_sentences(df_test,stopwords=False)\n",
        "print(cleaned_sentences_test);"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['how does job searching experience would look', 'please provide how ds jobs are in the market', 'how has the covid impacted ds hiring', 'what skills do employers want from data science positions', 'is advanced degree in ml required', 'what data scientist do in a typical day', 'for an interview in data science are we required to study algorithms', 'what is the mathematical background required to be a data scientist ', 'how many rounds of interview are required to clear data scientist interview ', 'how much proficient a data scientist should be in coding']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mjvX9XL3Cb5q"
      },
      "source": [
        "#cleaning original data questions\n",
        "cleaned_sentences=get_cleaned_sentences(df,stopwords=False)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "swFEEJZuCb5v",
        "outputId": "d566cc37-2861-42c4-8fe7-ecf39c3c264d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(cleaned_sentences)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['what does the job hunting experience look like ', 'any insights you can offer about the ds job market ', 'whats the impact of covid on hiring for ds roles', 'what skills and qualities do employers look for in a data scientist', 'do employers look for an advanced ml degree', 'how does a typical day of a data scientist look like', 'is preparation of algorithms and data structures needed for a data science interview ', 'what is the mathematical background required to be a data scientist ', 'what are the various rounds in a data scientist interview ', 'what level of proficiency is needed for a data scientist in coding']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r1OhthJeCb5z"
      },
      "source": [
        "#Embedding original question dataset\n",
        "\n",
        "sent_bertphrase_embeddings=[];\n",
        "\n",
        "for sent in cleaned_sentences:\n",
        "    sent_bertphrase_embeddings.append(bc.encode([sent]));"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gM_j-CbjCb52"
      },
      "source": [
        "#Embeding test question dataset\n",
        "\n",
        "sent_bertphrase_embeddings_test=[];\n",
        "for sent_test in cleaned_sentences_test:\n",
        "    sent_bertphrase_embeddings_test.append(bc.encode([sent_test]));\n"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XJZ5NkkJCb55",
        "outputId": "9707a6a6-101e-490b-d399-c369b42fd24e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import sklearn\n",
        "from sklearn.metrics.pairwise import cosine_similarity;\n",
        "def retrieveAndPrintFAQAnswer(question_embeddings_test1,sentence_embeddings1,FAQdf1,sentences1,FAQdf_test1,sentences_test1):\n",
        "  for index_test,faq_embedding_test in enumerate(question_embeddings_test1):\n",
        "    max_sim=-1;\n",
        "    index_sim=-1;\n",
        "    for index,faq_embedding in enumerate(sentence_embeddings1):\n",
        "        \n",
        "      sim=cosine_similarity(faq_embedding,faq_embedding_test)[0][0];\n",
        "      #print(index, sim, sentences[index])\n",
        "      if sim>max_sim:\n",
        "        max_sim=sim;\n",
        "        index_sim=index;\n",
        "\n",
        "    FAQdf_test1.iloc[index_test,2]=FAQdf1.iloc[index_sim,2]\n",
        "    FAQdf_test1.iloc[index_test,5]=FAQdf1.iloc[index_sim,0]\n",
        "    print(\"\\n\")\n",
        "    print(\"Question: \",sentences_test1[index_test])\n",
        "    print(\"\\n\");\n",
        "    print(\"Retrieved: \",FAQdf1.iloc[index_sim,0]) \n",
        "    print(\"Answer:\",FAQdf1.iloc[index_sim,1])        \n",
        "  df_test=FAQdf_test1 \n",
        "retrieveAndPrintFAQAnswer(sent_bertphrase_embeddings_test,sent_bertphrase_embeddings,df,cleaned_sentences,df_test,cleaned_sentences_test);"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Question:  how does job searching experience would look\n",
            "\n",
            "\n",
            "Retrieved:  What does the job hunting experience look like ?\n",
            "Answer: Job hunting experience involves networking to get in touch with the right people in various companies, applying to lots of jobs through various channels, preparing for interviews – while interviews are uncertain it is necessary to prepare well what you CAN prepare, smart scheduling of interviews to get the best job and salary you can.\n",
            "\n",
            "\n",
            "Question:  please provide how ds jobs are in the market\n",
            "\n",
            "\n",
            "Retrieved:  Any insights you can offer about the DS job market ?\n",
            "Answer: There are many kinds of roles, data scientist, analyst, data engineer and so on. There are many levels of companies. In smaller companies the roles are usually combined into one while in larger companies there are more nuanced roles. Since more and more companies are just getting started on their data journey, the overall demand is expected to increase in the next few years.\n",
            "\n",
            "\n",
            "Question:  how has the covid impacted ds hiring\n",
            "\n",
            "\n",
            "Retrieved:  What's the impact of Covid on hiring for DS roles?\n",
            "Answer: Hiring is going to slow down. First in small companies then eventually in enterprises in a couple of months. But it is likely to pick up in a few months. Hence, if you are looking to transition, this is a good time to ramp up skills and learn new things that might usually take some time.\n",
            "\n",
            "\n",
            "Question:  what skills do employers want from data science positions\n",
            "\n",
            "\n",
            "Retrieved:  What skills and qualities do employers look for in a data scientist?\n",
            "Answer: The following are some skills employers usually look for: \n",
            "\n",
            "(1) Should be good at coding \n",
            "(2) Should have good problem solving and analysis skills \n",
            "(3) Should be good with stats and building testing and deploying models. \n",
            "\n",
            "A data scientist is a software developer ++ with stats and modeling skills to build and deploy models and make inferences from data.\n",
            "\n",
            "\n",
            "Question:  is advanced degree in ml required\n",
            "\n",
            "\n",
            "Retrieved:  Do employers look for an advanced ML degree?\n",
            "Answer: For more senior roles: People typically look for practical experience for several years or a specialized degree with some experience. Employers typically want folks who have seen various kinds of ML problems and solved them, who can come up with the right solution for a new problem when they are looking for a more senior role.   \n",
            "\n",
            "For a beginner role: a degree usually adds some level of credibility, but if one has a good portfolio with good projects, it is usually an accepted substitute. Employers do hope to see if the candidate is comfortable with basic concepts in ML along with being able to write code.\n",
            "\n",
            "\n",
            "Question:  what data scientist do in a typical day\n",
            "\n",
            "\n",
            "Retrieved:  How does a typical day of a data scientist look like?\n",
            "Answer: Here are some tasks in the typical day of a data scientist:\n",
            "\n",
            "Make a plan for the day\n",
            "Look at data, what clean up is required, figure out what models can be built\n",
            "Talk to various stakeholders about what modeling is possible and help them narrow down to something useful for the business\n",
            "Build models, test and debug (takes a long time)\n",
            "Parameter tuning – test tons and tons of parameters (takes a long time)\n",
            "Come up with prod architecture to get deployment ready\n",
            "Write ML pipeline for production ready modeles – deploy them\n",
            "Wait for long time till we have a significant sample to see if they are working\n",
            "Analyze and see whether the models are working as expected, have any impact\n",
            "Come up with improvements/ corrections based on prod feedback and prepare for next iteration.\n",
            "Meeting with team members / daily sprints / bug triages based on production feedback – Interaction with ML Manager, Product Manager, Developers, Data engineers\n",
            "\n",
            "\n",
            "Question:  for an interview in data science are we required to study algorithms\n",
            "\n",
            "\n",
            "Retrieved:  Is preparation of algorithms and data structures needed for a data science interview ?\n",
            "Answer: Yes. In many data science interviews (ML Scientist, data scientist, ML Engineer, Data engineer) there is a coding round and an algorithms round. So preparing algorithms and data structures is necessary. Some of the Analyst interviews, such as a data analyst and a business analyst might not have an algorithms and data structures round.\n",
            "\n",
            "\n",
            "Question:  what is the mathematical background required to be a data scientist \n",
            "\n",
            "\n",
            "Retrieved:  What is the mathematical background required to be a data scientist ?\n",
            "Answer: The following three are the basic building blocks in terms of data science math background: Linear Algebra, Probability and Statistics and Calculus and optimization.\n",
            "\n",
            "\n",
            "Question:  how many rounds of interview are required to clear data scientist interview \n",
            "\n",
            "\n",
            "Retrieved:  Is preparation of algorithms and data structures needed for a data science interview ?\n",
            "Answer: Yes. In many data science interviews (ML Scientist, data scientist, ML Engineer, Data engineer) there is a coding round and an algorithms round. So preparing algorithms and data structures is necessary. Some of the Analyst interviews, such as a data analyst and a business analyst might not have an algorithms and data structures round.\n",
            "\n",
            "\n",
            "Question:  how much proficient a data scientist should be in coding\n",
            "\n",
            "\n",
            "Retrieved:  What level of proficiency is needed for a data scientist in coding?\n",
            "Answer: Needs to be reasonably proficient. Again, a data scientist is a developer ++. Usually expected to have the basic skills a developer has including understanding algorithms and data structures and being able to write clean, understable, efficient, well documented code.\n",
            "\n",
            "In many cases, the data scientist is expected to write production ready code and be able to understand the deployment process. In some cases, the coding could be for decision support.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rJ4-dIjrCb58",
        "outputId": "df03b779-6b76-46fe-c0e6-55b0756e6f79",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 511
        }
      },
      "source": [
        "df_test"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>ActualCode</th>\n",
              "      <th>PredictedCode</th>\n",
              "      <th>Prediction?</th>\n",
              "      <th>ActualQuestion</th>\n",
              "      <th>PredictedQuestion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How does job searching experience would look?</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Please provide how DS jobs are in the market?</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How has the Covid impacted DS hiring?</td>\n",
              "      <td>3</td>\n",
              "      <td>3.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>What skills do employers want from data scienc...</td>\n",
              "      <td>4</td>\n",
              "      <td>4.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Is advanced degree in ML required?</td>\n",
              "      <td>5</td>\n",
              "      <td>5.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>What data scientist do in a typical day?</td>\n",
              "      <td>6</td>\n",
              "      <td>6.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>For an interview in data science are we requir...</td>\n",
              "      <td>7</td>\n",
              "      <td>7.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>8</td>\n",
              "      <td>8.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>How many rounds of interview are required to c...</td>\n",
              "      <td>9</td>\n",
              "      <td>7.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>How much proficient a data scientist should be...</td>\n",
              "      <td>10</td>\n",
              "      <td>10.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ...                                  PredictedQuestion\n",
              "0      How does job searching experience would look?  ...   What does the job hunting experience look like ?\n",
              "1      Please provide how DS jobs are in the market?  ...  Any insights you can offer about the DS job ma...\n",
              "2              How has the Covid impacted DS hiring?  ...  What's the impact of Covid on hiring for DS ro...\n",
              "3  What skills do employers want from data scienc...  ...  What skills and qualities do employers look fo...\n",
              "4                 Is advanced degree in ML required?  ...       Do employers look for an advanced ML degree?\n",
              "5           What data scientist do in a typical day?  ...  How does a typical day of a data scientist loo...\n",
              "6  For an interview in data science are we requir...  ...  Is preparation of algorithms and data structur...\n",
              "7  What is the mathematical background required t...  ...  What is the mathematical background required t...\n",
              "8  How many rounds of interview are required to c...  ...  Is preparation of algorithms and data structur...\n",
              "9  How much proficient a data scientist should be...  ...  What level of proficiency is needed for a data...\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Wq2JCiOCb5_",
        "outputId": "649a6641-c117-44c0-c071-ceabf486872a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 511
        }
      },
      "source": [
        "for i in range(df_test.shape[0]):\n",
        " if df_test.iloc[i,1]!=df_test.iloc[i,2]:\n",
        "   df_test.iloc[i,3]='InCorrect'\n",
        " else:\n",
        "   df_test.iloc[i,3]='correct'\n",
        "df_test"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>ActualCode</th>\n",
              "      <th>PredictedCode</th>\n",
              "      <th>Prediction?</th>\n",
              "      <th>ActualQuestion</th>\n",
              "      <th>PredictedQuestion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How does job searching experience would look?</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Please provide how DS jobs are in the market?</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How has the Covid impacted DS hiring?</td>\n",
              "      <td>3</td>\n",
              "      <td>3.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>What skills do employers want from data scienc...</td>\n",
              "      <td>4</td>\n",
              "      <td>4.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Is advanced degree in ML required?</td>\n",
              "      <td>5</td>\n",
              "      <td>5.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>What data scientist do in a typical day?</td>\n",
              "      <td>6</td>\n",
              "      <td>6.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>For an interview in data science are we requir...</td>\n",
              "      <td>7</td>\n",
              "      <td>7.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>8</td>\n",
              "      <td>8.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>How many rounds of interview are required to c...</td>\n",
              "      <td>9</td>\n",
              "      <td>7.0</td>\n",
              "      <td>InCorrect</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>How much proficient a data scientist should be...</td>\n",
              "      <td>10</td>\n",
              "      <td>10.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ...                                  PredictedQuestion\n",
              "0      How does job searching experience would look?  ...   What does the job hunting experience look like ?\n",
              "1      Please provide how DS jobs are in the market?  ...  Any insights you can offer about the DS job ma...\n",
              "2              How has the Covid impacted DS hiring?  ...  What's the impact of Covid on hiring for DS ro...\n",
              "3  What skills do employers want from data scienc...  ...  What skills and qualities do employers look fo...\n",
              "4                 Is advanced degree in ML required?  ...       Do employers look for an advanced ML degree?\n",
              "5           What data scientist do in a typical day?  ...  How does a typical day of a data scientist loo...\n",
              "6  For an interview in data science are we requir...  ...  Is preparation of algorithms and data structur...\n",
              "7  What is the mathematical background required t...  ...  What is the mathematical background required t...\n",
              "8  How many rounds of interview are required to c...  ...  Is preparation of algorithms and data structur...\n",
              "9  How much proficient a data scientist should be...  ...  What level of proficiency is needed for a data...\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aaNd0JCACb6B"
      },
      "source": [
        "list_data_answers=df_test.ActualCode.values.tolist()\n",
        "list_data_FoundCode=df_test.PredictedCode.values.tolist()"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jrRSWtyCCb6E",
        "outputId": "2310adf5-0a04-423b-8ab0-4734bfdf5cae",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#printing confusion matrix\n",
        "from sklearn.metrics import multilabel_confusion_matrix\n",
        "from sklearn import metrics\n",
        "multilabel_confusion_matrix(list_data_answers, list_data_FoundCode)\n",
        "#tn, fp, fn, tp = confusion_matrix([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1.0, 2.0, 2.0, 10.0, 4.0, 1.0, 2.0, 1.0, 2.0, 2.0]).ravel()\n",
        "#print(metrics.classification_report(list_data_answers, list_data_FoundCode))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[8, 1],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [1, 0]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mCIK2sTsCb6H"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from mlxtend.evaluate import confusion_matrix\n",
        "\n",
        "\n",
        "cm = confusion_matrix(list_data_answers, list_data_FoundCode, binary=False)"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l7X3QrM-Cb6K",
        "outputId": "ba770137-1ca7-4116-d39e-20bac874a729",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from mlxtend.evaluate import confusion_matrix\n",
        "from mlxtend.plotting import plot_confusion_matrix\n",
        "\n",
        "import pprint\n",
        "#printing confusion matrix\n",
        "fig, ax = plot_confusion_matrix(conf_mat=cm)\n",
        "plt.show()"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQMAAAEGCAYAAABhHPB4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUFklEQVR4nO3de5ScdX3H8feHxFQCEaiJ7SGBJtwJHCGyWBTrUaBUBMF6sILaFlHTIsrNG1oVrXrQQq1WrYLg5VQEJGAVysVqFTWFkCWRa4hGrgmxJKiYg9aQ8O0f81sZkt3Zyc78duab/bzOyWFn9tnP893nhG+eZ+b5fUcRgZnZNr0uwMz6g5uBmQFuBmZWuBmYGeBmYGbF5F4X0EyTtw1Nmdb13Hn77tr1TLOMHnjgftauXavhvtdfzWDKNP5g77/qeu7CRZ/peqZZRof+6cCI3/NlgpkBbgZmVrgZmBngZmBmhZuBmQHJmsHnz3kdD3z3XAaveG/Xs799w/U8d7+92W+fPTjvnz7Wt5nOrZc50XOrNgNJL5O0XNIKSWd3mvfvV9/Mcad+thulPc3GjRs547RT+ebV17H09ru54rJLWXb33X2X6dx8tWbKrdYMJE0CPgscBcwFTpQ0t5PMhUt+xi8e+003ynuaxbfcwu6778Gc3XZjypQpvPo1J3DN1d/su0zn5qs1U27NM4PnAysi4t6IWA9cBhxXcX9j9vDDq5g1a5ffP545cxarVq3qu0zn1st0bt1mMBN4qOnxyvLc00iaL2lQ0mBs+G3FcsyslZ6/gBgRF0bEQEQMaPK2Palh551nsnLlU31r1aqVzJy5Wd/qeaZz62U6t24zWAXs0vR4Vnmu7wwcfDArVvyU+++7j/Xr13PF5Zdx9DHH9l2mc/PVmim35kKlxcCekubQaAInAK/tJPAr557Enx20J9N33J4V13+YD3/+Wr7yHzd1XOjkyZP5l099hlcc/Rds3LiRvz3pZObut1/fZTo3X62ZclVzIKqklwOfBCYBX4yIj7bafpupz4kaqxZ/udirFs2gsWrx1lsHx38Jc0RcC1xbcx9m1h09fwHRzPqDm4GZAW4GZla4GZgZ4GZgZkVfDUSdt++uVYaX7nTwW7ueCX7L0rYuPjMwM8DNwMwKNwMzA9wMzKxwMzAzwM3AzIp0zaDGlNlaU5czTMTNmpup1iy5qZpBrSmzNaYuZ5mImzE3U62ZclM1g1pTZmtMXc4yETdjbqZaM+Wmaga1pszWkGUibsbcTLVmyk3VDMysnlTNoNaU2RqyTMTNmJup1ky5qZpBrSmzNWSZiJsxN1OtmXL7atXiaGpNma0xdTnLRNyMuZlqzZRbdTryljrooIFYuGiw67lewmzW0Go6cqrLBDOrx83AzAA3AzMr3AzMDHAzMLPCzcDMgGT3GYxVrbcAa7xl6bcrrVd8ZmBmgJuBmRVuBmYGuBmYWeFmYGaAm4GZFemaQYYps0M8dblebqZas+RWawaSdpH0PUl3S7pL0umdZmaZMjvEU5c9HTlTbs0zgw3A2yNiLnAIcKqkuZ0EZpkyO8RTlz0dOVNutWYQEasjYkn5eh2wDOho8FuWKbM1ZTsGno6cJ3dcXjOQNBuYBywa5nvzJQ1KGlyzds14lGNmw6jeDCRtD1wJnBERv970+xFxYUQMRMTAjOkzWmZlmTJbU7Zj4OnIeXKrNgNJz6DRCC6JiKs6zcsyZbambMfA05Hz5FZbtShJwMXAsoj4RDcys0yZHeKpy56OnCm32nRkSS8CfgjcATxZnn5vRFw70s/Umo5ci5cwWzatpiNXOzOIiB8Bw+7UzPpPujsQzawONwMzA9wMzKxwMzAzYIIMRK2lxiv//lxI6xWfGZgZ4GZgZoWbgZkBbgZmVrgZmBngZmBmRbpmkGGwZM1M8KDVWpkTPTdVM8gyWLJmreBBq5lqzZSbqhlkGSxZs1bwoNVMtWbKTdUMsgyWrJVZk4+tc1M1AzOrJ1UzyDJYslZmTT62zk3VDLIMlqxZay0+ts5NtWoxy2DJmrWCB61mqjVTbrWBqGORbSBqDV7CbDW1Goia6jLBzOpxMzAzwM3AzAo3AzMD3AzMrHAzMDMg2X0GE0GttwD9lqWNZsRmIGkdMHQTwtD7klG+joh4VuXazGwcjdgMImLaeBZiZr3V1msGkl4k6Q3l6+mS5tQty8zG26jNQNI5wLuB95SnpgBfrVmUmY2/ds4M/hI4FngcICIeBnwJYbaVaacZrI/GaqYAkLRd3ZLMrBfaaQZfl3QBsKOkNwPfAb5Qt6yRZZgyWzOzVm6ticvgY5sld9RmEBHnAwuAK4G9gA9ExKfb3YGkSZKWSrpm7GU2ZJkym61WqDNxGXxsM+W2ewfiHcAPgR+Ur7fE6cCyLfyZYWWZMputVqgzcRl8bDPltvNuwpuAW4BXAccDN0s6uZ1wSbOAo4GLxlxhkyxTZmtl1sytxcc2T247tyO/E5gXEY8CSHo28D/AF9v42U8C76LFuw+S5gPzAXbZddc2Is2shnYuEx4F1jU9Xleea0nSMcAjEXFrq+0i4sKIGIiIgRnTZ7TMzDJltlZmzdxafGzz5I7YDCSdJeksYAWwSNIHyw1INwM/aSP7UOBYSfcDlwGHSeroZqUsU2az1VqTj22e3FaXCUOn9j8rf4a09QpFRLyHcteipJcA74iI14+hxt/LMmU2W61QZ+JyrXqzHdssueMyHbmpGRzTajtPR67HS5gNWk9HHvUFREkzaLwIuB/wzKHnI+KwdguIiO8D3293ezMbf+28gHgJcA8wB/gQcD+wuGJNZtYD7TSDZ0fExcATEXFjRJwMtH1WYGY5tHOfwRPlv6slHQ08DPxhvZLMrBfaaQYfkbQD8Hbg08CzgDOrVmVm427UZhARQwuMHgNeWrccM+uVVgNRP81TA1E3ExGnVanIqvDUZRtNqzMDv+FvNoG0mo78lfEsxMx6y5+oZGaAm4GZFW4GZga0N+loL0nflXRnefxcSe+rX5qZjad2zgy+QGMp8hMAEXE7cELNolrJMGW2Zma23FpTlzMdgyy57TSDqRFxyybPbehor2OUZcpstlpr5taYupztGGTJbacZrJW0O099iMrxwOox77EDWabMZqu1Zm6NqcvZjkGW3HaawanABcA+klYBZwCnjHmPHcgyZbZWZsbcGrIdgyy57axNuBc4onys2jYRsW60nzGzfNqZdPSBTR4DEBH/WKmmEWWZMlsrM2NuDdmOQZbcdi4THm/6sxE4Cpg95j12IMuU2Wy11sytIdsxyJLbzmXCPzc/lnQ+cMOY99iBLFNms9VaM7fG1OVsxyBL7hZPR5a0E7A4IvYY815H4OnI+XgJcy6dTke+g6fmGkwCZgDj/nqBmdXVztiz5s862AD8b0T05KYjM6unZTOQNAm4ISL2Gad6zKxHWr6bEBEbgeWS/PHIZlu5di4TdgLuknQLjbcXAYiI/nzfyczGpJ1m8P7qVZhZz7XTDF4eEe9ufkLSx4Eb65RkmXjq8tajnTsQ/3yY547qdiFm1lutPjfhFOAtwG6Sbm/61jRgYe3CzGx8tbpM+BpwHXAucHbT8+si4hdVqzKzcdfqcxMeo/GRaieOXzlm1iuejmxmgJuBmRXpmkGGKbM1M53b4KnL3c+t2gwk7ShpgaR7JC2T9IJO8rJMmc1Wa8ZcT13uzXTkTnwKuL4sdDoAWNZJWJYps9lqzZjrqcu9mY48JpJ2AF4MXAwQEesj4ledZGaZMlsr07l1ZTsG3c6teWYwB1gDfEnSUkkXlQnLTyNpvqRBSYNr1q6pWI6ZtVKzGUwGngd8LiLm0VjxePamG0XEhRExEBEDM6bPaBmYZcpsrUzn1pXtGPRiOvJYrQRWRsSi8ngBjeYwZlmmzGarNWNuDdmOwbhPRx6riPi5pIck7R0Ry4HDgY5eQs0yZTZbrRlzPXW5D6Yjb1G4dCBwETAFuBd4Q0T8cqTtPR3ZhngJcx0dTUfuRET8GBiouQ8z6450dyCaWR1uBmYGuBmYWeFmYGaAm4GZFVXfTTAbq0xTl7eWtyt9ZmBmgJuBmRVuBmYGuBmYWeFmYGZAwmaQYbBkzUzn1suEiT1oNVUzyDJYMlut2XJr1QoTe9BqqmaQZbBktlqz5daqFSb2oNVUzSDLYMlamc6tl1lTlmObqhmYWT2pmkGWwZK1Mp1bL7OmLMc2VTPIMlgyW63ZcjMNWYU8xzbVQqUsgyWz1Zott1atMLEHrVYdiLqlPBDVapvoqxZbDURNdZlgZvW4GZgZ4GZgZoWbgZkBbgZmVrgZmBmQ7D4Dmzj8WYvjz2cGZga4GZhZ4WZgZoCbgZkVbgZmBrgZmFmRrhlkmDJbM9O5DRN5inGt3KrNQNKZku6SdKekSyU9s5O8LFNms9WaMXciTzGulVutGUiaCZwGDETE/sAk4IROMrNMmc1Wa8bciTzFuFZu7cuEycC2kiYDU4GHOwnLMmW2VqZz68p2DNJMR46IVcD5wIPAauCxiPj2pttJmi9pUNLgmrVrapVjZqOoeZmwE3AcMAfYGdhO0us33S4iLoyIgYgYmDF9RsvMLFNma2U6t65sxyDTdOQjgPsiYk1EPAFcBbywk8AsU2az1Zoxt4ZsxyDTdOQHgUMkTQV+CxwOdDTtNMuU2Wy1ZsydyFOMa+VWnY4s6UPAa4ANwFLgTRHxu5G293RkG+IlzHW0mo5cdZ5BRJwDnFNzH2bWHenuQDSzOtwMzAxwMzCzws3AzAA3AzMrPB3Z+tJEfwsQ6ry9+rvlD474PZ8ZmBngZmBmhZuBmQFuBmZWuBmYGeBmYGZFumaQYcpszUzn1svMltvtCdGpmkGWKbPZas2Wm6nWmrndnhCdqhlkmTKbrdZsuZlqrZnb7QnRqZpBlimztTKdWy8zY263pWoGZlZPqmaQZcpsrUzn1svMmNttqZpBlimz2WrNlpup1pq53ZZq1WKWKbPZas2Wm6nWmrndnhBddTrylvJ0ZLOn1FnC/HWe/M0jw05HTnWZYGb1uBmYGeBmYGaFm4GZAW4GZla4GZgZ0GdvLUpaAzzQxqbTgbWVy+mmTPVmqhVy1dsPtf5JRMwY7ht91QzaJWkwIgZ6XUe7MtWbqVbIVW+/1+rLBDMD3AzMrMjaDC7sdQFbKFO9mWqFXPX2da0pXzMws+7LemZgZl3mZmBmQMJmIOllkpZLWiHp7F7XMxJJu0j6nqS7Jd0l6fRe19QOSZMkLZV0Ta9raUXSjpIWSLpH0jJJL+h1Ta1IOrP8PbhT0qWSntnrmjaVqhlImgR8FjgKmAucKGlub6sa0Qbg7RExFzgEOLWPa212OrCs10W04VPA9RGxD3AAfVyzpJnAacBAROwPTAJO6G1Vm0vVDIDnAysi4t6IWA9cBhzX45qGFRGrI2JJ+Xodjb+s/Tf4romkWcDRwEW9rqUVSTsALwYuBoiI9RHxq95WNarJwLaSJgNTgYd7XM9msjWDmcBDTY9X0uf/gwFImg3MAxb1tpJRfRJ4F/BkrwsZxRxgDfClcklzkaTtel3USCJiFXA+8CCwGngsIr7d26o2l60ZpCNpe+BK4IyI+HWv6xmJpGOARyLi1l7X0obJwPOAz0XEPOBxoJ9fP9qJxhnsHGBnYDtJr+9tVZvL1gxWAbs0PZ5VnutLkp5BoxFcEhFX9bqeURwKHCvpfhqXX4dJ+mpvSxrRSmBlRAydaS2g0Rz61RHAfRGxJiKeAK4CXtjjmjaTrRksBvaUNEfSFBovwnyrxzUNS5JoXNMui4hP9Lqe0UTEeyJiVkTMpnFc/zsi+u5fL4CI+DnwkKS9y1OHA51/eGE9DwKHSJpa/l4cTh++4JlqVHpEbJD0VuAGGq/IfjEi7upxWSM5FPhr4A5JPy7PvTciru1hTVuTtwGXlH8U7gXe0ON6RhQRiyQtAJbQeJdpKX14a7JvRzYzIN9lgplV4mZgZoCbgZkVbgZmBrgZmFnhZjBBSXrJ0MpESce2WgFaVgi+ZQz7+KCkd7T7/CbbfFnS8Vuwr9mS7tzSGu0pbgZbmbKyc4tExLci4mMtNtkR2OJmYLm4GSRR/uW7R9IlZf3+AklTy/ful/RxSUuAV0s6UtJNkpZIuqKsjxiaBXFP2e5VTdknSfpM+fqPJH1D0m3lzwuBjwG7S/qxpPPKdu+UtFjS7ZI+1JT1D5J+IulHwN6MQtKbS85tkq4c+p2KIyQNlrxjyvaTJJ3XtO+/6/TYWoObQS57A/8WEfsCv+bp/1o/GhHPA74DvA84ojweBM4qwzS+ALwCOAj44xH28a/AjRFxAI37/e+isQjoZxFxYES8U9KRwJ40lpQfCBwk6cWSDqJxK/OBwMuBg9v4na6KiIPL/pYBb2z63uyyj6OBz5ff4Y00Vv0dXPLfLGlOG/uxUaS6Hdl4KCIWlq+/SmNgxvnl8eXlv4fQGPyysHEbPFOAm4B9aCyW+SlAWYQ0f5h9HAb8DUBEbAQeK6vumh1Z/iwtj7en0RymAd+IiN+UfbSzbmR/SR+hcSmyPY1bzYd8PSKeBH4q6d7yOxwJPLfp9YQdyr5/0sa+rAU3g1w2vXe8+fHj5b8C/isiTmzeUNKBXaxDwLkRccEm+zhjDFlfBl4ZEbdJOgl4SdP3hvt9BbwtIpqbxtDMCOuALxNy2bVp1t9rgR8Ns83NwKGS9gCQtJ2kvYB7gNmSdi/bnTjMzwJ8Fzil/OykMlVoHY1/9YfcAJzc9FrETEnPAX4AvFLStpKm0bgkGc00YHVZ7v26Tb73aknblJp3A5aXfZ9StkfSXv082CQTN4NcltOYpbgM2An43KYbRMQa4CTgUkm3Uy4RIuL/aFwW/Gd5AfGREfZxOvBSSXcAtwJzI+JRGpcdd0o6r0zp+RpwU9luATCtjHm7HLgNuI7GkvPRvJ/GBKiFNBpWsweBW0rW35ff4SIay5WXlLcSL8BnuF3hVYtJlNPga8pATbOu85mBmQE+MzCzwmcGZga4GZhZ4WZgZoCbgZkVbgZmBsD/AyLloCKlwSipAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qZC_6oThCb6P",
        "outputId": "9c5073a0-371c-4480-8ee3-9a4db191b3eb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(metrics.classification_report(list_data_answers, list_data_FoundCode))"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       1.00      1.00      1.00         1\n",
            "           2       1.00      1.00      1.00         1\n",
            "           3       1.00      1.00      1.00         1\n",
            "           4       1.00      1.00      1.00         1\n",
            "           5       1.00      1.00      1.00         1\n",
            "           6       1.00      1.00      1.00         1\n",
            "           7       0.50      1.00      0.67         1\n",
            "           8       1.00      1.00      1.00         1\n",
            "           9       0.00      0.00      0.00         1\n",
            "          10       1.00      1.00      1.00         1\n",
            "\n",
            "    accuracy                           0.90        10\n",
            "   macro avg       0.85      0.90      0.87        10\n",
            "weighted avg       0.85      0.90      0.87        10\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sW_uaCk7EBJI"
      },
      "source": [
        ""
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B0l2AmXFEI8u"
      },
      "source": [
        "#Dataset3-Hard"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "90EtirsFEI8w"
      },
      "source": [
        "##Import Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DZteovWMEI83",
        "outputId": "145312bb-d301-4019-9ded-980ef5904d5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        }
      },
      "source": [
        "import pandas as pd;\n",
        "\n",
        "#Loading dataset and renaming columns\n",
        "\n",
        "df=pd.read_csv(\"FAQ_standard.csv\");\n",
        "df.columns=[\"questions\",\"answers\",\"Label\"];\n",
        "\n",
        "df"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>answers</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>Job hunting experience involves networking to ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>There are many kinds of roles, data scientist,...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>Hiring is going to slow down. First in small c...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>The following are some skills employers usuall...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>For more senior roles: People typically look f...</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>Here are some tasks in the typical day of a da...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>Yes. In many data science interviews (ML Scien...</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>The following three are the basic building blo...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>Usually the data science interview has a subse...</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>Needs to be reasonably proficient. Again, a da...</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ... Label\n",
              "0   What does the job hunting experience look like ?  ...     1\n",
              "1  Any insights you can offer about the DS job ma...  ...     2\n",
              "2  What's the impact of Covid on hiring for DS ro...  ...     3\n",
              "3  What skills and qualities do employers look fo...  ...     4\n",
              "4       Do employers look for an advanced ML degree?  ...     5\n",
              "5  How does a typical day of a data scientist loo...  ...     6\n",
              "6  Is preparation of algorithms and data structur...  ...     7\n",
              "7  What is the mathematical background required t...  ...     8\n",
              "8  What are the various rounds in a data scientis...  ...     9\n",
              "9  What level of proficiency is needed for a data...  ...    10\n",
              "\n",
              "[10 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1trD-yCaEI9A",
        "outputId": "375c7c9c-8ce8-4dc5-c787-6b9bb2c8e1a4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 377
        }
      },
      "source": [
        "import pandas as pd;\n",
        "\n",
        "#Load dataset and examine dataset, rename columns to questions and answers\n",
        "\n",
        "df_test=pd.read_excel(\"FAQ_test_Hard.xlsx\")\n",
        "df_test.columns=[\"questions\",\"ActualCode\",\"PredictedCode\",\"Prediction?\",\"ActualQuestion\",\"PredictedQuestion\"]\n",
        "\n",
        "df_test"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>ActualCode</th>\n",
              "      <th>PredictedCode</th>\n",
              "      <th>Prediction?</th>\n",
              "      <th>ActualQuestion</th>\n",
              "      <th>PredictedQuestion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How difficult is it to find a job?</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Please provide how data science jobs are in th...</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How has the Covid impacted job?</td>\n",
              "      <td>3</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Tell me about the skills required for Data sci...</td>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Is advanced knowledge of ML required?</td>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>What are the tasks performed by a data scienti...</td>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>How much is data structure important for data ...</td>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>How much maths knowledge is required to get a ...</td>\n",
              "      <td>8</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>How many rounds of interview is done for data ...</td>\n",
              "      <td>9</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>How much coding experience is required for dat...</td>\n",
              "      <td>10</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ...  PredictedQuestion\n",
              "0                 How difficult is it to find a job?  ...                NaN\n",
              "1  Please provide how data science jobs are in th...  ...                NaN\n",
              "2                    How has the Covid impacted job?  ...                NaN\n",
              "3  Tell me about the skills required for Data sci...  ...                NaN\n",
              "4              Is advanced knowledge of ML required?  ...                NaN\n",
              "5  What are the tasks performed by a data scienti...  ...                NaN\n",
              "6  How much is data structure important for data ...  ...                NaN\n",
              "7  How much maths knowledge is required to get a ...  ...                NaN\n",
              "8  How many rounds of interview is done for data ...  ...                NaN\n",
              "9  How much coding experience is required for dat...  ...                NaN\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KuDHbcxAEI9G"
      },
      "source": [
        "##BERT Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E8VS3-e8EI9H",
        "outputId": "a12e8c02-42c8-40e3-d944-49ddf854c59d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#cleaning test data questions\n",
        "cleaned_sentences_test=get_cleaned_sentences(df_test,stopwords=False)\n",
        "print(cleaned_sentences_test);"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['how difficult is it to find a job', 'please provide how data science jobs are in the market', 'how has the covid impacted job', 'tell me about the skills required for data science in the industry', 'is advanced knowledge of ml required', 'what are the tasks performed by a data scientist in a day', 'how much is data structure important for data science positions', 'how much maths knowledge is required to get a data science job', 'how many rounds of interview is done for data science roles', 'how much coding experience is required for data science positions']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I8S37IGeEI9L"
      },
      "source": [
        "#cleaning original data questions\n",
        "cleaned_sentences=get_cleaned_sentences(df,stopwords=False)"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FznmiaYZEI9P",
        "outputId": "f333820f-f2c1-4d26-fd1c-7d40ae2de289",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(cleaned_sentences)"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['what does the job hunting experience look like ', 'any insights you can offer about the ds job market ', 'whats the impact of covid on hiring for ds roles', 'what skills and qualities do employers look for in a data scientist', 'do employers look for an advanced ml degree', 'how does a typical day of a data scientist look like', 'is preparation of algorithms and data structures needed for a data science interview ', 'what is the mathematical background required to be a data scientist ', 'what are the various rounds in a data scientist interview ', 'what level of proficiency is needed for a data scientist in coding']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lmaL7vuGEI9T"
      },
      "source": [
        "#Embedding original question dataset\n",
        "\n",
        "sent_bertphrase_embeddings=[];\n",
        "\n",
        "for sent in cleaned_sentences:\n",
        "    sent_bertphrase_embeddings.append(bc.encode([sent]));"
      ],
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C8sB0K6JEI9X"
      },
      "source": [
        "#Embeding test question dataset\n",
        "\n",
        "sent_bertphrase_embeddings_test=[];\n",
        "for sent_test in cleaned_sentences_test:\n",
        "    sent_bertphrase_embeddings_test.append(bc.encode([sent_test]));\n"
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dpBDrz5rEI9c",
        "outputId": "effcc202-08d8-4d34-a14d-d19f31b0f576",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import sklearn\n",
        "from sklearn.metrics.pairwise import cosine_similarity;\n",
        "def retrieveAndPrintFAQAnswer(question_embeddings_test1,sentence_embeddings1,FAQdf1,sentences1,FAQdf_test1,sentences_test1):\n",
        "  for index_test,faq_embedding_test in enumerate(question_embeddings_test1):\n",
        "    max_sim=-1;\n",
        "    index_sim=-1;\n",
        "    for index,faq_embedding in enumerate(sentence_embeddings1):\n",
        "        \n",
        "      sim=cosine_similarity(faq_embedding,faq_embedding_test)[0][0];\n",
        "      #print(index, sim, sentences[index])\n",
        "      if sim>max_sim:\n",
        "        max_sim=sim;\n",
        "        index_sim=index;\n",
        "\n",
        "    FAQdf_test1.iloc[index_test,2]=FAQdf1.iloc[index_sim,2]\n",
        "    FAQdf_test1.iloc[index_test,5]=FAQdf1.iloc[index_sim,0]\n",
        "    print(\"\\n\")\n",
        "    print(\"Question: \",sentences_test1[index_test])\n",
        "    print(\"\\n\");\n",
        "    print(\"Retrieved: \",FAQdf1.iloc[index_sim,0]) \n",
        "    print(\"Answer:\",FAQdf1.iloc[index_sim,1])        \n",
        "  df_test=FAQdf_test1 \n",
        "retrieveAndPrintFAQAnswer(sent_bertphrase_embeddings_test,sent_bertphrase_embeddings,df,cleaned_sentences,df_test,cleaned_sentences_test);"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Question:  how difficult is it to find a job\n",
            "\n",
            "\n",
            "Retrieved:  What is the mathematical background required to be a data scientist ?\n",
            "Answer: The following three are the basic building blocks in terms of data science math background: Linear Algebra, Probability and Statistics and Calculus and optimization.\n",
            "\n",
            "\n",
            "Question:  please provide how data science jobs are in the market\n",
            "\n",
            "\n",
            "Retrieved:  Any insights you can offer about the DS job market ?\n",
            "Answer: There are many kinds of roles, data scientist, analyst, data engineer and so on. There are many levels of companies. In smaller companies the roles are usually combined into one while in larger companies there are more nuanced roles. Since more and more companies are just getting started on their data journey, the overall demand is expected to increase in the next few years.\n",
            "\n",
            "\n",
            "Question:  how has the covid impacted job\n",
            "\n",
            "\n",
            "Retrieved:  What's the impact of Covid on hiring for DS roles?\n",
            "Answer: Hiring is going to slow down. First in small companies then eventually in enterprises in a couple of months. But it is likely to pick up in a few months. Hence, if you are looking to transition, this is a good time to ramp up skills and learn new things that might usually take some time.\n",
            "\n",
            "\n",
            "Question:  tell me about the skills required for data science in the industry\n",
            "\n",
            "\n",
            "Retrieved:  What level of proficiency is needed for a data scientist in coding?\n",
            "Answer: Needs to be reasonably proficient. Again, a data scientist is a developer ++. Usually expected to have the basic skills a developer has including understanding algorithms and data structures and being able to write clean, understable, efficient, well documented code.\n",
            "\n",
            "In many cases, the data scientist is expected to write production ready code and be able to understand the deployment process. In some cases, the coding could be for decision support.\n",
            "\n",
            "\n",
            "Question:  is advanced knowledge of ml required\n",
            "\n",
            "\n",
            "Retrieved:  Is preparation of algorithms and data structures needed for a data science interview ?\n",
            "Answer: Yes. In many data science interviews (ML Scientist, data scientist, ML Engineer, Data engineer) there is a coding round and an algorithms round. So preparing algorithms and data structures is necessary. Some of the Analyst interviews, such as a data analyst and a business analyst might not have an algorithms and data structures round.\n",
            "\n",
            "\n",
            "Question:  what are the tasks performed by a data scientist in a day\n",
            "\n",
            "\n",
            "Retrieved:  What are the various rounds in a data scientist interview ?\n",
            "Answer: Usually the data science interview has a subset of these rounds.\n",
            "\n",
            "Resume deep dive\n",
            "ML Concepts\n",
            "ML Scenario and Problem solving\n",
            "Algorithms and data structures\n",
            "Coding\n",
            "Behavioural\n",
            "\n",
            "Sometimes, some of these rounds might be combined, for instance there might not be two separate rounds for coding and algorithms. Similarly there might be a single round for ML concepts and scenario based problem solving.\n",
            "\n",
            "\n",
            "Question:  how much is data structure important for data science positions\n",
            "\n",
            "\n",
            "Retrieved:  What is the mathematical background required to be a data scientist ?\n",
            "Answer: The following three are the basic building blocks in terms of data science math background: Linear Algebra, Probability and Statistics and Calculus and optimization.\n",
            "\n",
            "\n",
            "Question:  how much maths knowledge is required to get a data science job\n",
            "\n",
            "\n",
            "Retrieved:  What is the mathematical background required to be a data scientist ?\n",
            "Answer: The following three are the basic building blocks in terms of data science math background: Linear Algebra, Probability and Statistics and Calculus and optimization.\n",
            "\n",
            "\n",
            "Question:  how many rounds of interview is done for data science roles\n",
            "\n",
            "\n",
            "Retrieved:  Is preparation of algorithms and data structures needed for a data science interview ?\n",
            "Answer: Yes. In many data science interviews (ML Scientist, data scientist, ML Engineer, Data engineer) there is a coding round and an algorithms round. So preparing algorithms and data structures is necessary. Some of the Analyst interviews, such as a data analyst and a business analyst might not have an algorithms and data structures round.\n",
            "\n",
            "\n",
            "Question:  how much coding experience is required for data science positions\n",
            "\n",
            "\n",
            "Retrieved:  What level of proficiency is needed for a data scientist in coding?\n",
            "Answer: Needs to be reasonably proficient. Again, a data scientist is a developer ++. Usually expected to have the basic skills a developer has including understanding algorithms and data structures and being able to write clean, understable, efficient, well documented code.\n",
            "\n",
            "In many cases, the data scientist is expected to write production ready code and be able to understand the deployment process. In some cases, the coding could be for decision support.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2FEa3-uzEI9h",
        "outputId": "cca8ffc4-e189-47b7-b3dc-e58107e14bc3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 511
        }
      },
      "source": [
        "df_test"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>ActualCode</th>\n",
              "      <th>PredictedCode</th>\n",
              "      <th>Prediction?</th>\n",
              "      <th>ActualQuestion</th>\n",
              "      <th>PredictedQuestion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How difficult is it to find a job?</td>\n",
              "      <td>1</td>\n",
              "      <td>8.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Please provide how data science jobs are in th...</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How has the Covid impacted job?</td>\n",
              "      <td>3</td>\n",
              "      <td>3.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Tell me about the skills required for Data sci...</td>\n",
              "      <td>4</td>\n",
              "      <td>10.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Is advanced knowledge of ML required?</td>\n",
              "      <td>5</td>\n",
              "      <td>7.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>What are the tasks performed by a data scienti...</td>\n",
              "      <td>6</td>\n",
              "      <td>9.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>How much is data structure important for data ...</td>\n",
              "      <td>7</td>\n",
              "      <td>8.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>How much maths knowledge is required to get a ...</td>\n",
              "      <td>8</td>\n",
              "      <td>8.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>How many rounds of interview is done for data ...</td>\n",
              "      <td>9</td>\n",
              "      <td>7.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>How much coding experience is required for dat...</td>\n",
              "      <td>10</td>\n",
              "      <td>10.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ...                                  PredictedQuestion\n",
              "0                 How difficult is it to find a job?  ...  What is the mathematical background required t...\n",
              "1  Please provide how data science jobs are in th...  ...  Any insights you can offer about the DS job ma...\n",
              "2                    How has the Covid impacted job?  ...  What's the impact of Covid on hiring for DS ro...\n",
              "3  Tell me about the skills required for Data sci...  ...  What level of proficiency is needed for a data...\n",
              "4              Is advanced knowledge of ML required?  ...  Is preparation of algorithms and data structur...\n",
              "5  What are the tasks performed by a data scienti...  ...  What are the various rounds in a data scientis...\n",
              "6  How much is data structure important for data ...  ...  What is the mathematical background required t...\n",
              "7  How much maths knowledge is required to get a ...  ...  What is the mathematical background required t...\n",
              "8  How many rounds of interview is done for data ...  ...  Is preparation of algorithms and data structur...\n",
              "9  How much coding experience is required for dat...  ...  What level of proficiency is needed for a data...\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_jKj-_yvEI9j",
        "outputId": "96a3140d-3bf6-458d-f195-35cfa05ca0b7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 511
        }
      },
      "source": [
        "for i in range(df_test.shape[0]):\n",
        " if df_test.iloc[i,1]!=df_test.iloc[i,2]:\n",
        "   df_test.iloc[i,3]='InCorrect'\n",
        " else:\n",
        "   df_test.iloc[i,3]='correct'\n",
        "df_test"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>questions</th>\n",
              "      <th>ActualCode</th>\n",
              "      <th>PredictedCode</th>\n",
              "      <th>Prediction?</th>\n",
              "      <th>ActualQuestion</th>\n",
              "      <th>PredictedQuestion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How difficult is it to find a job?</td>\n",
              "      <td>1</td>\n",
              "      <td>8.0</td>\n",
              "      <td>InCorrect</td>\n",
              "      <td>What does the job hunting experience look like ?</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Please provide how data science jobs are in th...</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "      <td>Any insights you can offer about the DS job ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How has the Covid impacted job?</td>\n",
              "      <td>3</td>\n",
              "      <td>3.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "      <td>What's the impact of Covid on hiring for DS ro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Tell me about the skills required for Data sci...</td>\n",
              "      <td>4</td>\n",
              "      <td>10.0</td>\n",
              "      <td>InCorrect</td>\n",
              "      <td>What skills and qualities do employers look fo...</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Is advanced knowledge of ML required?</td>\n",
              "      <td>5</td>\n",
              "      <td>7.0</td>\n",
              "      <td>InCorrect</td>\n",
              "      <td>Do employers look for an advanced ML degree?</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>What are the tasks performed by a data scienti...</td>\n",
              "      <td>6</td>\n",
              "      <td>9.0</td>\n",
              "      <td>InCorrect</td>\n",
              "      <td>How does a typical day of a data scientist loo...</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>How much is data structure important for data ...</td>\n",
              "      <td>7</td>\n",
              "      <td>8.0</td>\n",
              "      <td>InCorrect</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>How much maths knowledge is required to get a ...</td>\n",
              "      <td>8</td>\n",
              "      <td>8.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "      <td>What is the mathematical background required t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>How many rounds of interview is done for data ...</td>\n",
              "      <td>9</td>\n",
              "      <td>7.0</td>\n",
              "      <td>InCorrect</td>\n",
              "      <td>What are the various rounds in a data scientis...</td>\n",
              "      <td>Is preparation of algorithms and data structur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>How much coding experience is required for dat...</td>\n",
              "      <td>10</td>\n",
              "      <td>10.0</td>\n",
              "      <td>correct</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "      <td>What level of proficiency is needed for a data...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           questions  ...                                  PredictedQuestion\n",
              "0                 How difficult is it to find a job?  ...  What is the mathematical background required t...\n",
              "1  Please provide how data science jobs are in th...  ...  Any insights you can offer about the DS job ma...\n",
              "2                    How has the Covid impacted job?  ...  What's the impact of Covid on hiring for DS ro...\n",
              "3  Tell me about the skills required for Data sci...  ...  What level of proficiency is needed for a data...\n",
              "4              Is advanced knowledge of ML required?  ...  Is preparation of algorithms and data structur...\n",
              "5  What are the tasks performed by a data scienti...  ...  What are the various rounds in a data scientis...\n",
              "6  How much is data structure important for data ...  ...  What is the mathematical background required t...\n",
              "7  How much maths knowledge is required to get a ...  ...  What is the mathematical background required t...\n",
              "8  How many rounds of interview is done for data ...  ...  Is preparation of algorithms and data structur...\n",
              "9  How much coding experience is required for dat...  ...  What level of proficiency is needed for a data...\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qbiMxZ6IEI9m"
      },
      "source": [
        "list_data_answers=df_test.ActualCode.values.tolist()\n",
        "list_data_FoundCode=df_test.PredictedCode.values.tolist()"
      ],
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5xAxr5KVEI9q",
        "outputId": "f5f7dde4-a15c-4c52-ea68-9c816339cfa3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#printing confusion matrix\n",
        "from sklearn.metrics import multilabel_confusion_matrix\n",
        "from sklearn import metrics\n",
        "multilabel_confusion_matrix(list_data_answers, list_data_FoundCode)\n",
        "#tn, fp, fn, tp = confusion_matrix([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1.0, 2.0, 2.0, 10.0, 4.0, 1.0, 2.0, 1.0, 2.0, 2.0]).ravel()\n",
        "#print(metrics.classification_report(list_data_answers, list_data_FoundCode))"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[[9, 0],\n",
              "        [1, 0]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [1, 0]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [1, 0]],\n",
              "\n",
              "       [[9, 0],\n",
              "        [1, 0]],\n",
              "\n",
              "       [[7, 2],\n",
              "        [1, 0]],\n",
              "\n",
              "       [[7, 2],\n",
              "        [0, 1]],\n",
              "\n",
              "       [[8, 1],\n",
              "        [1, 0]],\n",
              "\n",
              "       [[8, 1],\n",
              "        [0, 1]]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MlKi8mP0EI9t"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from mlxtend.evaluate import confusion_matrix\n",
        "\n",
        "\n",
        "cm = confusion_matrix(list_data_answers, list_data_FoundCode, binary=False)"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZqMwlEqNEI9v",
        "outputId": "d235c9ae-66a9-40dc-f935-f70591f88007",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from mlxtend.evaluate import confusion_matrix\n",
        "from mlxtend.plotting import plot_confusion_matrix\n",
        "\n",
        "import pprint\n",
        "#printing confusion matrix\n",
        "fig, ax = plot_confusion_matrix(conf_mat=cm)\n",
        "plt.show()"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQMAAAEGCAYAAABhHPB4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUfklEQVR4nO3dfZRcdX3H8ffHxBzlQaBm2x4SIOHZ4FGQRdFYjwKlIgjWo0dQ2yJqWkQBn9GqaNUDFmq1ahUEH05FEAJWpQhW62MKIZtEHkM08hASYtmgYg60BsK3f8xvZUx2Z2d35rcz3+zndU5Odmbufu53L+G798699zuKCMzMntDrAsysP7gZmBngZmBmhZuBmQFuBmZWzOx1Ac1mz54de+01r9dl2HZs5aq1Xc885Gl7dj2zlnvuuZuNGzdqtNf6qhnstdc8liwd6nUZth3b7bA3dz1zydJPdz2zloXPGRzzNR8mmBngZmBmhZuBmQFuBmZWuBmYGZCwGXznumt5xkEHcNCB+3LeP57b17mZas2WW6vWz539Gu753jkMXfHermVCjm1btRlIerGk1ZLWSDqr07wtW7Zw5umn8Y1vfZuVN9/OFZddyqrbb++4zhq5mWrNllurVoB/+9YNnHDaZ7qSNSLLtq3WDCTNAD4DHAMsAE6StKCTzGU33sg+++zL/L33ZtasWbzyVSdy9be+0XGtNXIz1Zott1atAEtW/IJfPfhwV7JGZNm2NfcMng2siYg7I2IzcBlwQieB9923nrlz9/j94zlz5rJ+/frOqqyUm6nWbLm1aq0ly7at2QzmAPc2PV5XnvsDkhZJGpI0NLxxuGI5ZtZKz99AjIgLI2IwIgYHZg+0XHb33eewbt3j/WX9+nXMmbNNf5mwGrmZas2WW6vWWrJs25rNYD2wR9PjueW5SRs87DDWrPk5d991F5s3b+aKr13Gsccd31GRtXIz1Zott1attWTZtjVvVFoG7CdpPo0mcCLw6k4CZ86cyT9/8tO89Ni/YMuWLfzNyaew4KCDOi60Rm6mWrPl1qoV4MvnnMyfHbofs3fdiTXXfpgPf+4avvzv1/dlvd3OVc2BqJJeAnwCmAF8ISI+2mr5Qw8dDN+1aDXVuGvx18ty3bW4fPnQ1N/CHBHXANfUXIeZdUfP30A0s/7gZmBmgJuBmRVuBmYGuBmYWdFXA1FrqXE6CXKdUrIG/zcbm/cMzAxwMzCzws3AzAA3AzMr3AzMDHAzMLMiXTOoMWV2Ok/EzZqbqdYsuamaQa0ps9N5Im7G3Ey1ZspN1QxqTZmdzhNxM+ZmqjVTbqpmkGkqbpaJuBlzM9WaKTdVMzCzelI1g0xTcbNMxM2Ym6nWTLmpmkGmqbhZJuJmzM1Ua6bcVHct1poyO50n4mbMzVRrptyq05EnqtZ0ZN/CbNbQajpyqsMEM6vHzcDMADcDMyvcDMwMcDMws8LNwMyAZNcZTFatU4DT/UM8bfviPQMzA9wMzKxwMzAzwM3AzAo3AzMD3AzMrEjXDDJMmR3hqcv1cjPVmiW3WjOQtIek70u6XdJtks7oNDPLlNkRnrrs6ciZcmvuGTwKvD0iFgCHA6dJWtBJYJYpsyM8ddnTkTPlVmsGEbEhIlaUrzcBq4COBr9lmTJbU7Zt4OnIeXKn5D0DSfOAQ4Clo7y2SNKQpKHhjcNTUY6ZjaJ6M5C0E3AlcGZE/Hbr1yPiwogYjIjBgdkDLbOyTJmtKds28HTkPLlVm4GkJ9JoBJdExFWd5mWZMltTtm3g6ch5cqvdtShJwMXAqoj4eDcys0yZHeGpy56OnCm32nRkSc8HfgzcAjxWnn5vRFwz1vfUmo5ci29htmxaTUeutmcQET8BRl2pmfWfdFcgmlkdbgZmBrgZmFnhZmBmwDQZiFqL3/m3mmqcrfrd6rVjvuY9AzMD3AzMrHAzMDPAzcDMCjcDMwPcDMysSNcMMgyWrJnp3HqZ2XK7PXA3VTPIMlgyW63ZcjPVWjO32wN3UzWDLIMls9WaLTdTrTVzuz1wN1UzyDJYslamc+tlZszttlTNwMzqSdUMsgyWrJXp3HqZGXO7LVUzyDJYMlut2XIz1Vozt9tS3bWYZbBktlqz5WaqtWZutwfuVhuIOhnZBqKa1VTnFubLeezh+0edTZrqMMHM6nEzMDPAzcDMCjcDMwPcDMyscDMwMyDZdQY2fdQ4rQa5JlrXqHXhc24Y87Uxm4GkTcDIRQgj5yWjfB0R8ZRuFWhmvTdmM4iInaeyEDPrrbbeM5D0fEmvK1/PljS/bllmNtXGbQaSzgbeDbynPDUL+ErNosxs6rWzZ/CXwPHAQwARcR/gQwiz7Uw7zWBzNO5mCgBJO9Ytycx6oZ1mcLmkC4BdJb0R+C7w+bpljS3T9NpMtWbL7fZk4BGZtkG3c8dtBhFxPrAYuBLYH/hARHyq3RVImiFppaSrJ19mQ6bptZlqzZjb7cnAkG8bdDu33SsQbwF+DPyofD0RZwCrJvg9o8o0vTZTrRlzuz0ZGPJtg27ntnM24Q3AjcDLgVcAN0g6pZ1wSXOBY4GLJl1hk0zTazPVmjG3hmzboNu57VyO/E7gkIh4AEDSU4H/Br7Qxvd+AngXLc4+SFoELALYY88924g0sxraOUx4ANjU9HhTea4lSccB90fE8lbLRcSFETEYEYMDswdaZmaaXpup1oy5NWTbBt3OHbMZSHqbpLcBa4Clkj5YLkC6AfhZG9kLgeMl3Q1cBhwhqaOLlTJNr81Ua8bcGrJtg27ntjpMGNm1/0X5M6Ktdygi4j2UqxYlvRB4R0S8dhI1/l6m6bWZas2Y2+3JwDVrzZI7JdORm5rBca2W83RkG+FbmOtY+JxBli8fGnU68rhvIEoaoPEm4EHAk0aej4gj2i0gIn4A/KDd5c1s6rXzBuIlwB3AfOBDwN3Asoo1mVkPtNMMnhoRFwOPRMQPI+IUoO29AjPLoZ3rDB4pf2+QdCxwH/BH9Uoys15opxl8RNIuwNuBTwFPAd5atSozm3LjNoOIGLnB6EHgRXXLMbNeaTUQ9VM8PhB1GxFxepWKzMh1CnB7OQ3aas/AJ/zNppFW05G/PJWFmFlv+ROVzAxwMzCzws3AzID2Jh3tL+l7km4tj58h6X31SzOzqdTOnsHnadyK/AhARNwMnFizqFYyTJmtmencepm1cmtNcoYpno4M7BARN2713KMdrXWSskyZzVZrttxMtUKdSc7Qm+nIGyXtw+MfovIKYMOk19iBLFNms9WaLTdTrVBnkjP0YDoycBpwAXCgpPXAmcCpk15jB7JMma2V6dx6mTVza5ny6cgRcSdwVPlYtSdExKbxvsfM8mln0tEHtnoMQET8Q6WaxpRlymytTOfWy6yZW8uUTUdu8lDTny3AMcC8Sa+xA1mmzGarNVtuplprmsrpyABExD81P5Z0PnDdpNfYgSxTZrPVmi03U61QZ5JzjXonPB1Z0m7AsojYd9JrHYOnI1tGmW5h7nQ68i08PtdgBjAATPn7BWZWVztjz5o/6+BR4H8ioicXHZlZPS2bgaQZwHURceAU1WNmPdLybEJEbAFWS/LHI5tt59o5TNgNuE3SjTROLwIQEf17zsXMJqydZvD+6lWYWc+10wxeEhHvbn5C0seAH9YpyayeGqcBM01ybqWdKxD/fJTnjul2IWbWW60+N+FU4E3A3pJubnppZ2BJ7cLMbGq1Okz4KvBt4BzgrKbnN0XEr6pWZWZTrtXnJjxI4yPVTpq6csysVzwd2cwANwMzK9I1g0xTcTPVmi23Vq21Jhln2LZVm4GkXSUtlnSHpFWSnttJXqapuJlqzZZbq1aoM8k4y7atvWfwSeDacqPTM4FVnYRlmoqbqdZsubVqhTqTjLNs22rNQNIuwAuAiwEiYnNE/KaTzExTcTPVmi13uk8xrpVbc89gPjAMfFHSSkkXlQnLf0DSIklDkoaGNw5XLMfMWqnZDGYCzwI+GxGH0Ljj8aytF4qICyNiMCIGB2YPtAzMNBU3U63Zcqf7FONauTWbwTpgXUQsLY8X02gOk5ZpKm6mWrPlTvcpxrVy27lrcVIi4peS7pV0QESsBo4EOnoLNdNU3Ey1ZsutVSvUmWScZdtOeDryhMKlg4GLgFnAncDrIuLXYy3v6chW23S/hbmj6cidiIifAoM112Fm3ZHuCkQzq8PNwMwANwMzK9wMzAxwMzCzws3AzAA3AzMr3AzMDHAzMLPCzcDMADcDMyvSNYMMgyVrZjq3XiZ4IGoaWQZLZqs1W64HotbJTdUMsgyWzFZrtlwPRK2Tm6oZZBksWSvTufUya8qybVM1AzOrJ1UzyDJYslamc+tl1pRl26ZqBlkGS2arNVuuB6LWya069qzbsgyWzFZrtlwPRK2TW3Ug6kR5IKrV5oGoYw9ETXWYYGb1uBmYGeBmYGaFm4GZAW4GZla4GZgZkOw6A5s+apwChFynAaea9wzMDHAzMLPCzcDMADcDMyvcDMwMcDMwsyJdM8gwZbZmpnMbpvMU41q5VZuBpLdKuk3SrZIulfSkTvKyTJnNVmvG3Ok8xbhWbrVmIGkOcDowGBFPB2YAJ3aSmWXKbLZaM+ZO5ynGtXJrHybMBJ4saSawA3BfJ2FZpszWynRuXdm2QZrpyBGxHjgfWAtsAB6MiO9svZykRZKGJA0NbxyuVY6ZjaPmYcJuwAnAfGB3YEdJr916uYi4MCIGI2JwYPZAy8wsU2ZrZTq3rmzbINN05KOAuyJiOCIeAa4CntdJYJYps9lqzZhbQ7ZtkGk68lrgcEk7AP8LHAl0NO00y5TZbLVmzJ3OU4xr5VadjizpQ8CrgEeBlcAbIuJ3Yy3v6cg2wrcw19FqOnLVeQYRcTZwds11mFl3pLsC0czqcDMwM8DNwMwKNwMzA9wMzKzwdGTrS9P9FCDUOb36u9Vrx3zNewZmBrgZmFnhZmBmgJuBmRVuBmYGuBmYWZGuGWSYMlsz07n1MrPldntCdKpmkGXKbLZas+VmqrVmbrcnRKdqBlmmzGarNVtuplpr5nZ7QnSqZpBlymytTOfWy8yY222pmoGZ1ZOqGWSZMlsr07n1MjPmdluqZpBlymy2WrPlZqq1Zm63pbprMcuU2Wy1ZsvNVGvN3G5PiK46HXmiPB3Z7HF1bmG+nMcevn/U6cipDhPMrB43AzMD3AzMrHAzMDPAzcDMCjcDMwP67NSipGHgnjYWnQ1srFxON2WqN1OtkKvefqh1r4gYGO2FvmoG7ZI0FBGDva6jXZnqzVQr5Kq332v1YYKZAW4GZlZkbQYX9rqACcpUb6ZaIVe9fV1ryvcMzKz7su4ZmFmXuRmYGZCwGUh6saTVktZIOqvX9YxF0h6Svi/pdkm3STqj1zW1Q9IMSSslXd3rWlqRtKukxZLukLRK0nN7XVMrkt5a/h3cKulSSU/qdU1bS9UMJM0APgMcAywATpK0oLdVjelR4O0RsQA4HDitj2ttdgawqtdFtOGTwLURcSDwTPq4ZklzgNOBwYh4OjADOLG3VW0rVTMAng2siYg7I2IzcBlwQo9rGlVEbIiIFeXrTTT+sfbf4LsmkuYCxwIX9bqWViTtArwAuBggIjZHxG96W9W4ZgJPljQT2AG4r8f1bCNbM5gD3Nv0eB19/j8YgKR5wCHA0t5WMq5PAO8CHut1IeOYDwwDXyyHNBdJ2rHXRY0lItYD5wNrgQ3AgxHxnd5Wta1szSAdSTsBVwJnRsRve13PWCQdB9wfEct7XUsbZgLPAj4bEYcADwH9/P7RbjT2YOcDuwM7Snptb6vaVrZmsB7Yo+nx3PJcX5L0RBqN4JKIuKrX9YxjIXC8pLtpHH4dIekrvS1pTOuAdRExsqe1mEZz6FdHAXdFxHBEPAJcBTyvxzVtI1szWAbsJ2m+pFk03oT5Zo9rGpUk0TimXRURH+91PeOJiPdExNyImEdju/5XRPTdby+AiPglcK+kA8pTRwKdf3hhPWuBwyXtUP5dHEkfvuGZalR6RDwq6c3AdTTekf1CRNzW47LGshD4K+AWST8tz703Iq7pYU3bk7cAl5RfCncCr+txPWOKiKWSFgMraJxlWkkfXprsy5HNDMh3mGBmlbgZmBngZmBmhZuBmQFuBmZWuBlMU5JeOHJnoqTjW90BWu4QfNMk1vFBSe9o9/mtlvmSpFdMYF3zJN060RrtcW4G25lyZ+eERMQ3I+LcFovsCky4GVgubgZJlN98d0i6pNy/v1jSDuW1uyV9TNIK4JWSjpZ0vaQVkq4o90eMzIK4oyz38qbskyV9unz9J5K+Lumm8ud5wLnAPpJ+Kum8stw7JS2TdLOkDzVl/b2kn0n6CXAA45D0xpJzk6QrR36m4ihJQyXvuLL8DEnnNa37bzvdttbgZpDLAcC/RsTTgN/yh7+tH4iIZwHfBd4HHFUeDwFvK8M0Pg+8FDgU+NMx1vEvwA8j4pk0rve/jcZNQL+IiIMj4p2Sjgb2o3FL+cHAoZJeIOlQGpcyHwy8BDisjZ/pqog4rKxvFfD6ptfmlXUcC3yu/Ayvp3HX32El/42S5rexHhtHqsuRjXsjYkn5+is0BmacXx5/rfx9OI3BL0sal8EzC7geOJDGzTI/Byg3IS0aZR1HAH8NEBFbgAfLXXfNji5/VpbHO9FoDjsDX4+Ih8s62rlv5OmSPkLjUGQnGpeaj7g8Ih4Dfi7pzvIzHA08o+n9hF3Kun/WxrqsBTeDXLa+drz58UPlbwH/GREnNS8o6eAu1iHgnIi4YKt1nDmJrC8BL4uImySdDLyw6bXRfl4Bb4mI5qYxMjPCOuDDhFz2bJr192rgJ6MscwOwUNK+AJJ2lLQ/cAcwT9I+ZbmTRvlegO8Bp5bvnVGmCm2i8Vt/xHXAKU3vRcyR9MfAj4CXSXqypJ1pHJKMZ2dgQ7nd+zVbvfZKSU8oNe8NrC7rPrUsj6T9+3mwSSZuBrmspjFLcRWwG/DZrReIiGHgZOBSSTdTDhEi4v9oHBb8R3kD8f4x1nEG8CJJtwDLgQUR8QCNw45bJZ1XpvR8Fbi+LLcY2LmMefsacBPwbRq3nI/n/TQmQC2h0bCarQVuLFl/V36Gi2jcrryinEq8AO/hdoXvWkyi7AZfXQZqmnWd9wzMDPCegZkV3jMwM8DNwMwKNwMzA9wMzKxwMzAzAP4fBYmiqh2amcQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eBU8rxdLEI9z",
        "outputId": "e9c33626-62ec-4813-d1da-94da71d9359c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(metrics.classification_report(list_data_answers, list_data_FoundCode))"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.00      0.00      0.00         1\n",
            "           2       1.00      1.00      1.00         1\n",
            "           3       1.00      1.00      1.00         1\n",
            "           4       0.00      0.00      0.00         1\n",
            "           5       0.00      0.00      0.00         1\n",
            "           6       0.00      0.00      0.00         1\n",
            "           7       0.00      0.00      0.00         1\n",
            "           8       0.33      1.00      0.50         1\n",
            "           9       0.00      0.00      0.00         1\n",
            "          10       0.50      1.00      0.67         1\n",
            "\n",
            "    accuracy                           0.40        10\n",
            "   macro avg       0.28      0.40      0.32        10\n",
            "weighted avg       0.28      0.40      0.32        10\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BwHC3FlnES2Y"
      },
      "source": [
        ""
      ],
      "execution_count": 59,
      "outputs": []
    }
  ]
}