evals.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "import os\n",
    "from trulens_eval import TruLlama, FeedbackMode, Feedback, Tru\n",
    "from trulens_eval.feedback import Groundedness\n",
    "from trulens_eval import OpenAI as fOpenAI\n",
    "from llama_index import StorageContext, load_index_from_storage\n",
    "import streamlit as st\n",
    "import numpy as np\n",
    "import google.auth\n",
    "import pandas as pd\n",
    "import nest_asyncio\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_file():\n",
    "    key_path = st.secrets[\"JSON_PATH\"]\n",
    "    with open(key_path, \"w+\") as f:\n",
    "        creds = {\n",
    "            \"type\": st.secrets[\"type\"],\n",
    "            \"project_id\": st.secrets[\"project_id\"],\n",
    "            \"private_key_id\": st.secrets[\"private_key_id\"],\n",
    "            \"private_key\": st.secrets[\"private_key\"],\n",
    "            \"client_email\": st.secrets[\"client_email\"],\n",
    "            \"client_id\": st.secrets[\"client_id\"],\n",
    "            \"auth_uri\": st.secrets[\"auth_uri\"],\n",
    "            \"token_uri\": st.secrets[\"token_uri\"],\n",
    "            \"auth_provider_x509_cert_url\": st.secrets[\"auth_provider_x509_cert_url\"],\n",
    "            \"client_x509_cert_url\": st.secrets[\"client_x509_cert_url\"],\n",
    "            \"universe_domain\": st.secrets[\"universe_domain\"]\n",
    "        }\n",
    "        json.dump(creds, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Authenticate with Google Cloud\n",
    "write_file()\n",
    "key_path = st.secrets[\"JSON_PATH\"]\n",
    "os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"] = key_path\n",
    "credentials, project_id = google.auth.default()\n",
    "\n",
    "# Authenticate with OpenAI\n",
    "os.environ[\"OPENAI_API_KEY\"] = st.secrets[\"OPEN_AI_API_KEY\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rebuild storage context\n",
    "storage_context = StorageContext.from_defaults(persist_dir=\"ask_priya_index\")\n",
    "\n",
    "# Load index from the storage context\n",
    "index = load_index_from_storage(storage_context)\n",
    "\n",
    "query_engine = index.as_query_engine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ In Answer Relevance, input prompt will be set to __record__.main_input or `Select.RecordInput` .\n",
      "✅ In Answer Relevance, input response will be set to __record__.main_output or `Select.RecordOutput` .\n"
     ]
    }
   ],
   "source": [
    "provider = fOpenAI()\n",
    "f_qa_relevance = Feedback(\n",
    "    provider.relevance_with_cot_reasons,\n",
    "    name=\"Answer Relevance\"\n",
    ").on_input_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ In Context Relevance, input question will be set to __record__.main_input or `Select.RecordInput` .\n",
      "✅ In Context Relevance, input statement will be set to __record__.app.query.rets.source_nodes[:].node.text .\n"
     ]
    }
   ],
   "source": [
    "context_selection = TruLlama.select_source_nodes().node.text\n",
    "f_qs_relevance = (\n",
    "    Feedback(provider.qs_relevance_with_cot_reasons,\n",
    "             name=\"Context Relevance\")\n",
    "    .on_input()\n",
    "    .on(context_selection)\n",
    "    .aggregate(np.mean)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ In Groundedness, input source will be set to __record__.app.query.rets.source_nodes[:].node.text .\n",
      "✅ In Groundedness, input statement will be set to __record__.main_output or `Select.RecordOutput` .\n"
     ]
    }
   ],
   "source": [
    "grounded = Groundedness(groundedness_provider=provider)\n",
    "f_groundedness = (\n",
    "    Feedback(grounded.groundedness_measure_with_cot_reasons,\n",
    "             name=\"Groundedness\"\n",
    "            )\n",
    "    .on(context_selection)\n",
    "    .on_output()\n",
    "    .aggregate(grounded.grounded_statements_aggregator)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🦑 Tru initialized with db url sqlite:///default.sqlite .\n",
      "🛑 Secret keys may be written to the database. See the `database_redact_keys` option of `Tru` to prevent this.\n"
     ]
    }
   ],
   "source": [
    "tru = Tru()\n",
    "tru.reset_database()\n",
    "tru_recorder = TruLlama(\n",
    "    query_engine,\n",
    "    app_id=\"evaluator\",\n",
    "    feedbacks=[\n",
    "        f_qa_relevance,\n",
    "        f_qs_relevance,\n",
    "        f_groundedness\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load evaluation questions\n",
    "eval_questions = []\n",
    "with open('data/eval_questions.txt', 'r') as file:\n",
    "    for line in file:\n",
    "        # Remove newline character and convert to integer\n",
    "        item = line.strip()\n",
    "        eval_questions.append(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run evaluation engine on each eval question\n",
    "for question in eval_questions:\n",
    "    with tru_recorder as recording:\n",
    "        query_engine.query(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "      <th>Groundedness</th>\n",
       "      <th>Answer Relevance</th>\n",
       "      <th>Context Relevance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"Explain the process to apply for OPT for a college senior.\"</td>\n",
       "      <td>\"To apply for Optional Practical Training (OPT) as a college senior, the student should follow the following process:\\n\\n1. Meet with the Designated School Official (DSO): The student should schedule a meeting with their DSO to discuss their eligibility for OPT and to understand the application process.\\n\\n2. Submit Form I-765: The student needs to complete and submit Form I-765, Application for Employment Authorization, to the U.S. Citizenship and Immigration Services (USCIS). This form can be found on the USCIS website.\\n\\n3. Gather supporting documents: The student should gather the necessary supporting documents, which may include a copy of their current Form I-20, passport, visa, I-94 record, and any other required documentation specified by the USCIS.\\n\\n4. Pay the application fee: The student needs to pay the required application fee, which is specified on the USCIS website. The fee can be paid online or by mail, following the instructions provided by the USCIS.\\n\\n5. Submit the application: The student should submit the completed Form I-765, along with the supporting documents and application fee, to the USCIS. The submission can be done online or by mail, depending on the instructions provided by the USCIS.\\n\\n6. Receive the Employment Authorization Document (EAD): If the application is approved, the student will receive an Employment Authorization Document (EAD) from the USCIS. This document serves as proof of the student's employment authorization during the OPT period.\\n\\nIt is important for the student to consult with their DSO throughout the application process to ensure they are following the correct procedures and meeting all the requirements. The DSO can provide guidance and assistance in completing the application and addressing any questions or concerns that may arise.\"</td>\n",
       "      <td>0.187500</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Explain the process to apply to H1B.\"</td>\n",
       "      <td>\"I'm sorry, but the given context information does not provide any information about the process to apply for an H-1B visa.\"</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"What are the differences between single-intent and dual-intent visas?\"</td>\n",
       "      <td>\"A single-intent visa is a type of visa that requires the bearer to have the intention to depart the United States when their nonimmigrant stay expires. This means that the individual must demonstrate that they do not have any intention to immigrate or stay permanently in the United States.\\n\\nOn the other hand, a dual-intent visa allows the bearer to have both the intention to temporarily stay in the United States and the intention to potentially immigrate or stay permanently in the country. This means that individuals with a dual-intent visa can apply for entry to the United States in a certain nonimmigrant classification, such as a temporary worker or student, while also having the option to pursue permanent residency in the future.\\n\\nIt is important to note that the issuance of a visa does not guarantee entry into the United States. The Department of State is responsible for visa adjudication, while U.S. Customs and Border Protection immigration inspectors determine the admission, length of stay, and conditions of stay at a port of entry.\"</td>\n",
       "      <td>0.466667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"Can a persion on OPT travel outside the U.S. with an expired visa stamp?\"</td>\n",
       "      <td>\"No, a person on OPT cannot travel outside the U.S. with an expired visa stamp.\"</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"Who qualify for STEM OPT extension?\"</td>\n",
       "      <td>\"F-1 students who receive science, technology, engineering, and mathematics (STEM) degrees included on the STEM Designated Degree Program List, are employed by employers enrolled in and maintain good standing with E-Verify, and who have received an initial grant of post-completion OPT employment authorization related to such a degree, may qualify for a 24-month extension of such authorization.\"</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>\"When is the earliest a student can apply for OPT?\"</td>\n",
       "      <td>\"The earliest a student can apply for OPT is after their program end date.\"</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>\"How is H1B different from OPT?\"</td>\n",
       "      <td>\"H-1B and OPT (Optional Practical Training) are different in terms of their purpose and eligibility criteria. H-1B is a nonimmigrant visa category that allows U.S. employers to hire foreign workers in specialty occupations. It is typically used for highly skilled professionals who possess specialized knowledge or advanced degrees. On the other hand, OPT is a temporary employment authorization program for F-1 students who have completed their degree programs in the U.S. OPT allows students to gain practical work experience in their field of study for a certain period of time after completing their studies. Unlike H-1B, which requires sponsorship from an employer, OPT is granted directly to the student by the U.S. Citizenship and Immigration Services (USCIS).\"</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>\"What are the documents required to apply for a student F1 visa?\"</td>\n",
       "      <td>\"To apply for a student F-1 visa, you would need to provide documents such as a valid passport, a completed Form DS-160, a SEVIS-generated I-20 form from the school you plan to attend, proof of financial support, and evidence of your intent to return to your home country after completing your studies.\"</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>\"What are the documents required to apply for a H1B work visa?\"</td>\n",
       "      <td>\"The documents required to apply for an H-1B work visa include:\\n1. Form G-28 (if represented by an attorney or accredited representative)\\n2. Copy of the Registration Selection Notice for the Beneficiary Named in the Petition\\n3. Form I-129, Petition for a Nonimmigrant Worker\\n4. Addendums/Attachments\\n5. H Classification Supplement to Form I-129 and/or Trade Agreement Supplement to Form I-129 (for H-1B1 Chile-Singapore petitions)\\n6. H-1B and H-1B1 Data Collection and Filing Fee Exemption Supplement\\n7. All supporting documentation to establish eligibility. Provide a table of contents for supporting documentation and separate the items as listed in the table.\\n8. Arrival-Departure Record (Form I-94) if the beneficiary is in the United States\\n9. SEVIS Form I-20 if the beneficiary is a current or former F-1 student or F-2 dependent\\n10. SEVIS Form DS-2019 if the beneficiary is a current or former J-1 or J-2\\n11. Form I-566 if the beneficiary is a current A or G nonimmigrant\\n12. Department of Labor certified LCA, Form ETA 9035\\n13. Employer/attorney/representative letter(s)\\n14. Other supporting documentation.\"</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>\"Can someone on OPT start a business?\"</td>\n",
       "      <td>\"Yes, someone on OPT can start a business. They may start, own, and work for their own business during their initial period of OPT. However, if they want to qualify for a STEM OPT extension, the start-up entity must meet certain requirements, such as being an employer in good standing with E-Verify and having the resources to comply with the proposed training plan. Additionally, the work performed for the business must directly relate to the individual's major area of study.\"</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>\"Can someone on H1B start a business?\"</td>\n",
       "      <td>\"Yes, someone on an H-1B visa can start a business in the United States. However, they must be aware that the H-1B visa is tied to their employment with a specific employer, so they would need to maintain their employment with that employer while also managing their business. Additionally, starting a business may have implications for their immigration status, so it is important to consult with an attorney to understand the specific requirements and considerations.\"</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>\"Can someone on L1 start a business?\"</td>\n",
       "      <td>\"Yes, someone on an L-1 visa can start a business in the United States. The L-1 visa is available for intracompany transferees who are executives or managers and have worked abroad for a qualifying organization for at least one year within the three years preceding the filing of the L-1 petition. The organization can transfer the individual to a U.S. entity that is already doing business or to start a new office. However, there are specific requirements that need to be met, such as demonstrating a qualifying relationship between the new U.S. office and the foreign employer, securing sufficient physical space for the new office, and ensuring that the new office is active and operating within one year.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>\"How to apply for premium processing of OPT application?\"</td>\n",
       "      <td>\"Premium processing is not available for OPT applications. OPT applications are processed by USCIS under regular processing times. Premium processing is typically available for certain employment-based visa categories, such as H-1B visas. However, it is not applicable to OPT applications.\"</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>\"How long does it usually take to process an OPT application?\"</td>\n",
       "      <td>\"The processing time for an OPT application can vary. It is recommended to apply for OPT up to 90 days before completing a full academic year for pre-completion OPT, and within 30 days after the Designated School Official (DSO) enters the recommendation into the SEVIS record for post-completion OPT. USCIS must receive the Form I-765 for post-completion OPT during the period that is no earlier than 90 days before completing the degree and ending no later than 60 days after completing the degree. For a 24-month STEM OPT extension, the Form I-765 must be received by USCIS during the 90-day period prior to the expiration of the post-completion OPT and within 60 days after the DSO enters the recommendation for STEM OPT into the SEVIS record. It is important to note that processing times can vary, so it is advisable to apply as early as possible to allow for any potential delays.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>\"How to apply for premium processing of H1B application?\"</td>\n",
       "      <td>\"To apply for premium processing of an H-1B application, you can follow the instructions provided on the USCIS website. Additional information on how to request premium processing can be found on the \\\"How Do I Request Premium Processing?\\\" page, which is available on the USCIS website.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>\"When is the deadline to apply for H1B in 2023?\"</td>\n",
       "      <td>\"The deadline to apply for H1B in 2023 is not provided in the given context information.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>\"When is the deadline to renew H1B in 2023?\"</td>\n",
       "      <td>\"There is no information provided in the given context about the deadline to renew H1B visas in 2023.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>\"How long can someone reside and work in the U.S. with an H1B visa?\"</td>\n",
       "      <td>\"Someone can reside and work in the U.S. with an H-1B visa for a period of up to three years. This time period may be extended, but generally cannot exceed a total of six years. However, there are some exceptions to this rule under certain circumstances.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>\"Can a student on OPT with an expired F1 visa travel to the Bahamas?\"</td>\n",
       "      <td>\"Yes, a student on OPT with an expired F1 visa can travel to the Bahamas. The context information does not provide any restrictions or limitations on travel to the Bahamas for students on OPT with an expired F1 visa. However, it is always advisable to check with the U.S. Customs and Border Protection (CBP) officer for the final determination on whether to admit an applicant for admission after inspection at a port-of-entry.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>\"Can a student on OPT with an expired F1 visa travel to the Canada?\"</td>\n",
       "      <td>\"Yes, a student on OPT with an expired F1 visa can travel to Canada.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>\"Can someone with an Indian passport apply for H1B stamp in Canada?\"</td>\n",
       "      <td>\"No, someone with an Indian passport cannot apply for an H1B stamp in Canada. The context information only mentions that Canadian citizens can establish eligibility for TN classification at certain U.S. ports of entry or pre-clearance/pre-flight inspection stations. It does not provide any information about Indian passport holders applying for an H1B stamp in Canada.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>\"Can someone with a Vietnamese passport apply for H1B stamp in Canada?\"</td>\n",
       "      <td>\"No, someone with a Vietnamese passport cannot apply for an H1B stamp in Canada. The context information only mentions that Canadian citizens can establish eligibility for TN classification at certain ports of entry in the United States. It does not provide any information about H1B stamps or the eligibility of Vietnamese passport holders.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>\"Can someone on OPT work from France?\"</td>\n",
       "      <td>\"No, someone on OPT cannot work from France. OPT stands for Optional Practical Training, which allows F-1 students in the United States to work in a position directly related to their major area of study. OPT is only valid for employment within the United States, and individuals on OPT are required to physically be in the United States to work. Therefore, they cannot work remotely from another country like France.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>\"How can someone get an O1 visa?\"</td>\n",
       "      <td>\"To obtain an O-1 visa, an individual must have an employer or agent file a Form I-129, Petition for Nonimmigrant Worker, on their behalf. The petition should be accompanied by the required evidence according to the form instructions. The employer or agent should file the Form I-129 at least 45 days before the date of employment to avoid delays. \\n\\nIn addition to the Form I-129, the petitioner must provide documentary evidence to establish the individual's extraordinary ability or achievement. This evidence should demonstrate the individual's current essentiality, critical skills, and experience with the O-1 beneficiary. It should also establish that the individual has substantial experience performing the critical skills and essential support services for the O-1.\\n\\nThe petitioner must also provide a consultation from an appropriate labor organization or a person with expertise in the beneficiary's area of ability. If the O-1 petition is for an individual with extraordinary achievement in motion picture or television, the consultation must come from an appropriate labor union and a management organization with expertise in the beneficiary's area of ability.\\n\\nOnce the petition is approved, the individual can apply for the O-1 visa at a U.S. Embassy or Consulate. The Department of State establishes visa application processing and issuance fees.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>\"How many days of unemployment does a student on OPT have?\"</td>\n",
       "      <td>\"A student on OPT may be unemployed for up to 90 days during their initial post-completion OPT period. If they are granted a 24-month extension, they may be unemployed for an additional 60 days, for a total of 150 days of unemployment during their OPT period.\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                         input  \\\n",
       "0                 \"Explain the process to apply for OPT for a college senior.\"   \n",
       "1                                       \"Explain the process to apply to H1B.\"   \n",
       "2      \"What are the differences between single-intent and dual-intent visas?\"   \n",
       "3   \"Can a persion on OPT travel outside the U.S. with an expired visa stamp?\"   \n",
       "4                                        \"Who qualify for STEM OPT extension?\"   \n",
       "5                          \"When is the earliest a student can apply for OPT?\"   \n",
       "6                                             \"How is H1B different from OPT?\"   \n",
       "7            \"What are the documents required to apply for a student F1 visa?\"   \n",
       "8              \"What are the documents required to apply for a H1B work visa?\"   \n",
       "9                                       \"Can someone on OPT start a business?\"   \n",
       "10                                      \"Can someone on H1B start a business?\"   \n",
       "11                                       \"Can someone on L1 start a business?\"   \n",
       "12                   \"How to apply for premium processing of OPT application?\"   \n",
       "13              \"How long does it usually take to process an OPT application?\"   \n",
       "14                   \"How to apply for premium processing of H1B application?\"   \n",
       "15                            \"When is the deadline to apply for H1B in 2023?\"   \n",
       "16                                \"When is the deadline to renew H1B in 2023?\"   \n",
       "17        \"How long can someone reside and work in the U.S. with an H1B visa?\"   \n",
       "18       \"Can a student on OPT with an expired F1 visa travel to the Bahamas?\"   \n",
       "19        \"Can a student on OPT with an expired F1 visa travel to the Canada?\"   \n",
       "20        \"Can someone with an Indian passport apply for H1B stamp in Canada?\"   \n",
       "21     \"Can someone with a Vietnamese passport apply for H1B stamp in Canada?\"   \n",
       "22                                      \"Can someone on OPT work from France?\"   \n",
       "23                                           \"How can someone get an O1 visa?\"   \n",
       "24                 \"How many days of unemployment does a student on OPT have?\"   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         output  \\\n",
       "0   \"To apply for Optional Practical Training (OPT) as a college senior, the student should follow the following process:\\n\\n1. Meet with the Designated School Official (DSO): The student should schedule a meeting with their DSO to discuss their eligibility for OPT and to understand the application process.\\n\\n2. Submit Form I-765: The student needs to complete and submit Form I-765, Application for Employment Authorization, to the U.S. Citizenship and Immigration Services (USCIS). This form can be found on the USCIS website.\\n\\n3. Gather supporting documents: The student should gather the necessary supporting documents, which may include a copy of their current Form I-20, passport, visa, I-94 record, and any other required documentation specified by the USCIS.\\n\\n4. Pay the application fee: The student needs to pay the required application fee, which is specified on the USCIS website. The fee can be paid online or by mail, following the instructions provided by the USCIS.\\n\\n5. Submit the application: The student should submit the completed Form I-765, along with the supporting documents and application fee, to the USCIS. The submission can be done online or by mail, depending on the instructions provided by the USCIS.\\n\\n6. Receive the Employment Authorization Document (EAD): If the application is approved, the student will receive an Employment Authorization Document (EAD) from the USCIS. This document serves as proof of the student's employment authorization during the OPT period.\\n\\nIt is important for the student to consult with their DSO throughout the application process to ensure they are following the correct procedures and meeting all the requirements. The DSO can provide guidance and assistance in completing the application and addressing any questions or concerns that may arise.\"   \n",
       "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \"I'm sorry, but the given context information does not provide any information about the process to apply for an H-1B visa.\"   \n",
       "2                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          \"A single-intent visa is a type of visa that requires the bearer to have the intention to depart the United States when their nonimmigrant stay expires. This means that the individual must demonstrate that they do not have any intention to immigrate or stay permanently in the United States.\\n\\nOn the other hand, a dual-intent visa allows the bearer to have both the intention to temporarily stay in the United States and the intention to potentially immigrate or stay permanently in the country. This means that individuals with a dual-intent visa can apply for entry to the United States in a certain nonimmigrant classification, such as a temporary worker or student, while also having the option to pursue permanent residency in the future.\\n\\nIt is important to note that the issuance of a visa does not guarantee entry into the United States. The Department of State is responsible for visa adjudication, while U.S. Customs and Border Protection immigration inspectors determine the admission, length of stay, and conditions of stay at a port of entry.\"   \n",
       "3                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \"No, a person on OPT cannot travel outside the U.S. with an expired visa stamp.\"   \n",
       "4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \"F-1 students who receive science, technology, engineering, and mathematics (STEM) degrees included on the STEM Designated Degree Program List, are employed by employers enrolled in and maintain good standing with E-Verify, and who have received an initial grant of post-completion OPT employment authorization related to such a degree, may qualify for a 24-month extension of such authorization.\"   \n",
       "5                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \"The earliest a student can apply for OPT is after their program end date.\"   \n",
       "6                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \"H-1B and OPT (Optional Practical Training) are different in terms of their purpose and eligibility criteria. H-1B is a nonimmigrant visa category that allows U.S. employers to hire foreign workers in specialty occupations. It is typically used for highly skilled professionals who possess specialized knowledge or advanced degrees. On the other hand, OPT is a temporary employment authorization program for F-1 students who have completed their degree programs in the U.S. OPT allows students to gain practical work experience in their field of study for a certain period of time after completing their studies. Unlike H-1B, which requires sponsorship from an employer, OPT is granted directly to the student by the U.S. Citizenship and Immigration Services (USCIS).\"   \n",
       "7                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \"To apply for a student F-1 visa, you would need to provide documents such as a valid passport, a completed Form DS-160, a SEVIS-generated I-20 form from the school you plan to attend, proof of financial support, and evidence of your intent to return to your home country after completing your studies.\"   \n",
       "8                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \"The documents required to apply for an H-1B work visa include:\\n1. Form G-28 (if represented by an attorney or accredited representative)\\n2. Copy of the Registration Selection Notice for the Beneficiary Named in the Petition\\n3. Form I-129, Petition for a Nonimmigrant Worker\\n4. Addendums/Attachments\\n5. H Classification Supplement to Form I-129 and/or Trade Agreement Supplement to Form I-129 (for H-1B1 Chile-Singapore petitions)\\n6. H-1B and H-1B1 Data Collection and Filing Fee Exemption Supplement\\n7. All supporting documentation to establish eligibility. Provide a table of contents for supporting documentation and separate the items as listed in the table.\\n8. Arrival-Departure Record (Form I-94) if the beneficiary is in the United States\\n9. SEVIS Form I-20 if the beneficiary is a current or former F-1 student or F-2 dependent\\n10. SEVIS Form DS-2019 if the beneficiary is a current or former J-1 or J-2\\n11. Form I-566 if the beneficiary is a current A or G nonimmigrant\\n12. Department of Labor certified LCA, Form ETA 9035\\n13. Employer/attorney/representative letter(s)\\n14. Other supporting documentation.\"   \n",
       "9                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \"Yes, someone on OPT can start a business. They may start, own, and work for their own business during their initial period of OPT. However, if they want to qualify for a STEM OPT extension, the start-up entity must meet certain requirements, such as being an employer in good standing with E-Verify and having the resources to comply with the proposed training plan. Additionally, the work performed for the business must directly relate to the individual's major area of study.\"   \n",
       "10                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \"Yes, someone on an H-1B visa can start a business in the United States. However, they must be aware that the H-1B visa is tied to their employment with a specific employer, so they would need to maintain their employment with that employer while also managing their business. Additionally, starting a business may have implications for their immigration status, so it is important to consult with an attorney to understand the specific requirements and considerations.\"   \n",
       "11                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \"Yes, someone on an L-1 visa can start a business in the United States. The L-1 visa is available for intracompany transferees who are executives or managers and have worked abroad for a qualifying organization for at least one year within the three years preceding the filing of the L-1 petition. The organization can transfer the individual to a U.S. entity that is already doing business or to start a new office. However, there are specific requirements that need to be met, such as demonstrating a qualifying relationship between the new U.S. office and the foreign employer, securing sufficient physical space for the new office, and ensuring that the new office is active and operating within one year.\"   \n",
       "12                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \"Premium processing is not available for OPT applications. OPT applications are processed by USCIS under regular processing times. Premium processing is typically available for certain employment-based visa categories, such as H-1B visas. However, it is not applicable to OPT applications.\"   \n",
       "13                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      \"The processing time for an OPT application can vary. It is recommended to apply for OPT up to 90 days before completing a full academic year for pre-completion OPT, and within 30 days after the Designated School Official (DSO) enters the recommendation into the SEVIS record for post-completion OPT. USCIS must receive the Form I-765 for post-completion OPT during the period that is no earlier than 90 days before completing the degree and ending no later than 60 days after completing the degree. For a 24-month STEM OPT extension, the Form I-765 must be received by USCIS during the 90-day period prior to the expiration of the post-completion OPT and within 60 days after the DSO enters the recommendation for STEM OPT into the SEVIS record. It is important to note that processing times can vary, so it is advisable to apply as early as possible to allow for any potential delays.\"   \n",
       "14                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \"To apply for premium processing of an H-1B application, you can follow the instructions provided on the USCIS website. Additional information on how to request premium processing can be found on the \\\"How Do I Request Premium Processing?\\\" page, which is available on the USCIS website.\"   \n",
       "15                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    \"The deadline to apply for H1B in 2023 is not provided in the given context information.\"   \n",
       "16                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \"There is no information provided in the given context about the deadline to renew H1B visas in 2023.\"   \n",
       "17                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \"Someone can reside and work in the U.S. with an H-1B visa for a period of up to three years. This time period may be extended, but generally cannot exceed a total of six years. However, there are some exceptions to this rule under certain circumstances.\"   \n",
       "18                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \"Yes, a student on OPT with an expired F1 visa can travel to the Bahamas. The context information does not provide any restrictions or limitations on travel to the Bahamas for students on OPT with an expired F1 visa. However, it is always advisable to check with the U.S. Customs and Border Protection (CBP) officer for the final determination on whether to admit an applicant for admission after inspection at a port-of-entry.\"   \n",
       "19                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        \"Yes, a student on OPT with an expired F1 visa can travel to Canada.\"   \n",
       "20                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \"No, someone with an Indian passport cannot apply for an H1B stamp in Canada. The context information only mentions that Canadian citizens can establish eligibility for TN classification at certain U.S. ports of entry or pre-clearance/pre-flight inspection stations. It does not provide any information about Indian passport holders applying for an H1B stamp in Canada.\"   \n",
       "21                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \"No, someone with a Vietnamese passport cannot apply for an H1B stamp in Canada. The context information only mentions that Canadian citizens can establish eligibility for TN classification at certain ports of entry in the United States. It does not provide any information about H1B stamps or the eligibility of Vietnamese passport holders.\"   \n",
       "22                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \"No, someone on OPT cannot work from France. OPT stands for Optional Practical Training, which allows F-1 students in the United States to work in a position directly related to their major area of study. OPT is only valid for employment within the United States, and individuals on OPT are required to physically be in the United States to work. Therefore, they cannot work remotely from another country like France.\"   \n",
       "23                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \"To obtain an O-1 visa, an individual must have an employer or agent file a Form I-129, Petition for Nonimmigrant Worker, on their behalf. The petition should be accompanied by the required evidence according to the form instructions. The employer or agent should file the Form I-129 at least 45 days before the date of employment to avoid delays. \\n\\nIn addition to the Form I-129, the petitioner must provide documentary evidence to establish the individual's extraordinary ability or achievement. This evidence should demonstrate the individual's current essentiality, critical skills, and experience with the O-1 beneficiary. It should also establish that the individual has substantial experience performing the critical skills and essential support services for the O-1.\\n\\nThe petitioner must also provide a consultation from an appropriate labor organization or a person with expertise in the beneficiary's area of ability. If the O-1 petition is for an individual with extraordinary achievement in motion picture or television, the consultation must come from an appropriate labor union and a management organization with expertise in the beneficiary's area of ability.\\n\\nOnce the petition is approved, the individual can apply for the O-1 visa at a U.S. Embassy or Consulate. The Department of State establishes visa application processing and issuance fees.\"   \n",
       "24                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \"A student on OPT may be unemployed for up to 90 days during their initial post-completion OPT period. If they are granted a 24-month extension, they may be unemployed for an additional 60 days, for a total of 150 days of unemployment during their OPT period.\"   \n",
       "\n",
       "    Groundedness  Answer Relevance  Context Relevance  \n",
       "0       0.187500               1.0               0.75  \n",
       "1       0.000000               1.0               0.00  \n",
       "2       0.466667               1.0               0.75  \n",
       "3       0.000000               1.0               0.50  \n",
       "4       1.000000               1.0               1.00  \n",
       "5       0.000000               1.0               0.80  \n",
       "6       0.000000               1.0               0.80  \n",
       "7       0.200000               1.0               0.60  \n",
       "8       1.000000               1.0               0.95  \n",
       "9       1.000000               1.0                NaN  \n",
       "10      0.333333               1.0                NaN  \n",
       "11           NaN               1.0                NaN  \n",
       "12      0.125000               1.0                NaN  \n",
       "13           NaN               1.0                NaN  \n",
       "14           NaN               1.0                NaN  \n",
       "15           NaN               1.0                NaN  \n",
       "16           NaN               1.0                NaN  \n",
       "17           NaN               1.0                NaN  \n",
       "18           NaN               NaN                NaN  \n",
       "19           NaN               NaN                NaN  \n",
       "20           NaN               NaN                NaN  \n",
       "21           NaN               NaN                NaN  \n",
       "22           NaN               NaN                NaN  \n",
       "23           NaN               NaN                NaN  \n",
       "24           NaN               NaN                NaN  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records, feedback = tru.get_records_and_feedback(app_ids=[])\n",
    "pd.set_option(\"display.max_colwidth\", None)\n",
    "records[[\"input\", \"output\"] + feedback]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Groundedness</th>\n",
       "      <th>Answer Relevance</th>\n",
       "      <th>Context Relevance</th>\n",
       "      <th>latency</th>\n",
       "      <th>total_cost</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>app_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>evaluator</th>\n",
       "      <td>0.359375</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.683333</td>\n",
       "      <td>3.72</td>\n",
       "      <td>0.001881</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Groundedness  Answer Relevance  Context Relevance  latency  \\\n",
       "app_id                                                                  \n",
       "evaluator      0.359375               1.0           0.683333     3.72   \n",
       "\n",
       "           total_cost  \n",
       "app_id                 \n",
       "evaluator    0.001881  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# View aggregated metrics for all eval questions\n",
    "tru.get_leaderboard(app_ids=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting dashboard ...\n",
      "Config file already exists. Skipping writing process.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "157308606f8f4d7fbe33931e646c0b91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Accordion(children=(VBox(children=(VBox(children=(Label(value='STDOUT'), Output())), VBox(children=(Label(valu…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dashboard started at http://192.168.86.55:8502 .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Popen: returncode: None args: ['streamlit', 'run', '--server.headless=True'...>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nest_asyncio.apply()\n",
    "tru.run_dashboard()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}