Solution/luis_solution/P10_train_luis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Apprentissage LUIS (Language Understanding Intelligence Service)\n",
    "\n",
    "helper : https://docs.microsoft.com/en-us/azure/bot-service/bot-builder-howto-v4-luis?view=azure-bot-service-4.0&tabs=python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import os\n",
    "\n",
    "from azure.cognitiveservices.language.luis.authoring import LUISAuthoringClient\n",
    "from msrest.authentication import CognitiveServicesCredentials\n",
    "\n",
    "AUTHORING_KEY_LUIS = os.environ[\"AUTHORING_KEY_LUIS\"]\n",
    "AUTHORING_ENDPOINT_LUIS = os.environ[\"AUTHORING_ENDPOINT_LUIS\"]\n",
    "# Instantiate a LUIS client\n",
    "client = LUISAuthoringClient(\n",
    "    AUTHORING_ENDPOINT_LUIS, CognitiveServicesCredentials(AUTHORING_KEY_LUIS))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "app_id = \"8958f321-de6e-4cee-bf26-e2468b5f381f\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created LUIS app Fly Bot\n",
      "    with ID 8958f321-de6e-4cee-bf26-e2468b5f381f\n"
     ]
    }
   ],
   "source": [
    "\n",
    "app_name = \"Fly Bot\"\n",
    "app_desc = \"Flight booking app built with LUIS Python SDK.\"\n",
    "app_version = \"0.1\"\n",
    "app_locale = \"en-us\"\n",
    "\n",
    "# app_id = client.apps.add(dict(name=app_name,\n",
    "#                                   initial_version_id=app_version,\n",
    "#                                   description=app_desc,\n",
    "#                                   culture=app_locale))\n",
    "\n",
    "client.apps.get(app_id=app_id)\n",
    "\n",
    "print(\"Created LUIS app {}\\n    with ID {}\".format(app_name, app_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "information = client.apps.get(app_id)\n",
    "\n",
    "app_version = information.active_version\n",
    "app_locale = information.culture\n",
    "app_desc = information.description\n",
    "app_name = information.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from luis_module.get_data import LuisData \n",
    "\n",
    "luis_data = LuisData(\"./data/frames.json\")\n",
    "luis_data.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "luis_transformed_data = luis_data.transform_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_entities(app_id, app_version):\n",
    "    '''Création des entitées :\n",
    "        - 'dst_city'\n",
    "        - 'or_city'\n",
    "        - 'str_date'\n",
    "        - 'end_date'\n",
    "        - 'n_adult'\n",
    "        - 'n_children'\n",
    "        - 'budget'\n",
    "        - 'seat'\n",
    "            \n",
    "            Parameters:\n",
    "                    app_id : ID de l\\'application \n",
    "                    app_version : version de l\\'application LUIS\n",
    "            Returns:\n",
    "                    None\n",
    "    '''\n",
    "    dst_city_entity_id = client.model.add_entity(\n",
    "        app_id, app_version, name=\"dst_city\")\n",
    "    print(\"destinationEntityId {} added.\".format(dst_city_entity_id))\n",
    "\n",
    "    or_city_entity_id = client.model.add_entity(\n",
    "        app_id, app_version, name=\"or_city\")\n",
    "    print(\"classEntityId {} added.\".format(or_city_entity_id))\n",
    "\n",
    "    str_date_entity_id = client.model.add_entity(\n",
    "        app_id, app_version, name=\"str_date\")\n",
    "    print(\"flightEntityId {} added.\".format(str_date_entity_id))\n",
    "\n",
    "    end_date_entity_id = client.model.add_entity(\n",
    "        app_id, app_version, name=\"end_date\")\n",
    "    print(\"flightEntityId {} added.\".format(end_date_entity_id))\n",
    "\n",
    "    n_adult_entity_id = client.model.add_entity(\n",
    "        app_id, app_version, name=\"n_adult\")\n",
    "    print(\"flightEntityId {} added.\".format(n_adult_entity_id))\n",
    "\n",
    "    n_children_entity_id = client.model.add_entity(\n",
    "        app_id, app_version, name=\"n_children\")\n",
    "    print(\"flightEntityId {} added.\".format(n_children_entity_id))\n",
    "\n",
    "    budget_entity_id = client.model.add_entity(\n",
    "        app_id, app_version, name=\"budget\")\n",
    "    print(\"flightEntityId {} added.\".format(budget_entity_id))\n",
    "\n",
    "    seat_entity_id = client.model.add_entity(\n",
    "        app_id, app_version, name=\"seat\")\n",
    "    print(\"flightEntityId {} added.\".format(seat_entity_id))\n",
    "\n",
    "def add_intents(app_id, app_version):\n",
    "    '''\n",
    "        Création des intention :\n",
    "            - BookFlight\n",
    "            Parameters:\n",
    "                    app_id : ID de l\\'application \n",
    "                    app_version : version de l\\'application LUIS\n",
    "            Returns:\n",
    "                    None\n",
    "    '''\n",
    "    intent_id = client.model.add_intent(app_id, app_version, \"BookFlight\")\n",
    "    print(\"Intent BookFlight {} added.\".format(intent_id))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_entities(app_id=app_id,app_version=app_version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Intent BookFlight 024458ec-3211-40d5-a41f-85f83f07cd04 added.\n"
     ]
    }
   ],
   "source": [
    "add_intents(app_id=app_id,app_version=app_version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': \"i'd like to book a trip to atlantis from caprica on saturday, august 13, 2016 for 8 adults. i have a tight budget of 1700.\",\n",
       " 'intent_name': 'BookFlight',\n",
       " 'entity_labels': [{'entity_name': 'dst_city',\n",
       "   'start_char_index': 27,\n",
       "   'end_char_index': 35},\n",
       "  {'entity_name': 'or_city', 'start_char_index': 41, 'end_char_index': 48},\n",
       "  {'entity_name': 'str_date', 'start_char_index': 52, 'end_char_index': 77},\n",
       "  {'entity_name': 'budget', 'start_char_index': 117, 'end_char_index': 121}]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "luis_transformed_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = luis_transformed_data[:-99]\n",
    "test_data = luis_transformed_data[-99:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0, len(train_data), 100):\n",
    "    j = (i + 100)\n",
    "    if j > len(luis_transformed_data):\n",
    "        j = len(luis_transformed_data)\n",
    "    # On ajoute 100 par 100 car les requêtes sont bloqué au dela de 100\n",
    "    client.examples.batch(app_id,app_version,train_data[i:j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting 10 seconds for training to complete...\n",
      "Waiting 10 seconds for training to complete...\n",
      "Waiting 10 seconds for training to complete...\n",
      "trained\n"
     ]
    }
   ],
   "source": [
    "client.train.train_version(app_id, app_version)\n",
    "waiting = True\n",
    "while waiting:\n",
    "    info = client.train.get_status(app_id, app_version)\n",
    "\n",
    "    # get_status returns a list of training statuses, one for each model. Loop through them and make sure all are done.\n",
    "    waiting = any(map(lambda x: 'Queued' == x.details.status or 'InProgress' == x.details.status, info))\n",
    "    if waiting:\n",
    "        print (\"Waiting 10 seconds for training to complete...\")\n",
    "        time.sleep(10)\n",
    "    else: \n",
    "        print (\"trained\")\n",
    "        waiting = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.apps.update_settings(app_id, is_public=True)\n",
    "\n",
    "responseEndpointInfo = client.apps.publish(app_id, app_version, is_staging=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "endpoint = responseEndpointInfo.endpoint_url + \\\n",
    "\"?subscription-key=\" + AUTHORING_KEY_LUIS + \"&q=\"\n",
    "print(\"Your app is published. You can now go to test it on\\n{}\".format(endpoint))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test de prédiction du modèle déployé"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azure.cognitiveservices.language.luis.runtime import LUISRuntimeClient\n",
    "\n",
    "PREDICTION_KEY = os.environ[\"RUNTIME_KEY_LUIS\"]\n",
    "PREDICTION_ENDPOINT = os.environ[\"RUNTIME_ENDPOINT_LUIS\"]\n",
    "clientRuntime = LUISRuntimeClient(PREDICTION_ENDPOINT, CognitiveServicesCredentials(PREDICTION_KEY))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "request = { \"query\" : \"book a flight from paris to berlin for 2020 between 21th september and 2nd october\" }\n",
    "\n",
    "# Note be sure to specify, using the slot_name parameter, whether your application is in staging or production.\n",
    "response = clientRuntime.prediction.get_slot_prediction(app_id=app_id, slot_name=\"production\", prediction_request=request)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'or_city': ['paris'],\n",
       " 'dst_city': ['berlin'],\n",
       " 'str_date': ['september'],\n",
       " 'end_date': ['2nd october']}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.prediction.entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BookFlight\n"
     ]
    }
   ],
   "source": [
    "for intent in response.prediction.intents.keys():\n",
    "    print(intent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top intent: BookFlight\n",
      "Intents: \n",
      "\t\"BookFlight\"\n",
      "Entities: {'or_city': ['paris'], 'dst_city': ['berlin'], 'str_date': ['september'], 'end_date': ['2nd october']}\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "print(\"Top intent: {}\".format(response.prediction.top_intent))\n",
    "print(\"Intents: \")\n",
    "\n",
    "for intent in response.prediction.intents:\n",
    "\tprint(\"\\t{}\".format (json.dumps (intent)))\n",
    "\n",
    "print(\"Entities: {}\".format (response.prediction.entities))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "\n",
    "def predict(query):\n",
    "    response = requests.get(endpoint+query)\n",
    "    response = response.json()\n",
    "    print(\"Top intent: {}\".format(response[\"topScoringIntent\"][\"intent\"]))\n",
    "    print(\"Entities: \")\n",
    "\n",
    "    for entity in response[\"entities\"]:\n",
    "        data = entity\n",
    "        key = data[\"type\"]\n",
    "        value = data[\"entity\"]\n",
    "        print(\"{\\'\",key,\"\\' : \\'\",value,\"\\'}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = \"we're from punta cana. i'd like to bring them while they're still out of school, sometime between wednesday september 14 and tuesday october 4th would be great.\"\n",
    "# test = \"book a flight from paris to berlin for 2020 between 21th september and 2nd october\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top intent: BookFlight\n",
      "Entities: \n",
      "{' end_date ' : ' october 4th '}\n",
      "{' or_city ' : ' punta cana . '}\n",
      "{' str_date ' : ' wednesday september 14 '}\n"
     ]
    }
   ],
   "source": [
    "predict(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test de l'object LuiApp permettant de simplifier les interactions avec LUIS Azure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from luis_module.luis_utility import LuisApp\n",
    "import os\n",
    "\n",
    "AUTHORING_KEY_LUIS = os.environ[\"AUTHORING_KEY_LUIS\"]\n",
    "AUTHORING_ENDPOINT_LUIS = os.environ[\"AUTHORING_ENDPOINT_LUIS\"]\n",
    "\n",
    "luis_app = LuisApp(AUTHORING_ENDPOINT_LUIS,AUTHORING_KEY_LUIS)\n",
    "luis_app.get_app(\"8958f321-de6e-4cee-bf26-e2468b5f381f\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'end_date': 'october 4th',\n",
       " 'or_city': 'punta cana .',\n",
       " 'str_date': 'wednesday september 14'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "luis_app.predict(\"we're from punta cana. i'd like to bring them while they're still out of school, sometime between wednesday september 14 and tuesday october 4th would be great.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from luis_module.get_data import LuisData \n",
    "\n",
    "luis_data = LuisData(\"./data/frames.json\")\n",
    "luis_data.load_data()\n",
    "\n",
    "luis_transformed_data = luis_data.transform_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = luis_transformed_data[:-99]\n",
    "test_data = luis_transformed_data[-99:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_score, score = luis_app.evaluate(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy score : 0.61\n"
     ]
    }
   ],
   "source": [
    "print(\"accuracy score :\",round(score,2)) "
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "ac59ebe37160ed0dfa835113d9b8498d9f09ceb179beaac4002f036b9467c963"
  },
  "kernelspec": {
   "display_name": "Python 3.9.2 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}