api/api_usage.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup\n",
    "1. Run YAP API server\n",
    "1. `pip install -r requirements.txt`\n",
    "1. Run NEMO API server `uvicorn api_main:app --port 8090`\n",
    "1. Have a look at the swagger OpenAPI documentation by opening http://localhost:8090/docs in your browser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'message': 'Please specify command in URL path in a POST request and provide some input text in the request body.',\n",
       " 'available_commands': ['run_ncrf_model',\n",
       "  'multi_align_hybrid',\n",
       "  'multi_to_single',\n",
       "  'morph_yap',\n",
       "  'morph_hybrid',\n",
       "  'morph_hybrid_align_tokens']}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "requests.get('http://localhost:8090/').json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = [\n",
    "    'מלך השערים במונדיאל 2006 היה מירוסלב קלוזה.',\n",
    "    \"עשרות אנשים מגיעים מתאילנד לישראל.\\nתופעה זו התבררה אתמול בוועדת העבודה והרווחה של הכנסת.\"\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calling the API\n",
    "1. All endpoints are HTTP POST\n",
    "1. Request body contains a json with Hebrew `sentences` and optional `tokenized` flag for signaling whether they are tokenized or not.\n",
    "1. Request URL may include further optional parameters for choosing models/scenarios (in all but `run_ner_model` there is no need to touch these)\n",
    "1. Response is in the response body as a JSON "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run NER Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers =   {   'accept': 'application/json', \n",
    "                'Content-Type': 'application/json' }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.58 ms, sys: 1.88 ms, total: 5.46 ms\n",
      "Wall time: 30.8 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([('עשרות', 'O'),\n",
       "  ('אנשים', 'O'),\n",
       "  ('מגיעים', 'O'),\n",
       "  ('מתאילנד', 'O^S-GPE'),\n",
       "  ('לישראל', 'O^S-GPE'),\n",
       "  ('.', 'O')],\n",
       " [('תופעה', 'O'),\n",
       "  ('זו', 'O'),\n",
       "  ('התבררה', 'O'),\n",
       "  ('אתמול', 'O'),\n",
       "  ('בוועדת', 'O^B-ORG'),\n",
       "  ('העבודה', 'I-ORG^I-ORG'),\n",
       "  ('והרווחה', 'I-ORG^I-ORG^E-ORG'),\n",
       "  ('של', 'O'),\n",
       "  ('הכנסת', 'B-ORG^E-ORG'),\n",
       "  ('.', 'O')])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "params = {\n",
    "            'model_name': 'token-multi', # / 'token-single' / 'morph' \n",
    "}\n",
    "payload = { 'sentences': texts[1],\n",
    "            #'tokenized': True,\n",
    "          }\n",
    "\n",
    "res = requests.post('http://localhost:8090/run_ncrf_model', params=params, json=payload, headers=headers).json()\n",
    "list(zip(res[0]['tokenized_text'], res[0]['ncrf_preds'])), list(zip(res[1]['tokenized_text'], res[1]['ncrf_preds']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### multi_to_single (get token-single predictions using a token-multi model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.17 ms, sys: 2.51 ms, total: 4.67 ms\n",
      "Wall time: 28.2 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'text': 'עשרות אנשים מגיעים מתאילנד לישראל.',\n",
       "  'ents': {'token': {'nemo_multi_align_token': [{'text': 'מתאילנד',\n",
       "      'label': 'GPE',\n",
       "      'start': 3,\n",
       "      'end': 4},\n",
       "     {'text': 'לישראל', 'label': 'GPE', 'start': 4, 'end': 5}]}},\n",
       "  'tokens': [{'text': 'עשרות', 'nemo_multi_align_token': 'O'},\n",
       "   {'text': 'אנשים', 'nemo_multi_align_token': 'O'},\n",
       "   {'text': 'מגיעים', 'nemo_multi_align_token': 'O'},\n",
       "   {'text': 'מתאילנד', 'nemo_multi_align_token': 'S-GPE'},\n",
       "   {'text': 'לישראל', 'nemo_multi_align_token': 'S-GPE'},\n",
       "   {'text': '.', 'nemo_multi_align_token': 'O'}]},\n",
       " {'text': 'תופעה זו התבררה אתמול בוועדת העבודה והרווחה של הכנסת.',\n",
       "  'ents': {'token': {'nemo_multi_align_token': [{'text': 'בוועדת העבודה והרווחה',\n",
       "      'label': 'ORG',\n",
       "      'start': 4,\n",
       "      'end': 7},\n",
       "     {'text': 'הכנסת', 'label': 'ORG', 'start': 8, 'end': 9}]}},\n",
       "  'tokens': [{'text': 'תופעה', 'nemo_multi_align_token': 'O'},\n",
       "   {'text': 'זו', 'nemo_multi_align_token': 'O'},\n",
       "   {'text': 'התבררה', 'nemo_multi_align_token': 'O'},\n",
       "   {'text': 'אתמול', 'nemo_multi_align_token': 'O'},\n",
       "   {'text': 'בוועדת', 'nemo_multi_align_token': 'B-ORG'},\n",
       "   {'text': 'העבודה', 'nemo_multi_align_token': 'I-ORG'},\n",
       "   {'text': 'והרווחה', 'nemo_multi_align_token': 'E-ORG'},\n",
       "   {'text': 'של', 'nemo_multi_align_token': 'O'},\n",
       "   {'text': 'הכנסת', 'nemo_multi_align_token': 'S-ORG'},\n",
       "   {'text': '.', 'nemo_multi_align_token': 'O'}]}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "params = {\n",
    "            #'multi_model_name': 'token-multi',\n",
    "\n",
    "}\n",
    "payload = { 'sentences': texts[1],\n",
    "            #'tokenized': True,\n",
    "          }\n",
    "\n",
    "res = requests.post('http://localhost:8090/multi_to_single', params=params, json=payload, headers=headers).json()\n",
    "res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MD + NER"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### morph_yap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.91 ms, sys: 2.69 ms, total: 5.6 ms\n",
      "Wall time: 176 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'text': 'עשרות אנשים מגיעים מתאילנד לישראל.',\n",
       " 'ents': {'morph': {'nemo_morph': [{'text': 'תאילנד',\n",
       "     'label': 'GPE',\n",
       "     'start': 4,\n",
       "     'end': 5},\n",
       "    {'text': 'ישראל', 'label': 'GPE', 'start': 6, 'end': 7}]}},\n",
       " 'tokens': [{'text': 'עשרות',\n",
       "   'morphs': [{'form': 'עשרות',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': 'עשר',\n",
       "     'pos': 'CDT',\n",
       "     'feats': 'gen=F|num=P'}]},\n",
       "  {'text': 'אנשים',\n",
       "   'morphs': [{'form': 'אנשים',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': 'איש',\n",
       "     'pos': 'NN',\n",
       "     'feats': 'gen=M|num=P'}]},\n",
       "  {'text': 'מגיעים',\n",
       "   'morphs': [{'form': 'מגיעים',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': 'הגיע',\n",
       "     'pos': 'BN',\n",
       "     'feats': 'gen=M|num=P|per=A'}]},\n",
       "  {'text': 'מתאילנד',\n",
       "   'morphs': [{'form': 'מ',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': 'מ',\n",
       "     'pos': 'PREPOSITION',\n",
       "     'feats': '_'},\n",
       "    {'form': 'תאילנד',\n",
       "     'nemo_morph': 'S-GPE',\n",
       "     'lemma': 'תאילנד',\n",
       "     'pos': 'NNP',\n",
       "     'feats': 'gen=F|num=S'}]},\n",
       "  {'text': 'לישראל',\n",
       "   'morphs': [{'form': 'ל',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': 'ל',\n",
       "     'pos': 'PREPOSITION',\n",
       "     'feats': '_'},\n",
       "    {'form': 'ישראל',\n",
       "     'nemo_morph': 'S-GPE',\n",
       "     'lemma': 'ישראל',\n",
       "     'pos': 'NNP',\n",
       "     'feats': 'gen=F|num=S'}]},\n",
       "  {'text': '.',\n",
       "   'morphs': [{'form': '.',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': '_',\n",
       "     'pos': 'yyDOT',\n",
       "     'feats': '_'}]}]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "params = {\n",
    "            #'morph_model_name': 'morph',\n",
    "}\n",
    "payload = { 'sentences': texts[1],\n",
    "            #'tokenized': True,\n",
    "          }\n",
    "\n",
    "res = requests.post('http://localhost:8090/morph_yap', params=params, json=payload, headers=headers).json()\n",
    "res[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### multi_align_hybrid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.38 ms, sys: 793 µs, total: 6.17 ms\n",
      "Wall time: 122 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'text': 'עשרות אנשים מגיעים מתאילנד לישראל.',\n",
       " 'ents': {'morph': {'nemo_multi_align_morph': [{'text': 'תאילנד',\n",
       "     'label': 'GPE',\n",
       "     'start': 4,\n",
       "     'end': 5},\n",
       "    {'text': 'ישראל', 'label': 'GPE', 'start': 6, 'end': 7}]}},\n",
       " 'tokens': [{'text': 'עשרות',\n",
       "   'morphs': [{'form': 'עשרות',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': 'עשר',\n",
       "     'pos': 'CDT',\n",
       "     'feats': 'gen=F|num=P'}]},\n",
       "  {'text': 'אנשים',\n",
       "   'morphs': [{'form': 'אנשים',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': 'איש',\n",
       "     'pos': 'NN',\n",
       "     'feats': 'gen=M|num=P'}]},\n",
       "  {'text': 'מגיעים',\n",
       "   'morphs': [{'form': 'מגיעים',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': 'הגיע',\n",
       "     'pos': 'BN',\n",
       "     'feats': 'gen=M|num=P|per=A'}]},\n",
       "  {'text': 'מתאילנד',\n",
       "   'morphs': [{'form': 'מ',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': 'מ',\n",
       "     'pos': 'PREPOSITION',\n",
       "     'feats': '_'},\n",
       "    {'form': 'תאילנד',\n",
       "     'nemo_multi_align_morph': 'S-GPE',\n",
       "     'lemma': 'תאילנד',\n",
       "     'pos': 'NNP',\n",
       "     'feats': 'gen=F|num=S'}]},\n",
       "  {'text': 'לישראל',\n",
       "   'morphs': [{'form': 'ל',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': 'ל',\n",
       "     'pos': 'PREPOSITION',\n",
       "     'feats': '_'},\n",
       "    {'form': 'ישראל',\n",
       "     'nemo_multi_align_morph': 'S-GPE',\n",
       "     'lemma': 'ישראל',\n",
       "     'pos': 'NNP',\n",
       "     'feats': 'gen=F|num=S'}]},\n",
       "  {'text': '.',\n",
       "   'morphs': [{'form': '.',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': '_',\n",
       "     'pos': 'yyDOT',\n",
       "     'feats': '_'}]}]}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "payload = { 'sentences': texts[1],\n",
    "            #'tokenized': True,\n",
    "          }\n",
    "\n",
    "res = requests.post('http://localhost:8090/multi_align_hybrid', params=params, json=payload, headers=headers).json()\n",
    "res[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### morph_hybrid\n",
    "The following models got the best results in our experiments.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.49 ms, sys: 1.5 ms, total: 5.99 ms\n",
      "Wall time: 117 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'text': 'עשרות אנשים מגיעים מתאילנד לישראל.',\n",
       " 'ents': {'morph': {'nemo_morph': [{'text': 'תאילנד',\n",
       "     'label': 'GPE',\n",
       "     'start': 4,\n",
       "     'end': 5},\n",
       "    {'text': 'ישראל', 'label': 'GPE', 'start': 6, 'end': 7}]}},\n",
       " 'tokens': [{'text': 'עשרות',\n",
       "   'morphs': [{'form': 'עשרות',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': 'עשר',\n",
       "     'pos': 'CDT',\n",
       "     'feats': 'gen=F|num=P'}]},\n",
       "  {'text': 'אנשים',\n",
       "   'morphs': [{'form': 'אנשים',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': 'איש',\n",
       "     'pos': 'NN',\n",
       "     'feats': 'gen=M|num=P'}]},\n",
       "  {'text': 'מגיעים',\n",
       "   'morphs': [{'form': 'מגיעים',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': 'הגיע',\n",
       "     'pos': 'BN',\n",
       "     'feats': 'gen=M|num=P|per=A'}]},\n",
       "  {'text': 'מתאילנד',\n",
       "   'morphs': [{'form': 'מ',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': 'מ',\n",
       "     'pos': 'PREPOSITION',\n",
       "     'feats': '_'},\n",
       "    {'form': 'תאילנד',\n",
       "     'nemo_morph': 'S-GPE',\n",
       "     'lemma': 'תאילנד',\n",
       "     'pos': 'NNP',\n",
       "     'feats': 'gen=F|num=S'}]},\n",
       "  {'text': 'לישראל',\n",
       "   'morphs': [{'form': 'ל',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': 'ל',\n",
       "     'pos': 'PREPOSITION',\n",
       "     'feats': '_'},\n",
       "    {'form': 'ישראל',\n",
       "     'nemo_morph': 'S-GPE',\n",
       "     'lemma': 'ישראל',\n",
       "     'pos': 'NNP',\n",
       "     'feats': 'gen=F|num=S'}]},\n",
       "  {'text': '.',\n",
       "   'morphs': [{'form': '.',\n",
       "     'nemo_morph': 'O',\n",
       "     'lemma': '_',\n",
       "     'pos': 'yyDOT',\n",
       "     'feats': '_'}]}]}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "params = {\n",
    "            #'multi_model_name': 'token-multi',\n",
    "            #'morph_model_name': 'morph',\n",
    "            # 'align_tokens': False,\n",
    "}\n",
    "payload = { 'sentences': texts[1],\n",
    "            #'tokenized': False,\n",
    "          }\n",
    "\n",
    "res = requests.post('http://localhost:8090/morph_hybrid', params=params, json=payload, headers=headers).json()\n",
    "res[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### morph_hybrid_align_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.44 ms, sys: 0 ns, total: 6.44 ms\n",
      "Wall time: 167 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'text': 'עשרות אנשים מגיעים מתאילנד לישראל.',\n",
       " 'ents': {'morph': {'nemo_morph': [{'text': 'תאילנד',\n",
       "     'label': 'GPE',\n",
       "     'start': 4,\n",
       "     'end': 5},\n",
       "    {'text': 'ישראל', 'label': 'GPE', 'start': 6, 'end': 7}],\n",
       "   'nemo_multi_align_morph': [{'text': 'תאילנד',\n",
       "     'label': 'GPE',\n",
       "     'start': 4,\n",
       "     'end': 5},\n",
       "    {'text': 'ישראל', 'label': 'GPE', 'start': 6, 'end': 7}]},\n",
       "  'token': {'nemo_multi_align_token': [{'text': 'מתאילנד',\n",
       "     'label': 'GPE',\n",
       "     'start': 3,\n",
       "     'end': 4},\n",
       "    {'text': 'לישראל', 'label': 'GPE', 'start': 4, 'end': 5}],\n",
       "   'nemo_morph_align_token': [{'text': 'מתאילנד',\n",
       "     'label': 'GPE',\n",
       "     'start': 3,\n",
       "     'end': 4},\n",
       "    {'text': 'לישראל', 'label': 'GPE', 'start': 4, 'end': 5}]}},\n",
       " 'tokens': [{'text': 'עשרות',\n",
       "   'nemo_multi': 'O',\n",
       "   'nemo_multi_align_token': 'O',\n",
       "   'nemo_morph_align_token': 'O',\n",
       "   'morphs': [{'form': 'עשרות',\n",
       "     'nemo_morph': 'O',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': 'עשר',\n",
       "     'pos': 'CDT',\n",
       "     'feats': 'gen=F|num=P'}]},\n",
       "  {'text': 'אנשים',\n",
       "   'nemo_multi': 'O',\n",
       "   'nemo_multi_align_token': 'O',\n",
       "   'nemo_morph_align_token': 'O',\n",
       "   'morphs': [{'form': 'אנשים',\n",
       "     'nemo_morph': 'O',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': 'איש',\n",
       "     'pos': 'NN',\n",
       "     'feats': 'gen=M|num=P'}]},\n",
       "  {'text': 'מגיעים',\n",
       "   'nemo_multi': 'O',\n",
       "   'nemo_multi_align_token': 'O',\n",
       "   'nemo_morph_align_token': 'O',\n",
       "   'morphs': [{'form': 'מגיעים',\n",
       "     'nemo_morph': 'O',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': 'הגיע',\n",
       "     'pos': 'BN',\n",
       "     'feats': 'gen=M|num=P|per=A'}]},\n",
       "  {'text': 'מתאילנד',\n",
       "   'nemo_multi': 'O^S-GPE',\n",
       "   'nemo_multi_align_token': 'S-GPE',\n",
       "   'nemo_morph_align_token': 'S-GPE',\n",
       "   'morphs': [{'form': 'מ',\n",
       "     'nemo_morph': 'O',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': 'מ',\n",
       "     'pos': 'PREPOSITION',\n",
       "     'feats': '_'},\n",
       "    {'form': 'תאילנד',\n",
       "     'nemo_morph': 'S-GPE',\n",
       "     'nemo_multi_align_morph': 'S-GPE',\n",
       "     'lemma': 'תאילנד',\n",
       "     'pos': 'NNP',\n",
       "     'feats': 'gen=F|num=S'}]},\n",
       "  {'text': 'לישראל',\n",
       "   'nemo_multi': 'O^S-GPE',\n",
       "   'nemo_multi_align_token': 'S-GPE',\n",
       "   'nemo_morph_align_token': 'S-GPE',\n",
       "   'morphs': [{'form': 'ל',\n",
       "     'nemo_morph': 'O',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': 'ל',\n",
       "     'pos': 'PREPOSITION',\n",
       "     'feats': '_'},\n",
       "    {'form': 'ישראל',\n",
       "     'nemo_morph': 'S-GPE',\n",
       "     'nemo_multi_align_morph': 'S-GPE',\n",
       "     'lemma': 'ישראל',\n",
       "     'pos': 'NNP',\n",
       "     'feats': 'gen=F|num=S'}]},\n",
       "  {'text': '.',\n",
       "   'nemo_multi': 'O',\n",
       "   'nemo_multi_align_token': 'O',\n",
       "   'nemo_morph_align_token': 'O',\n",
       "   'morphs': [{'form': '.',\n",
       "     'nemo_morph': 'O',\n",
       "     'nemo_multi_align_morph': 'O',\n",
       "     'lemma': '_',\n",
       "     'pos': 'yyDOT',\n",
       "     'feats': '_'}]}],\n",
       " 'ma_lattice': '0\\t1\\tעשרות\\tעשר\\tCDT\\tCDT\\tgen=F|num=P\\t1\\n0\\t1\\tעשרות\\tעשר\\tCD\\tCD\\tgen=F|num=P\\t1\\n1\\t2\\tאנשים\\tהנשים\\tVB\\tVB\\tgen=F|gen=M|num=S|per=1|tense=FUTURE\\t2\\n1\\t2\\tאנשים\\tאיש\\tNN\\tNN\\tgen=M|num=P\\t2\\n2\\t3\\tמגיעים\\tהגיע\\tVB\\tVB\\tgen=M|num=P|per=A|tense=BEINONI\\t3\\n2\\t3\\tמגיעים\\tהגיע\\tBN\\tBN\\tgen=M|num=P|per=A\\t3\\n3\\t4\\tמ\\tמ\\tPREPOSITION\\tPREPOSITION\\t_\\t4\\n3\\t5\\tמתאילנד\\tמתאילנד\\tNNP\\tNNP\\tgen=M|num=S\\t4\\n3\\t5\\tמתאילנד\\tמתאילנד\\tNN\\tNN\\tgen=M|num=P|num=S\\t4\\n3\\t5\\tמתאילנד\\tמתאילנד\\tNN\\tNN\\tgen=M|num=S\\t4\\n3\\t5\\tמתאילנד\\tמתאילנד\\tNNP\\tNNP\\tgen=F|num=S\\t4\\n3\\t5\\tמתאילנד\\tמתאילנד\\tNNP\\tNNP\\tgen=F|gen=M|num=S\\t4\\n3\\t5\\tמתאילנד\\tמתאילנד\\tNNP\\tNNP\\t_\\t4\\n3\\t5\\tמתאילנד\\tמתאילנד\\tNN\\tNN\\tgen=M|num=P\\t4\\n3\\t5\\tמתאילנד\\tמתאילנד\\tNN\\tNN\\tgen=F|num=S\\t4\\n3\\t5\\tמתאילנד\\tמתאילנד\\tNN\\tNN\\tgen=F|num=P\\t4\\n4\\t5\\tתאילנד\\tתאילנד\\tNNP\\tNNP\\tgen=F|num=S\\t4\\n5\\t6\\tל\\tל\\tPREPOSITION\\tPREPOSITION\\t_\\t5\\n5\\t8\\tלישראל\\tלישראל\\tNNP\\tNNP\\tgen=M|num=S\\t5\\n5\\t8\\tלישראל\\tלישראל\\tNN\\tNN\\tgen=M|num=P|num=S\\t5\\n5\\t8\\tלישראל\\tלישראל\\tNN\\tNN\\tgen=M|num=S\\t5\\n5\\t8\\tלישראל\\tלישראל\\tNNP\\tNNP\\tgen=F|num=S\\t5\\n5\\t8\\tלישראל\\tלישראל\\tNNP\\tNNP\\tgen=F|gen=M|num=S\\t5\\n5\\t8\\tלישראל\\tלישראל\\tNNP\\tNNP\\t_\\t5\\n5\\t8\\tלישראל\\tלישראל\\tNN\\tNN\\tgen=M|num=P\\t5\\n5\\t8\\tלישראל\\tלישראל\\tNN\\tNN\\tgen=F|num=S\\t5\\n5\\t8\\tלישראל\\tלישראל\\tNN\\tNN\\tgen=F|num=P\\t5\\n6\\t7\\tה\\tה\\tDEF\\tDEF\\t_\\t5\\n6\\t8\\tישראל\\tישראל\\tNNP\\tNNP\\tgen=M|num=S\\t5\\n6\\t8\\tישראל\\tישראל\\tNNP\\tNNP\\tgen=F|num=S\\t5\\n7\\t8\\tישראל\\tישראל\\tNNP\\tNNP\\tgen=F|num=S\\t5\\n7\\t8\\tישראל\\tישראל\\tNNP\\tNNP\\tgen=M|num=S\\t5\\n8\\t9\\t.\\t_\\tyyDOT\\tyyDOT\\t_\\t6',\n",
       " 'pruned_lattice': '0\\t1\\tעשרות\\tעשר\\tCDT\\tCDT\\tgen=F|num=P\\t1\\n0\\t1\\tעשרות\\tעשר\\tCD\\tCD\\tgen=F|num=P\\t1\\n1\\t2\\tאנשים\\tהנשים\\tVB\\tVB\\tgen=F|gen=M|num=S|per=1|tense=FUTURE\\t2\\n1\\t2\\tאנשים\\tאיש\\tNN\\tNN\\tgen=M|num=P\\t2\\n2\\t3\\tמגיעים\\tהגיע\\tVB\\tVB\\tgen=M|num=P|per=A|tense=BEINONI\\t3\\n2\\t3\\tמגיעים\\tהגיע\\tBN\\tBN\\tgen=M|num=P|per=A\\t3\\n3\\t4\\tמ\\tמ\\tPREPOSITION\\tPREPOSITION\\t_\\t4\\n4\\t5\\tתאילנד\\tתאילנד\\tNNP\\tNNP\\tgen=F|num=S\\t4\\n5\\t6\\tל\\tל\\tPREPOSITION\\tPREPOSITION\\t_\\t5\\n6\\t8\\tישראל\\tישראל\\tNNP\\tNNP\\tgen=M|num=S\\t5\\n6\\t8\\tישראל\\tישראל\\tNNP\\tNNP\\tgen=F|num=S\\t5\\n8\\t9\\t.\\t_\\tyyDOT\\tyyDOT\\t_\\t6',\n",
       " 'md_lattice': '0\\t1\\tעשרות\\tעשר\\tCDT\\tCDT\\tgen=F|num=P\\t1\\n1\\t2\\tאנשים\\tאיש\\tNN\\tNN\\tgen=M|num=P\\t2\\n2\\t3\\tמגיעים\\tהגיע\\tBN\\tBN\\tgen=M|num=P|per=A\\t3\\n3\\t4\\tמ\\tמ\\tPREPOSITION\\tPREPOSITION\\t_\\t4\\n4\\t5\\tתאילנד\\tתאילנד\\tNNP\\tNNP\\tgen=F|num=S\\t4\\n5\\t6\\tל\\tל\\tPREPOSITION\\tPREPOSITION\\t_\\t5\\n6\\t7\\tישראל\\tישראל\\tNNP\\tNNP\\tgen=F|num=S\\t5\\n7\\t8\\t.\\t_\\tyyDOT\\tyyDOT\\t_\\t6'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "params = {\n",
    "            'verbose': 1,\n",
    "            'include_yap_outputs': True,\n",
    "}\n",
    "payload = { 'sentences': texts[1],\n",
    "            #'tokenized': False,\n",
    "          }\n",
    "\n",
    "res = requests.post('http://localhost:8090/morph_hybrid_align_tokens', params=params, json=payload, headers=headers).json()\n",
    "res[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Display and view ents\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from notebook_utils import *\n",
    "e = EntityRenderer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "payload = { 'sentences': '\\n'.join(texts+['גנן גידל דגן בגן']),}\n",
    "params = {\n",
    "            'verbose': 0,\n",
    "            #'align_tokens': False,\n",
    "}\n",
    "res = requests.post('http://localhost:8090/morph_hybrid', params=params, json=payload, headers=headers).json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'morph': {'nemo_morph': [{'text': 'גן',\n",
       "    'label': 'LOC',\n",
       "    'start': 4,\n",
       "    'end': 5}]}}"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res[-1]['ents']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5; direction: rtl\">מלך ה שערים ב מונדיאל 2006 היה\n",
       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em\">\n",
       "    מירוסלב קלוזה\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-right: 0.5rem\">PER</span>\n",
       "</mark>\n",
       ".</div>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5; direction: rtl\">עשרות אנשים מגיעים מ\n",
       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em\">\n",
       "    תאילנד\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-right: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       "ל\n",
       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em\">\n",
       "    ישראל\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-right: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       ".</div>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5; direction: rtl\">תופעה זו התבררה אתמול ב\n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em\">\n",
       "    וועדת ה עבודה ו ה רווחה\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-right: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       "של\n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em\">\n",
       "    ה כנסת\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-right: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       ".</div>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5; direction: rtl\">גנן גידל דגן ב\n",
       "<mark class=\"entity\" style=\"background: #ff9561; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em\">\n",
       "    גן\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-right: 0.5rem\">LOC</span>\n",
       "</mark>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "display(HTML(e.render(res, 'morph', 'nemo_morph')))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>level</th>\n",
       "      <th>scenario</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>מירוסלב קלוזה</td>\n",
       "      <td>PER</td>\n",
       "      <td>morph</td>\n",
       "      <td>nemo_morph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>תאילנד</td>\n",
       "      <td>GPE</td>\n",
       "      <td>morph</td>\n",
       "      <td>nemo_morph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>ישראל</td>\n",
       "      <td>GPE</td>\n",
       "      <td>morph</td>\n",
       "      <td>nemo_morph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>וועדת ה עבודה ו ה רווחה</td>\n",
       "      <td>ORG</td>\n",
       "      <td>morph</td>\n",
       "      <td>nemo_morph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>ה כנסת</td>\n",
       "      <td>ORG</td>\n",
       "      <td>morph</td>\n",
       "      <td>nemo_morph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>גן</td>\n",
       "      <td>LOC</td>\n",
       "      <td>morph</td>\n",
       "      <td>nemo_morph</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sent_id                     text label  level    scenario\n",
       "0        0            מירוסלב קלוזה   PER  morph  nemo_morph\n",
       "1        1                   תאילנד   GPE  morph  nemo_morph\n",
       "2        1                    ישראל   GPE  morph  nemo_morph\n",
       "3        2  וועדת ה עבודה ו ה רווחה   ORG  morph  nemo_morph\n",
       "4        2                   ה כנסת   ORG  morph  nemo_morph\n",
       "5        3                       גן   LOC  morph  nemo_morph"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ents = ents_to_df(res)\n",
    "ents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}