Visual_Grounding_mp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Check Point: RPI path change /home/bobby/aida_copy/AIDA/M18_copy/data/rpi_ttl/RPI_TA1_PT002_r2 /home/bobby/aida_copy/AIDA/M18_copy/data/raw_files/RPI_TA1_E_asr/ltf_asr\n",
      "Check Point: Alireza path change: \n",
      " /home/bobby/aida_copy/AIDA/M18_copy/data/objdet_results/E/det_results_merged_34a_jpg.pkl \n",
      " /home/bobby/aida_copy/AIDA/M18_copy/data/objdet_results/E/det_results_merged_34b_kf.pkl \n",
      "\n",
      "CPU times: user 4.02 s, sys: 1.16 s, total: 5.19 s\n",
      "Wall time: 13.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pickle\n",
    "import os\n",
    "import json\n",
    "import sys\n",
    "import re\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.backends.backend_pdf\n",
    "from textwrap import wrap\n",
    "import utils\n",
    "from rdflib import Graph\n",
    "import copy\n",
    "import cv2\n",
    "from datetime import datetime\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1\"\n",
    "gpu_options = tf.GPUOptions(allow_growth=True)\n",
    "config = tf.ConfigProto(gpu_options=gpu_options,log_device_placement=True,allow_soft_placement=True)\n",
    "\n",
    "working_path = '/home/bobby/aida_copy/AIDA/M18_copy/data/'\n",
    "corpus_path = '/dvmm-filer2/projects/AIDA/data/ldc_eval_m18/LDC2019E42_AIDA_Phase_1_Evaluation_Source_Data_V1.0/'\n",
    "# corpus_path = '/dvmm-filer2/projects/AIDA/data/ldc_isi_dryrun3/dryrun-updated_tmp/dryrun/'\n",
    "#corpus_path = '/dvmm-filer2/projects/AIDA/data/ldc_isi_dryrun3/dryrun/'\n",
    "#corpus_path = '/dvmm-filer2/projects/AIDA/data/ldc_isi_dryrun/dryrun/' # D2\n",
    "\n",
    "# Checking change from AIF Data\n",
    "img_path = corpus_path\n",
    "kfrm_path = corpus_path + 'data/video_shot_boundaries/representative_frames'\n",
    "parent_child_tab = corpus_path + 'docs/parent_children.sorted.tab'\n",
    "kfrm_msb = corpus_path + 'docs/masterShotBoundary.msb'\n",
    "\n",
    "\n",
    "#rpi mention results in AIF\n",
    "p_f = 'PT002_r2' # E5\n",
    "p_f_share = 'E'\n",
    "p_f_rpi = 'PT002_r2'\n",
    "\n",
    "#grounding path\n",
    "grounding_dict_path = working_path + 'all_features/grounding_dict_'+p_f+'.pickle'\n",
    "#Todo: changed foldername\n",
    "RPI_AIF_path = working_path + 'rpi_ttl/RPI_TA1_'+p_f_rpi # 1/7th May\n",
    "#'RPI_TA1_20190504' 'rpi_ttl/RPI_dryrun_'+p_f\n",
    "ltf_path = corpus_path + 'data/ltf/ltf'\n",
    "# ltf_path = corpus_path + 'data/ltf/ltf'\n",
    "\n",
    "\n",
    "video_asr_path = working_path + 'raw_files/RPI_TA1_'+p_f_share+'_asr/ltf_asr'\n",
    "video_map_path = working_path + 'raw_files/RPI_TA1_'+p_f_share+'_asr/map_asr'\n",
    "print('Check Point: RPI path change',RPI_AIF_path,video_asr_path)\n",
    "\n",
    "\n",
    "# Checking change from Alireza\n",
    "det_results_path_img = working_path + 'objdet_results/'+p_f_share+'/det_results_merged_34a_jpg.pkl'\n",
    "# det_results_path_img = working_path + 'objdet_results/det_results_merged_32_jpg.pkl'\n",
    "det_results_path_kfrm = working_path + 'objdet_results/'+p_f_share+'/det_results_merged_34b_kf.pkl'\n",
    "# det_results_path_kfrm = working_path + 'objdet_results/det_results_merged_33_kf.pkl'\n",
    "print('Check Point: Alireza path change:','\\n',det_results_path_img,'\\n', det_results_path_kfrm,'\\n')\n",
    "\n",
    "\n",
    "def attn(e,v,e_bar,v_bar):\n",
    "    ## Inputs: local and global cap and img features ##\n",
    "    ## Output: Heatmap for each word, Global Heatmap, Attnded Vis features, Corr-vals\n",
    "    #e: ?xTxD, v: ?xNxD, e_bar: ?xD, v_bar: ?xN2xD\n",
    "    with tf.variable_scope('word_level_attn'):\n",
    "        #Eq.8\n",
    "        s = tf.einsum('bij,bkj->bik',e,v) #pair-wise ev^T: ?xTxN\n",
    "        \n",
    "        #s_bar = tf.nn.softmax(s, axis=1) #softmax on words\n",
    "\n",
    "        #Eq.9\n",
    "        #alpha = tf.nn.softmax(gamma_1*s_bar, axis=2) #softmax on regions\n",
    "        alpha = s\n",
    "        c = tf.einsum('bij,bjk->bik',alpha,v) #?xTxD attnded visual reps for each of T words\n",
    "        #Eq.10\n",
    "        c_norm = tf.nn.l2_normalize(c,axis=2)\n",
    "        e_norm = tf.nn.l2_normalize(e,axis=2)\n",
    "        R_i = tf.einsum('bik,bik->bi',c_norm,e_norm) #cosine for T (words,img_reps) for all pairs\n",
    "        R = tf.log(tf.pow(tf.reduce_sum(tf.exp(gamma_2*R_i),axis=1),1/gamma_2)) #? corrs\n",
    "        N0=int(np.sqrt(alpha.get_shape().as_list()[-1]))\n",
    "        heatmap_w = tf.reshape(alpha,[tf.shape(alpha)[0],tf.shape(alpha)[1],N0,N0])\n",
    "    \n",
    "    with tf.variable_scope('sen_level_attn'):\n",
    "        #Eq.8\n",
    "        s_s = tf.einsum('bj,bkj->bk',e_bar,v_bar) #pair-wise e_bar*v_bar^T: ?xN2\n",
    "        \n",
    "        #Eq.9\n",
    "        #alpha_s = tf.nn.softmax(gamma_1_s*s_s, axis=1) #softmax on regions\n",
    "        alpha_s = s_s\n",
    "        c_s = tf.einsum('bj,bjk->bk',alpha_s,v_bar) #?xD attnded visual reps for sen.\n",
    "        #Eq.10\n",
    "        c_s_norm = tf.nn.l2_normalize(c_s,axis=1)\n",
    "        e_bar_norm = tf.nn.l2_normalize(e_bar,axis=1)\n",
    "        R_s = tf.einsum('bk,bk->b',c_s_norm,e_bar_norm) #cosine for (sen,img_reps)\n",
    "        N0_g=int(np.sqrt(alpha_s.get_shape().as_list()[-1]))\n",
    "        heatmap_s = tf.reshape(alpha_s,[-1,N0_g,N0_g])\n",
    "        \n",
    "    return heatmap_w, heatmap_s, c_norm, c_s_norm, R_i, R, R_s\n",
    "def top_dict(kv_dict, num = 2):\n",
    "    k_list = list(kv_dict.keys())[:num]\n",
    "    subdict =  {k: kv_dict[k] for k in k_list}\n",
    "    print('\\ndict len:', len(kv_dict))\n",
    "    return subdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gathering keyframes info...\n",
      "Creating parent-child dictionaries...\n",
      "Creating path dictionary...\n",
      "Loading object detection results...\n",
      "Loading done.\n",
      "CPU times: user 1.34 s, sys: 408 ms, total: 1.75 s\n",
      "Wall time: 3.89 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print('Gathering keyframes info...')\n",
    "#preparing dicts\n",
    "id2dir_dict_kfrm = utils.create_dict_kfrm(kfrm_path, kfrm_msb, video_asr_path, video_map_path)\n",
    "id2time_dict_kfrm = utils.id2time(kfrm_msb)\n",
    "\n",
    "print('Creating parent-child dictionaries...')\n",
    "#generate parent-child dictionaries\n",
    "parent_dict, child_dict = utils.create_dict(parent_child_tab)\n",
    "\n",
    "# for i in child_dict.keys():\n",
    "#     if 'mp4' in i:\n",
    "#         print(i,child_dict[i])\n",
    "# print(top_dict(child_dict,5))\n",
    "\n",
    "print('Creating path dictionary...')\n",
    "#generate global id2path dictionary\n",
    "#ltf\n",
    "path_dict = utils.create_path_dict(os.path.join(corpus_path,'data/ltf/ltf/'))\n",
    "#jpg\n",
    "path_dict.update(utils.create_path_dict(os.path.join(corpus_path,'data/jpg/jpg/')))\n",
    "#mp4\n",
    "path_dict.update(utils.create_path_dict_kfrm(id2dir_dict_kfrm))\n",
    "\n",
    "#print('Loading AIF crawl dictionaries...')\n",
    "#with open(working_path + 'tmp/entity2mention_dict_r1.pickle', 'rb') as f:\n",
    "#    en2men = pickle.load(f)\n",
    "\n",
    "#with open(working_pathpath_pref + 'tmp/id2mentions_dict_r1.pickle', 'rb') as f:\n",
    "#    id2men = pickle.load(f)\n",
    "\n",
    "#loading a list of pronouns\n",
    "with open(working_path+'raw_files/pronouns.txt', 'r') as f:\n",
    "    lines = f.readlines()\n",
    "    lines = [line.strip('\\n') for line in lines]\n",
    "    pronouns = set(lines)\n",
    "    \n",
    "print('Loading object detection results...')\n",
    "#loading object detection results\n",
    "with open(det_results_path_img, 'rb') as f:\n",
    "    dict_obj = pickle.load(f)\n",
    "    \n",
    "with open(det_results_path_kfrm, 'rb') as f:\n",
    "    dict_obj.update(pickle.load(f))\n",
    "print('Loading done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading AIF crawl dictionaries...\n",
      "en2men len: 142899\n",
      "id2men len: 2046\n",
      "2019-07-11 23:17:10.843827\n",
      "CPU times: user 2.12 s, sys: 338 ms, total: 2.45 s\n",
      "Wall time: 3.41 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print('Loading AIF crawl dictionaries...')\n",
    "with open(working_path + 'tmp/entity2mention_dict_'+p_f+'.pickle', 'rb') as f:\n",
    "    en2men = pickle.load(f)\n",
    "\n",
    "with open(working_path + 'tmp/id2mentions_dict_'+p_f+'.pickle', 'rb') as f:\n",
    "    id2men = pickle.load(f)\n",
    "\n",
    "# Check Point: en2men and id2men maybe None\n",
    "print('en2men len:', len(en2men.keys()))\n",
    "print('id2men len:', len(id2men.keys()))\n",
    "print(datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating id-sentence-entity dictionary...\n",
      "2019-07-10 23:56:47.032572\n",
      "Creating dictionary and filtering out for grounding...\n",
      "Processing graphs done.\n",
      "2019-07-11 00:24:00.030831\n",
      "CPU times: user 26min 35s, sys: 19.7 s, total: 26min 55s\n",
      "Wall time: 27min 12s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# maybe changed: turtle_files comes from RPI, .ttl maybe changed \n",
    "# about 1 hour 15 min\n",
    "print('Creating id-sentence-entity dictionary...')\n",
    "print(datetime.now())\n",
    "en2men = {}\n",
    "turtle_files = os.listdir(RPI_AIF_path)\n",
    "import ltf_util as rpi_ltf_util\n",
    "ltf_util = rpi_ltf_util.LTF_util(ltf_path)\n",
    "for i,file in enumerate(turtle_files):\n",
    "    if \".ttl\" not in file:\n",
    "        continue\n",
    "#     print(i,file) # HC000VULD.ttl\n",
    "#     # Done: text mention filter checking\n",
    "#     if (file not in ['HC000VULD.ttl','HC000VULD.ttl']):\n",
    "#         continue\n",
    "#     else:\n",
    "#         print('file',file)\n",
    "    turtle_path = os.path.join(RPI_AIF_path, file)\n",
    "    #loading turtle content\n",
    "    turtle_content = open(turtle_path).read()\n",
    "    g = Graph().parse(data=turtle_content, format='n3')\n",
    "    # Check Point: en2men maybe changed\n",
    "    #find 'aida:Entity', then find 'aida:justifiedBy'\n",
    "#     print('g',len(en2men.keys()))\n",
    "    en2men.update(utils.get_entity2mention(g,ltf_util)) \n",
    "    sys.stdout.write('File {}/{} \\r'.format(i+1,len(turtle_files)))                \n",
    "    sys.stdout.flush()\n",
    "    \n",
    "## mention type: mention, nominal_mention, pronominal_mention, normalized_mention\n",
    "## Named entity types: PER PRG GPE LOC ORG WEA VEH\n",
    "## Filler types: BAL COM CRM LAW MON RES SID TTL VAL\n",
    "\n",
    "#short_to_long_map = {\n",
    "#    'BAL': 'Ballot',\n",
    "#    'COM': 'Commodity',\n",
    "#    'CRM': 'Crime',\n",
    "#    'FAC': 'Facility',\n",
    "#    'GPE': 'GeopoliticalEntity',\n",
    "#    'LAW': 'Law',\n",
    "#    'LOC': 'Location',\n",
    "#    'MON': 'Money',\n",
    "#    'ORG': 'Organization',\n",
    "#    'PER': 'Person',\n",
    "#    'RES': 'Result',\n",
    "#    'SID': 'Sides',\n",
    "#    'TTL': 'Title',\n",
    "#    'VAL': 'Value',\n",
    "#    'VEH': 'Vehicle',\n",
    "#    'WEA': 'Weapon',\n",
    "# }\n",
    "# filter_out=['pronominal_mention','GeopoliticalEntity','Organization',\n",
    "#             'Location','Money','NumericalValue','Time','URL','Sides',\n",
    "#             'Sentence','Results','Law','Crime','Ballot','Age']\n",
    "print('Creating dictionary and filtering out for grounding...')\n",
    "filter_out=[ 'pronominal_mention', 'GPE', 'ORG',\n",
    "       'LOC','MON', 'NumericalValue', 'VAL', 'Time', 'URL', 'COM', 'SID',\n",
    "        'Sentence', 'RES', 'LAW', 'CRM', 'BAL', 'Age', 'TTL'] \n",
    "\n",
    "id2men = utils.create_entity_dict(en2men, path_dict, caption_alignment_path=[], filter_out=filter_out)\n",
    "# uncomment\n",
    "print('Processing graphs done.')\n",
    "open('log_grounding_'+p_f+'.txt', 'w').write('Processing graphs done.\\n')\n",
    "with open(working_path + 'tmp/entity2mention_dict_'+p_f+'.pickle', 'wb') as f:\n",
    "    pickle.dump(en2men,f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "with open(working_path + '/tmp/id2mentions_dict_'+p_f+'.pickle', 'wb') as f:\n",
    "    pickle.dump(id2men,f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "print(datetime.now())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0711 23:17:10.896356 140076587128640 deprecation_wrapper.py:119] From /home/bobby/aida_copy/AIDA/M18_copy/code_coconut/utils.py:874: The name tf.InteractiveSession is deprecated. Please use tf.compat.v1.InteractiveSession instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading grounding pretrained model...\n",
      "Model Loading...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0711 23:17:13.951940 140076587128640 deprecation_wrapper.py:119] From /home/bobby/aida_copy/AIDA/M18_copy/code_coconut/utils.py:875: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead.\n",
      "\n",
      "W0711 23:17:13.979271 140076587128640 deprecation_wrapper.py:119] From /home/bobby/aida_copy/AIDA/M18_copy/code_coconut/utils.py:875: The name tf.tables_initializer is deprecated. Please use tf.compat.v1.tables_initializer instead.\n",
      "\n",
      "W0711 23:17:14.059724 140076587128640 deprecation_wrapper.py:119] From /home/bobby/aida_copy/AIDA/M18_copy/code_coconut/utils.py:877: The name tf.train.import_meta_graph is deprecated. Please use tf.compat.v1.train.import_meta_graph instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pass\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0711 23:17:20.086181 140076587128640 deprecation.py:323] From /dvmm-filer2/users/bobby/anaconda/envs/aida/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----Model Loaded-----\n",
      "Loading done.\n",
      "2019-07-11 23:17:37.603166\n",
      "CPU times: user 8.01 s, sys: 3.97 s, total: 12 s\n",
      "Wall time: 26.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#loading grounding pretrained model\n",
    "print('Loading grounding pretrained model...')\n",
    "gamma_1 = 10.0\n",
    "gamma_1_s = 10.0\n",
    "gamma_2 = 5.0\n",
    "gamma_3 = 10.0\n",
    "model_path = working_path + 'models/model_ELMo_PNASNET_VOA_norm'\n",
    "#model_path = '../model_CNN_avg'\n",
    "sess, graph = utils.load_model(model_path,config)\n",
    "input_img = graph.get_tensor_by_name(\"input_img:0\")\n",
    "text_batch = graph.get_tensor_by_name(\"text_input:0\")\n",
    "mode = graph.get_tensor_by_name(\"mode:0\")\n",
    "v = graph.get_tensor_by_name(\"image_local_features:0\")\n",
    "v_bar = graph.get_tensor_by_name(\"image_global_features:0\")\n",
    "w_embedding = graph.get_tensor_by_name(\"w_embedding:0\")\n",
    "sen_embedding = graph.get_tensor_by_name(\"sen_embedding:0\")\n",
    "heatmap_w, heatmap_s, c, c_s, R_i, R, R_s = attn(w_embedding,v,sen_embedding,v_bar)\n",
    "print('Loading done.')\n",
    "print(datetime.now())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grounding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-07-11 00:48:27.735141\n",
      "Grounding done.\n",
      "Averaging textual features to shape entity features...\n",
      "Averaging done.\n",
      "Clustering mention bboxes to shape entity bbox...\n",
      "Clustering done. grounding_dict.tmp len: 42905\n",
      "2019-07-11 15:35:37.795029\n",
      "CPU times: user 10d 9h 33min 47s, sys: 1d 44min 25s, total: 11d 10h 18min 13s\n",
      "Wall time: 14h 47min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# about 6-8 hours in total for Grounding\n",
    "#grounding, entity level\n",
    "print(datetime.now())\n",
    "# Todo: adjust parameters\n",
    "en_score_thr = .5 #.9\n",
    "sen_score_thr = .6 #.6\n",
    "suffix_tmp = '_' + p_f + '_5-6'\n",
    "\n",
    "en_to_img_dict = {}\n",
    "img_to_feat_dict = {}\n",
    "img_cnt_dict = {}\n",
    "for k,key in enumerate(id2men):\n",
    "    sys.stdout.write('Key {}/{} \\r'.format(k,len(id2men)))                \n",
    "    sys.stdout.flush()\n",
    "    open('log_grounding_'+p_f+'.txt', 'w').write('Key {}/{} \\r'.format(k,len(id2men)))\n",
    "    #get all sens of the given leaf doc, and all images of its root doc\n",
    "    imgs,ids = utils.fetch_img(key, parent_dict, child_dict, path_dict, level = 'Parent')\n",
    "    sens = list(id2men[key].keys())\n",
    "    if len(sens)==0:\n",
    "        continue\n",
    "    #generate all img-sen pairs and their corresponding grounding params\n",
    "    text_flag = len(imgs)==0\n",
    "    img_batch, sen_batch, img_info_batch = utils.img_cap_batch_gen(imgs,sens,ids,key,path_dict,id2time_dict_kfrm)\n",
    "    placeholders = [input_img, text_batch, mode]\n",
    "    tensor_list = [w_embedding, c, heatmap_w, R_i, R, R_s]\n",
    "    inputs = [img_batch,sen_batch]\n",
    "    EN_embd, IMG_embd, EN_heat, EN_score, avg_EN_score, sen_score = utils.batch_split_run(sess,tensor_list,placeholders,inputs,text_flag,b_size_thr=30)\n",
    "    for i,sen in enumerate(sen_batch):\n",
    "        #if sen_score[i] < sen_score_thr:\n",
    "        #    continue\n",
    "        for entity in id2men[key][sen]:\n",
    "            en_name = id2men[key][sen][entity]['name']\n",
    "            if en_name.lower() in pronouns:\n",
    "                continue\n",
    "            for mention in id2men[key][sen][entity]['mentions']:\n",
    "                men_dict = id2men[key][sen][entity]['mentions'][mention]\n",
    "                if men_dict['idx']==[]:\n",
    "                    continue\n",
    "            \n",
    "                #if np.mean(EN_score[i,men_dict['idx']]) < en_score_thr:\n",
    "                #    continue\n",
    "                men_name = men_dict['name']\n",
    "                if men_name.lower() in pronouns:\n",
    "                    continue\n",
    "                en_type = id2men[key][sen][entity]['type_rdf']\n",
    "                source_type = id2men[key][sen][entity]['source_type']\n",
    "                language = id2men[key][sen][entity]['language']\n",
    "                             \n",
    "                #men_embd = np.average(EN_embd[i,men_dict['idx'],:], weights = EN_score[i,men_dict['idx']], axis=0)\n",
    "                men_embd = np.mean(EN_embd[i,men_dict['idx'],:], axis=0) #for now, just averaging word embdngs\n",
    "                orig_img_id = img_info_batch[i][0]\n",
    "                \n",
    "                if text_flag:\n",
    "                    grnd = {}\n",
    "                elif sen_score[i] < sen_score_thr or np.mean(EN_score[i,men_dict['idx']]) < en_score_thr:\n",
    "                    grnd = {}\n",
    "                else:\n",
    "                    heatmap = np.average(EN_heat[i,men_dict['idx'],:], weights = EN_score[i,men_dict['idx']], axis=0)\n",
    "                    img_embd = np.average(IMG_embd[i,men_dict['idx'],:], weights = EN_score[i,men_dict['idx']], axis=0)\n",
    "                    orig_img_shape = img_info_batch[i][1]\n",
    "                    bbox_dict = utils.heat2bbox(heatmap,orig_img_shape)\n",
    "                    bbox, bbox_norm, score = utils.filter_bbox(bbox_dict=bbox_dict, order='xyxy')\n",
    "                    grnd = {'bbox': bbox, 'bbox_norm': bbox_norm, 'bbox_score': score,\n",
    "                            'heatmap': heatmap, 'sen-img-score': sen_score[i],\n",
    "                            'men-img-score': EN_score[i,men_dict['idx']],\n",
    "                            'grounding_features': img_embd}\n",
    "            \n",
    "                men_dict = {mention: {'grounding': {orig_img_id: grnd},\n",
    "                                      'textual_features': men_embd,\n",
    "                                      'name': men_name,\n",
    "                                      'sentence': sen}}\n",
    "                en_dict = {'textual_features': np.zeros((men_embd.shape),dtype='float32'),\n",
    "                           'name': en_name,\n",
    "                           'type_rdf': en_type,\n",
    "                           'mentions': men_dict,\n",
    "                           'source_type': source_type,\n",
    "                           'language': language}\n",
    "                if entity not in en_to_img_dict:\n",
    "                    en_to_img_dict[entity] = en_dict\n",
    "                elif mention not in en_to_img_dict[entity]['mentions']:\n",
    "                    en_to_img_dict[entity]['mentions'].update(men_dict)\n",
    "                elif orig_img_id not in en_to_img_dict[entity]['mentions'][mention]['grounding']:\n",
    "                    en_to_img_dict[entity]['mentions'][mention]['grounding'].update(men_dict[mention]['grounding'])\n",
    "sess.close()\n",
    "\n",
    "print('Grounding done.\\nAveraging textual features to shape entity features...')\n",
    "#averaging mention features to shape entity features\n",
    "for entity in en_to_img_dict:\n",
    "    for mention in en_to_img_dict[entity]['mentions']:\n",
    "        en_to_img_dict[entity]['textual_features'] += en_to_img_dict[entity]['mentions'][mention]['textual_features']\n",
    "    en_to_img_dict[entity]['textual_features'] /= len(en_to_img_dict[entity]['mentions'])\n",
    "\n",
    "print('Averaging done.\\nClustering mention bboxes to shape entity bbox...')\n",
    "#clustering mention bboxes for each entity\n",
    "for k,entity in enumerate(en_to_img_dict):\n",
    "    men_dict = copy.deepcopy(en_to_img_dict[entity]['mentions'])\n",
    "    # Done for USC: print dict of en_to_img_dict, and output\n",
    "    en_to_img_dict[entity]['grounding'] = utils.men2en_grnd(men_dict,dict_obj)\n",
    "    sys.stdout.write('Key {}/{} \\r'.format(k,len(en_to_img_dict)))                \n",
    "    sys.stdout.flush() \n",
    "#Todo alter filename\n",
    "with open(grounding_dict_path + suffix_tmp + '.tmp', 'wb') as f:\n",
    "    pickle.dump(en_to_img_dict,f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "print('Clustering done.', 'grounding_dict.tmp len:',len(en_to_img_dict))\n",
    "open('log_grounding_'+p_f+'.txt', 'w').write('\\nGrounding Stage done.')\n",
    "print(datetime.now())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Instance Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Loading...\n",
      "pass\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/dvmm-filer2/users/bobby/anaconda/envs/aida/lib/python3.6/site-packages/tensorflow/python/client/session.py:1735: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----Model Loaded-----\n",
      "CPU times: user 8.6 s, sys: 6.14 s, total: 14.7 s\n",
      "Wall time: 9.52 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#loading instance matching pretrained model\n",
    "model_path = working_path + 'models/model_universal_no_recons_ins_only'\n",
    "sess_ins, graph_ins = utils.load_model(model_path,config)\n",
    "input_img_ins = graph_ins.get_tensor_by_name(\"input_img:0\")\n",
    "mode_ins = graph_ins.get_tensor_by_name(\"mode:0\")\n",
    "img_vec_ins = graph_ins.get_tensor_by_name(\"img_vec:0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_of_bbox(img, bbox_norm):\n",
    "    img_batch = np.empty((len(bbox_norm),224,224,3), dtype='float32')\n",
    "    for i,bbox in enumerate(bbox_norm):\n",
    "        roi = utils.crop_resize_im(img, bbox, (224,224), order='xyxy')\n",
    "        img_batch[i,:,:,:] = roi\n",
    "    return img_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stored for image 42904 / 42905 \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-07-11 15:43:49.105216\n",
      "CPU times: user 1h 30min 8s, sys: 6min 35s, total: 1h 36min 44s\n",
      "Wall time: 8min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# about 2 hours\n",
    "suffix = suffix_tmp #'_8-6'\n",
    "with open(grounding_dict_path + suffix_tmp +'.tmp', 'rb') as f:\n",
    "    en_to_img_dict = pickle.load(f)\n",
    "print(len(en_to_img_dict))\n",
    "\n",
    "missed_children_ins = []\n",
    "maxbatch = 0\n",
    "for i,en in enumerate(en_to_img_dict):\n",
    "    try:\n",
    "#         if i in [11996,12085]:\n",
    "#             continue\n",
    "        for img_id in en_to_img_dict[en]['grounding']:\n",
    "            imgs,_ = utils.fetch_img(img_id, parent_dict, child_dict, path_dict, level = 'Child')\n",
    "            if len(imgs)==0:\n",
    "                missed_children_ins.append(key)\n",
    "                continue\n",
    "            bbox_norm = en_to_img_dict[en]['grounding'][img_id]['bbox_norm']\n",
    "            img_batch = batch_of_bbox(imgs[0], bbox_norm)\n",
    "            feed_dict = {input_img_ins: img_batch, mode_ins: 'test'}\n",
    "            img_vec_pred = sess_ins.run([img_vec_ins], feed_dict)[0]\n",
    "            en_to_img_dict[en]['grounding'][img_id]['instance_features']=[]\n",
    "            for j in range(len(bbox_norm)):\n",
    "                en_to_img_dict[en]['grounding'][img_id]['instance_features'].append(img_vec_pred[j,:])\n",
    "        sys.stderr.write(\"Stored for image {} / {} \\r\".format(i, len(en_to_img_dict)))\n",
    "        sys.stdout.flush() \n",
    "    except ValueError:\n",
    "        print(\"Oops!\",i,ValueError) \n",
    "sess_ins.close()\n",
    "\n",
    "with open(grounding_dict_path + suffix, 'wb') as f:\n",
    "    pickle.dump(en_to_img_dict,f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "print(datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.14.0-rc1'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess_ins.close()\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-07-11 15:43:49.218352\n",
      "All done.\n",
      "CPU times: user 5.26 s, sys: 2.24 s, total: 7.51 s\n",
      "Wall time: 9.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# # Todo: uncomment\n",
    "print(datetime.now())\n",
    "with open(grounding_dict_path + suffix, 'rb') as f:\n",
    "    en_to_img_dict = pickle.load(f)\n",
    "    \n",
    "with open(grounding_dict_path, 'wb') as f: #/home/bobby/aida_copy/AIDA/M18_copy/data/all_features\n",
    "    pickle.dump(en_to_img_dict,f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "print('All done.')\n",
    "open('log_grounding_'+p_f+'.txt', 'w').write('\\nAll done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# suffix = '_1-6' #suffix #'_75-6'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stored for image 20239 / 42905 \r"
     ]
    },
    {
     "ename": "error",
     "evalue": "OpenCV(4.1.0) /io/opencv/modules/core/src/alloc.cpp:55: error: (-4:Insufficient memory) Failed to allocate 814080 bytes in function 'OutOfMemoryError'\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31merror\u001b[0m                                     Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m~/aida_copy/AIDA/M18_copy/code_coconut/utils.py\u001b[0m in \u001b[0;36mfetch_img\u001b[0;34m(key, parent_dict, child_dict, path_dict, level)\u001b[0m\n\u001b[1;32m    644\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m'jpg'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mchild_id\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    645\u001b[0m             \u001b[0mfilename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpath_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mchild_id\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 646\u001b[0;31m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mldcc_load\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    647\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mimg\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m                 \u001b[0mimgs_in_parent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/aida_copy/AIDA/M18_copy/code_coconut/utils.py\u001b[0m in \u001b[0;36mldcc_load\u001b[0;34m(filename)\u001b[0m\n\u001b[1;32m    587\u001b[0m         \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1024\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m         \u001b[0mimgbin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 589\u001b[0;31m     \u001b[0mimgbgr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrombuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimgbin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'uint8'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIMREAD_COLOR\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    590\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mimgbgr\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    591\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31merror\u001b[0m: OpenCV(4.1.0) /io/opencv/modules/core/src/alloc.cpp:55: error: (-4:Insufficient memory) Failed to allocate 814080 bytes in function 'OutOfMemoryError'\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# test for mention level\n",
    "with open(grounding_dict_path + suffix, 'rb') as f:\n",
    "    en_to_img_dict = pickle.load(f)\n",
    "# todo testing\n",
    "g_suffix = suffix #'_2+5-6' #suffix #'_75-6'\n",
    "g_threshold_low = 0 #low bound: filter the entities with the en_score < threshold  \n",
    "g_threshold_high = 1 #high bound: filter the entities with the en_score < threshold  \n",
    "\n",
    "grounded_stat = {}\n",
    "\n",
    "def add_hist_stat(stat_dict, s1, s2):\n",
    "    key = str(round(s1,1)) + ' ' + str(round(s2,1))\n",
    "    if key not in stat_dict.keys():\n",
    "        stat_dict[key] = 1\n",
    "    else:\n",
    "        stat_dict[key] += 1\n",
    "    return stat_dict\n",
    "\n",
    "#mention level\n",
    "pdf = matplotlib.backends.backend_pdf.PdfPages(working_path+'Grounding_all'+g_suffix+'.pdf') #(\"G_RU_UK.pdf\")\n",
    "for k,en_id in enumerate(en_to_img_dict):\n",
    "    sys.stderr.write(\"Stored for image {} / {} \\r\".format(k, len(en_to_img_dict)))\n",
    "    sys.stdout.flush()\n",
    "    for men_id in en_to_img_dict[en_id]['mentions']:\n",
    "        en_name = en_to_img_dict[en_id]['mentions'][men_id]['name']\n",
    "        sen = en_to_img_dict[en_id]['mentions'][men_id]['sentence']\n",
    "        lan = en_to_img_dict[en_id]['language']\n",
    "        # Todo remove English\n",
    "#         if lan=='English':\n",
    "#             continue\n",
    "        for img_id in en_to_img_dict[en_id]['mentions'][men_id]['grounding']:\n",
    "            if len(en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id])==0:\n",
    "                continue\n",
    "            bboxes = en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id]['bbox_norm']\n",
    "            heatmap = en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id]['heatmap']\n",
    "            sen_score = en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id]['sen-img-score']\n",
    "            en_score = np.mean(en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id]['men-img-score'])\n",
    "            \n",
    "            if (en_score<=g_threshold_low or en_score>=g_threshold_high):\n",
    "                continue\n",
    "            #add\n",
    "            grounded_stat = add_hist_stat(grounded_stat, en_score, sen_score)\n",
    "            if img_id.find('mp4')>-1:\n",
    "                vname_arr = img_id.split('_')\n",
    "                vid = vname_arr[0] + '.' + '.'.join(vname_arr[1].split('.')[1:])             \n",
    "                fid = ' '.join(child_dict[vid])\n",
    "                title= fid + ' ' + img_id + ' '+ lan+'-Keyframe'\n",
    "#                 print(title)\n",
    "            else:\n",
    "                fid = ' '.join(child_dict[img_id])\n",
    "                title= fid + ' ' + img_id + ' '+ lan+'-JPG'\n",
    "            #img,_ = utils.fetch_img(img_id, parent_dict, child_dict, path_dict, level = 'Child')\n",
    "            title_str = title+', ' + 'Sen-Img Score: %.2f'%sen_score\n",
    "            title_str = title_str + '\\n sentence:' + sen\n",
    "            title = \"\\n\".join(wrap(title_str, 120))\n",
    "            #fig = utils.img_heat_bbox_disp(cv2.resize(img[0],(299,299)), heatmap, title=title, en_name=en_name+', '+'%.2f'%en_score, bboxes=bboxes, order='xyxy')\n",
    "            #pdf.savefig(fig)\n",
    "            if lan!='English':\n",
    "                en_name = ' '.join(en_name.split()[:2])\n",
    "            img,_ = utils.fetch_img(img_id, parent_dict, child_dict, path_dict, level = 'Child')\n",
    "            fig = utils.img_heat_bbox_disp(cv2.resize(img[0],(299,299)), heatmap, title=title, en_name=en_name+', '+'%.2f'%en_score, bboxes=bboxes, order='xyxy', show=False)\n",
    "            pdf.savefig(fig)\n",
    "pdf.close()\n",
    "print(datetime.now())\n",
    "print(sum(grounded_stat.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "#entity level\n",
    "import cv2\n",
    "\n",
    "# Load\n",
    "with open(grounding_dict_path + '_.5', 'rb') as f:\n",
    "    en_to_img_dict = pickle.load(f)\n",
    "print(len(en_to_img_dict))\n",
    "\n",
    "\n",
    "pdf = matplotlib.backends.backend_pdf.PdfPages(working_path+\"Grounding_visualization.pdf\")\n",
    "for k,en_id in enumerate(en_to_img_dict):\n",
    "#     if(k>5000):\n",
    "#         break\n",
    "    sys.stderr.write(\"Stored for image {} / {} \\r\".format(k, len(en_to_img_dict)))\n",
    "    \n",
    "#     for id in en_to_img_dict[en_id]['mentions']:\n",
    "#         e = en_to_img_dict[en_id]['mentions'][id]\n",
    "#         print(en_to_img_dict[en_id])\n",
    "#         print( e['name'],'|','sen:',e['sentence'])\n",
    "    \n",
    "    for img_id in en_to_img_dict[en_id]['grounding']:\n",
    "        en_name = en_to_img_dict[en_id]['name']\n",
    "        bboxes = en_to_img_dict[en_id]['grounding'][img_id]['bbox_norm']\n",
    "        heatmap = en_to_img_dict[en_id]['grounding'][img_id]['heatmap'][0]\n",
    "        en_score = np.mean(en_to_img_dict[en_id]['grounding'][img_id]['men-img-score'])\n",
    "        if img_id.find('mp4')>-1:\n",
    "            title='Keyframe'\n",
    "        else:\n",
    "            title='JPG'\n",
    "        img,_ = utils.fetch_img(img_id, parent_dict, child_dict, path_dict, level = 'Child')\n",
    "        fig = utils.img_heat_bbox_disp(cv2.resize(img[0],(299,299)), heatmap, title=title, en_name=en_name+', '+'%.2f'%en_score, bboxes=bboxes, order='xyxy', show=True)\n",
    "        pdf.savefig(fig)\n",
    "pdf.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "#mention level\n",
    "pdf = matplotlib.backends.backend_pdf.PdfPages(\"G_RU_UK.pdf\")\n",
    "for k,en_id in enumerate(en_to_img_dict):\n",
    "    sys.stderr.write(\"Stored for image {} / {} \\r\".format(k, len(en_to_img_dict)))\n",
    "    sys.stdout.flush()\n",
    "    for men_id in en_to_img_dict[en_id]['mentions']:\n",
    "        en_name = en_to_img_dict[en_id]['mentions'][men_id]['name']\n",
    "        sen = en_to_img_dict[en_id]['mentions'][men_id]['sentence']\n",
    "        lan = en_to_img_dict[en_id]['language']\n",
    "        if lan=='English':\n",
    "            continue\n",
    "        for img_id in en_to_img_dict[en_id]['mentions'][men_id]['grounding']:\n",
    "            if len(en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id])==0:\n",
    "                continue\n",
    "            bboxes = en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id]['bbox_norm']\n",
    "            heatmap = en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id]['heatmap']\n",
    "            sen_score = en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id]['sen-img-score']\n",
    "            en_score = np.mean(en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id]['men-img-score'])\n",
    "            if img_id.find('mp4')>-1:\n",
    "                title=lan+'-Keyframe'\n",
    "            else:\n",
    "                title=lan+'-JPG'\n",
    "            #img,_ = utils.fetch_img(img_id, parent_dict, child_dict, path_dict, level = 'Child')\n",
    "            #title_str = sen+',  Sen-Img Score: %.2f'%sen_score\n",
    "            #title = \"\\n\".join(wrap(title_str, 120))\n",
    "            #fig = utils.img_heat_bbox_disp(cv2.resize(img[0],(299,299)), heatmap, title=title, en_name=en_name+', '+'%.2f'%en_score, bboxes=bboxes, order='xyxy')\n",
    "            #pdf.savefig(fig)\n",
    "            if lan!='English':\n",
    "                en_name = ' '.join(en_name.split()[:2])\n",
    "            img,_ = utils.fetch_img(img_id, parent_dict, child_dict, path_dict, level = 'Child')\n",
    "            fig = utils.img_heat_bbox_disp(cv2.resize(img[0],(299,299)), heatmap, title=title, en_name=en_name+', '+'%.2f'%en_score, bboxes=bboxes, order='xyxy', show=False)\n",
    "            pdf.savefig(fig)\n",
    "pdf.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "suffix_tmp = '_' + p_f + '_8-6'\n",
    "suffix = suffix_tmp #'_8-6'\n",
    "with open(grounding_dict_path + suffix, 'rb') as f:\n",
    "    en_to_img_dict = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all entities 42604\n",
      "grounded entities 1569 \n",
      "\n",
      "all entities 42604\n",
      "all mentiones 80510\n",
      "grounded mentiones 5655 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "#all grounded entities\n",
    "s=0\n",
    "for en in en_to_img_dict:\n",
    "    for img_id in en_to_img_dict[en]['grounding']:\n",
    "        if len(en_to_img_dict[en]['grounding'][img_id])>0:\n",
    "            s+=1\n",
    "#             print(en_to_img_dict[en]['type_rdf'])\n",
    "print('all entities', len(en_to_img_dict))\n",
    "print('grounded entities',s,'\\n')\n",
    "\n",
    "\n",
    "\n",
    "#all grounded mentions\n",
    "e=0\n",
    "m=0\n",
    "s=0\n",
    "\n",
    "for k,en_id in enumerate(en_to_img_dict):\n",
    "    e+=1\n",
    "    for men_id in en_to_img_dict[en_id]['mentions']:\n",
    "        m+=1\n",
    "        for img_id in en_to_img_dict[en_id]['mentions'][men_id]['grounding']:\n",
    "            if len(en_to_img_dict[en_id]['mentions'][men_id]['grounding'][img_id])>0:\n",
    "                s+=1\n",
    "#             print(en_to_img_dict[en]['type_rdf'])\n",
    "print('all entities', len(en_to_img_dict))\n",
    "print('all mentiones',m)\n",
    "print('grounded mentiones',s,'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "                \n",
    "def check_grounding_dict(grounding_dict, child_dict):\n",
    "    # test\n",
    "#     key = ''\n",
    "#     print('arrrr', grounding_dict[key]['grounding']['system'])\n",
    "    \n",
    "    # Check Point: grounding_dict printing\n",
    "    print('cu_grounded')\n",
    "    for en in grounding_dict.keys():\n",
    "        for img_id in grounding_dict[en]['grounding']:\n",
    "            if img_id  =='system':\n",
    "                continue\n",
    "            n_b = len(grounding_dict[en]['grounding'][img_id]['bbox'])\n",
    "            if n_b>0:\n",
    "                img_dict = grounding_dict[en]['grounding'][img_id]\n",
    "                print(en)\n",
    "                if img_id.find('mp4')>-1:\n",
    "                    vname_arr = img_id.split('_')\n",
    "                    img_id = vname_arr[0] + '.' + '.'.join(vname_arr[1].split('.')[1:])  \n",
    "            \n",
    "                \n",
    "                print(child_dict[img_id], img_id, grounding_dict[en]['grounding'][img_id])\n",
    "                print(sum(sum(img_dict['grounding_features'])),sum(sum(img_dict['instance_features'])))\n",
    "                print('\\n')  \n",
    "check_grounding_dict(grounding_dict, child_dict)\n",
    "\n",
    "\n",
    "# fid = ' '.join(child_dict[img_id])\n",
    "\n",
    "\n",
    "#         if img_id.find('mp4')>-1:\n",
    "                 \n",
    "#                 fid = ' '.join(child_dict[img_id])\n",
    "#                 title= fid + ' ' + img_id + ' '+ lan+'-Keyframe'\n",
    "# #                 print(title)\n",
    "#             else:\n",
    "#                 fid = ' '.join(child_dict[img_id])\n",
    "#                 title= fid + ' ' + img_id + ' '+ lan+'-JPG'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2men"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for en in en_to_img_dict:\n",
    "    print(en_to_img_dict[en]['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check Point: grounding_dict printing\n",
    "sub_en2img_dict = {}\n",
    "for en in en_to_img_dict:\n",
    "    for img_id in en_to_img_dict[en]['grounding']:\n",
    "        n_b = len(en_to_img_dict[en]['grounding'][img_id]['bbox'])\n",
    "        if n_b>0:\n",
    "            print(en)\n",
    "            print(img_id, child_dict[img_id])\n",
    "            print('\\n')\n",
    "            sub_en2img_dict[en] = img_id\n",
    "\n",
    "# cu_grndg_ent_pref = 'http://www.columbia.edu/AIDA/DVMM/Entities/GroundingBox/'+merge_version+'/'\n",
    "# cu_grndg_type_pref = 'http://www.columbia.edu/AIDA/DVMM/TypeAssertions/GroundingBox/'+merge_version+'/'\n",
    "# cu_grndg_clstr_img_pref = 'http://www.columbia.edu/AIDA/DVMM/Clusters/BoxOverlap/'+merge_version+'/'\n",
    "# cu_grndg_clstr_txt_pref = 'http://www.columbia.edu/AIDA/DVMM/Clusters/Grounding/'+merge_version+'/'\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check Point: printing\n",
    "# en='http://www.isi.edu/gaia/entities/c5137f4c-0d65-4d67-ad50-8338c610285f'\n",
    "# gid = 'IC0011VED.jpg.ldcc'\n",
    "# en_to_img_dict[en]['grounding'][gid]\n",
    "def top_dict(kv_dict, num = 2):\n",
    "    k_list = list(kv_dict.keys())[:num]\n",
    "    subdict =  {k: kv_dict[k] for k in k_list}\n",
    "#     print(subdict)\n",
    "    return subdict\n",
    "# top_dict(sub_en2img_dict)\n",
    "print('sub_en2img_dict lens:',len(list(sub_en2img_dict.keys())))\n",
    "sub_enlist = list(sub_en2img_dict.keys())[:3]\n",
    "print(sub_enlist)\n",
    "for en in sub_enlist:\n",
    "    sub_imglist = list(en_to_img_dict[en]['grounding'].keys())[:1]\n",
    "    print(sub_imglist)\n",
    "    sub_imgdict =  {k: en_to_img_dict[en]['grounding'][k] for k in sub_imglist}\n",
    "    print(sub_imgdict)\n",
    "print(child_dict[img_id])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "train_writer = tf.summary.FileWriter('./logs', graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "!tensorboard --logdir ./logs --port=8993"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "#checking whether english entities are subsets of all or not\n",
    "# maybe changed\n",
    "from rdflib import RDF, URIRef\n",
    "# nist_ont_pref = '.../SM-KBP/2018/ontologies/InterchangeOntology#'\n",
    "nist_ont_pref = 'https://tac.nist.gov/tracks/SM-KBP/2019/ontologies/InterchangeOntology#'\n",
    "entity_ = URIRef(nist_ont_pref+'Entity')\n",
    "RPI_AIF_path_en = working_path + 'rpi_ttl/ttl_en_r0'\n",
    "RPI_AIF_path_all = working_path + 'rpi_ttl/ttl_all_r0'\n",
    "turtle_files_en = os.listdir(RPI_AIF_path_en)\n",
    "turtle_files_all = os.listdir(RPI_AIF_path_all)\n",
    "print('totel len:',len(turtle_files_en))\n",
    "for i,file in enumerate(turtle_files_en):\n",
    "    print(file)\n",
    "    if \".turtle\" not in file:\n",
    "        continue\n",
    "    if i%1000 ==0:\n",
    "        print('i = ',i)\n",
    "    turtle_path_en = os.path.join(RPI_AIF_path_en, file)\n",
    "    turtle_path_all = os.path.join(RPI_AIF_path_all, file)\n",
    "    #loading turtle content\n",
    "    turtle_content_en = open(turtle_path_en).read()\n",
    "    turtle_content_all = open(turtle_path_all).read()\n",
    "    g_en = Graph().parse(data=turtle_content_en, format='n3')\n",
    "    g_all = Graph().parse(data=turtle_content_all, format='n3')\n",
    "    entities_all = list(g_all.subjects(predicate=RDF.type,object=entity_))\n",
    "    entities_en = list(g_en.subjects(predicate=RDF.type,object=entity_))\n",
    "    for en in entities_en:\n",
    "        if en not in entities_all:\n",
    "            print(en)\n",
    "    print(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#old fetch img\n",
    "def fetch_img(child_id, parent_dict, child_dict, path_dict, level='Child'):\n",
    "    if child_id not in child_dict:\n",
    "        return [],[]\n",
    "    \n",
    "    elif level=='Child':\n",
    "        if child_id.find('jpg')!=-1:\n",
    "            filename = path_dict[child_id]\n",
    "            img = ldcc_load(filename)\n",
    "            if img is not None:\n",
    "                return [img],[child_id]\n",
    "        elif child_id.find('mp4')!=-1:\n",
    "            files = path_dict[child_id]\n",
    "            imgs_in_video = []\n",
    "            ids_in_video = []\n",
    "            for i,filename in enumerate(files):\n",
    "                img = ldcc_load(filename)\n",
    "                if img is not None:\n",
    "                    imgs_in_video.append(img)\n",
    "                    ids_in_video.append(child_id.split('.')[0]+'_'+str(i)+'.mp4.ldcc')\n",
    "            return imgs_in_video, ids_in_video\n",
    "        return [],[]\n",
    "    \n",
    "    elif level=='Parent':\n",
    "        imgs_in_parent = []\n",
    "        ids_in_parent = []\n",
    "        #load and return all jpgs in parent\n",
    "        for child_id in parent_dict[child_dict[child_id]]:\n",
    "            if child_id.find('jpg')!=-1:\n",
    "                filename = path_dict[child_id]\n",
    "                img = ldcc_load(filename)\n",
    "                if img is not None:\n",
    "                    imgs_in_parent.append(img)\n",
    "                    ids_in_parent.append(child_id)\n",
    "        return imgs_in_parent, ids_in_parent\n",
    "    else:\n",
    "        return [],[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#merging kfrm_cap_batch and img_cap_batch together\n",
    "def img_cap_batch_gen_test(imgs,sens,ids,key,path_dict,id2time_dict):\n",
    "    if len(imgs)==0:\n",
    "        return np.array([]),sens,['no_img']*len(sens)\n",
    "    \n",
    "    if key.split('.')[1]=='mp4':\n",
    "        dtype = 'Keyframe'\n",
    "        doc2time_dict = doc2time(key,path_dict)\n",
    "    else:\n",
    "        dtype = 'JPG'\n",
    "    \n",
    "    sen_batch = []\n",
    "    img_batch = []\n",
    "    img_info_batch = []\n",
    "    for sen in sens:\n",
    "        if dtype=='Keyframe':\n",
    "            img_ids = sen2kfrm(sen,key,doc2time_dict,id2time_dict)\n",
    "        elif dtype=='JPG':\n",
    "            img_ids = ids\n",
    "            \n",
    "        for i,img_id in enumerate(img_ids):\n",
    "            if dtype=='Keyframe':\n",
    "                num = int(img_id.split('_')[1])\n",
    "                img = imgs[num-1]\n",
    "                ftype = '.mp4.ldcc'\n",
    "            elif dtype=='JPG':\n",
    "                img = imgs[i]\n",
    "                ftype = '' #it already contains .jpg.ldcc\n",
    "            sen_batch.append(sen)\n",
    "            img_batch.append(cv2.resize(img,(299,299)))\n",
    "            img_info_batch.append([img_id+ftype,(img.shape[0:2])])\n",
    "    img_batch = np.array(img_batch)        \n",
    "   \n",
    "    return img_batch,sen_batch,img_info_batch\n",
    "\n",
    "def kfrm_cap_batch_gen(kfrms,sens,key,path_dict,id2time_dict):\n",
    "    doc2time_dict = doc2time(key,path_dict)\n",
    "    sen_batch = []\n",
    "    kfrm_batch = []\n",
    "    kfrm_info_batch = []\n",
    "    for sen in sens:\n",
    "        kfrm_ids = sen2kfrm(sen,key,doc2time_dict,id2time_dict)\n",
    "        for kfrm_id in kfrm_ids:\n",
    "            num = int(kfrm_id.split('_')[1])\n",
    "            kfrm = kfrms[num-1]\n",
    "            sen_batch.append(sen)\n",
    "            kfrm_batch.append(cv2.resize(kfrm,(299,299)))\n",
    "            kfrm_info_batch.append([kfrm_id+'.mp4.ldcc',(kfrm.shape[0:2])])\n",
    "    kfrm_batch = np.array(kfrm_batch)        \n",
    "    return kfrm_batch,sen_batch,kfrm_info_batch\n",
    "\n",
    "def img_cap_batch_gen(imgs,sens,ids):\n",
    "    sen_batch = []\n",
    "    img_batch = []\n",
    "    img_info_batch = []\n",
    "    for sen in sens:\n",
    "        for i,img in enumerate(imgs):\n",
    "            sen_batch.append(sen)\n",
    "            img_batch.append(cv2.resize(img,(299,299)))\n",
    "            img_info_batch.append([ids[i],(img.shape[0:2])])\n",
    "    img_batch = np.array(img_batch)        \n",
    "    return img_batch,sen_batch,img_info_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "print('Creating dictionary and filtering out for grounding...')\n",
    "filter_out=['pronominal_mention','GeopoliticalEntity','Organization',\n",
    "            'Location','Money','NumericalValue','Time','URL']\n",
    "id2men = utils.create_entity_dict(en2men, path_dict, caption_alignment_path=[], filter_out=filter_out)\n",
    "en_types_filtered=set([])\n",
    "for id_ in id2men:\n",
    "    for sen in id2men[id_]:\n",
    "        for en in id2men[id_][sen]:\n",
    "            en_type = id2men[id_][sen][en]['type_rdf'].split('#')[1]\n",
    "            en_types_filtered.update([en_type])\n",
    "en_types = set([])\n",
    "for en in en2men:\n",
    "    type_rdf = en2men[en]['type_rdf']\n",
    "    en_type = type_rdf.toPython().split('#')[1]\n",
    "    en_types.update([en_type])\n",
    "men_types = set([])\n",
    "for en in en2men:\n",
    "    for men in en2men[en]['mentions']:\n",
    "        men_type = en2men[en]['mentions'][men]['mention_type']\n",
    "        men_types.update([men_type])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}