RelationshipSimilarities.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mining 'synonymous' relationships using multiple types of features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from collections import Counter, defaultdict\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.cluster import DBSCAN\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Useful function for spatial features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_box_deltas(subj_boxes, obj_boxes, height, width):\n",
    "    \"\"\"\n",
    "    Another spatial feature.\n",
    "    (D(S,O), D(S,P), D(O,P)), with D(S,O)=\n",
    "    (xs-xo)/ws, (ys-yo)/hs, log(ws/wo),log(hs/ho), (xo-xs)/wo, (yo-ys)/ho\n",
    "    \"\"\"\n",
    "    pred_boxes = _compute_predicate_boxes(subj_boxes, obj_boxes)\n",
    "    x_subj = (subj_boxes[:, 2] + subj_boxes[:, 3]) / 2\n",
    "    y_subj = (subj_boxes[:, 0] + subj_boxes[:, 1]) / 2\n",
    "    x_pred = (pred_boxes[:, 2] + pred_boxes[:, 3]) / 2\n",
    "    y_pred = (pred_boxes[:, 0] + pred_boxes[:, 1]) / 2\n",
    "    x_obj = (obj_boxes[:, 2] + obj_boxes[:, 3]) / 2\n",
    "    y_obj = (obj_boxes[:, 0] + obj_boxes[:, 1]) / 2\n",
    "    w_subj = subj_boxes[:, 3] - subj_boxes[:, 2]\n",
    "    h_subj = subj_boxes[:, 1] - subj_boxes[:, 0]\n",
    "    w_pred = pred_boxes[:, 3] - pred_boxes[:, 2]\n",
    "    h_pred = pred_boxes[:, 1] - pred_boxes[:, 0]\n",
    "    w_obj = obj_boxes[:, 3] - obj_boxes[:, 2]\n",
    "    h_obj = obj_boxes[:, 1] - obj_boxes[:, 0]\n",
    "    return np.stack((\n",
    "        (x_subj - x_obj) / w_subj, (y_subj - y_obj) / h_subj,\n",
    "        np.log(w_subj / w_obj), np.log(h_subj / h_obj),\n",
    "        (x_obj - x_subj) / w_obj, (y_obj - y_subj) / h_obj,\n",
    "        (x_subj - x_pred) / w_subj, (y_subj - y_pred) / h_subj,\n",
    "        np.log(w_subj / w_pred), np.log(h_subj / h_pred),\n",
    "        (x_pred - x_subj) / w_pred, (y_pred - y_subj) / h_pred,\n",
    "        (x_obj - x_pred) / w_obj, (y_obj - y_pred) / h_obj,\n",
    "        np.log(w_obj / w_pred), np.log(h_obj / h_pred),\n",
    "        (x_pred - x_obj) / w_pred, (y_pred - y_obj) / h_pred,\n",
    "        subj_boxes[:, 0] / height, subj_boxes[:, 1] / height,\n",
    "        subj_boxes[:, 2] / width, subj_boxes[:, 3] / width,\n",
    "        obj_boxes[:, 0] / height, obj_boxes[:, 1] / height,\n",
    "        obj_boxes[:, 2] / width, obj_boxes[:, 3] / width,\n",
    "        pred_boxes[:, 0] / height, pred_boxes[:, 1] / height,\n",
    "        pred_boxes[:, 2] / width, pred_boxes[:, 3] / width,\n",
    "        w_subj * h_subj / (height * width),\n",
    "        w_obj * h_obj / (height * width),\n",
    "        w_pred / w_subj, h_pred / h_subj,\n",
    "        w_pred / w_obj, h_pred / h_obj,\n",
    "        w_obj / w_subj, h_obj / h_subj\n",
    "    ), axis=1)\n",
    "\n",
    "\n",
    "def _compute_predicate_boxes(subj_boxes, obj_boxes):\n",
    "    return np.stack([\n",
    "        np.minimum(subj_boxes[:, 0], obj_boxes[:, 0]),\n",
    "        np.maximum(subj_boxes[:, 1], obj_boxes[:, 1]),\n",
    "        np.minimum(subj_boxes[:, 2], obj_boxes[:, 2]),\n",
    "        np.maximum(subj_boxes[:, 3], obj_boxes[:, 3])\n",
    "    ], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('prerequisites/sgg_annos/VRD_preddet.json') as fid:\n",
    "    ANNOS = json.load(fid)\n",
    "for anno in ANNOS:\n",
    "    anno['objects'] = {\n",
    "        'names': np.array(anno['objects']['names']),\n",
    "        'ids': np.array(anno['objects']['ids']),\n",
    "        'boxes': np.array(anno['objects']['boxes'])\n",
    "    }\n",
    "    anno['relations'] = {\n",
    "        'names': np.array(anno['relations']['names']),\n",
    "        'ids': np.array(anno['relations']['ids']),\n",
    "        'subj_ids': np.array(anno['relations']['subj_ids']),\n",
    "        'obj_ids': np.array(anno['relations']['obj_ids'])\n",
    "    }\n",
    "ANNOS = {anno['filename']: anno for anno in ANNOS}\n",
    "\n",
    "with open('prerequisites/sgg_annos/' + 'VRD' + '_objects.json') as fid:\n",
    "    OBJECTS = np.array(json.load(fid))\n",
    "with open('prerequisites/sgg_annos/'+ 'VRD' + '_predicates.json') as fid:\n",
    "    PREDICATES = np.array(json.load(fid))\n",
    "\n",
    "INV_PRED = {pred: p for p, pred in enumerate(PREDICATES)}\n",
    "INV_OBJ = {obj: o for o, obj in enumerate(OBJECTS)}\n",
    "\n",
    "RELATIONSHIPS = sorted(list(set(\n",
    "    '_'.join([subj, pred, obj])\n",
    "    for anno in ANNOS.values()\n",
    "    for subj, pred, obj in zip(\n",
    "        np.array(anno['objects']['names'])[anno['relations']['subj_ids']],\n",
    "        anno['relations']['names'],\n",
    "        np.array(anno['objects']['names'])[anno['relations']['obj_ids']]\n",
    "    )\n",
    ")))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Similarities using spatial features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [04:22,  2.62s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 1.0,\n",
       " -1.0,\n",
       " 0.3333333333333333,\n",
       " -1.0,\n",
       " 0.3333333333333333,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " 0.16666666666666666,\n",
       " 1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " 1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.8461538461538461,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.2,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " 0.0]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_spatial_features(subj, obj):\n",
    "    \"\"\"Get spatial features for pairs with given subj.-obj.\"\"\"\n",
    "    spatial_features = defaultdict(list)\n",
    "    for name, anno in ANNOS.items():\n",
    "        inds = (\n",
    "            (anno['objects']['names'][anno['relations']['subj_ids']] == subj)\n",
    "            & (anno['objects']['names'][anno['relations']['obj_ids']] == obj)\n",
    "        )\n",
    "        if inds.any():\n",
    "            subj_boxes = anno['objects']['boxes'][anno['relations']['subj_ids'][inds]]\n",
    "            obj_boxes = anno['objects']['boxes'][anno['relations']['obj_ids'][inds]]\n",
    "            deltas = get_box_deltas(subj_boxes, obj_boxes, anno['height'], anno['width'])\n",
    "            preds = anno['relations']['names'][inds]\n",
    "            for pred, delta in zip(preds, deltas):\n",
    "                spatial_features[str(pred)].append(delta)\n",
    "    for key in spatial_features:\n",
    "        spatial_features[key] = np.array(spatial_features[key])\n",
    "    return spatial_features\n",
    "\n",
    "\n",
    "similarities = {}\n",
    "for s, subj in tqdm(enumerate(OBJECTS)):\n",
    "    for o, obj in enumerate(OBJECTS):\n",
    "        # Get and sort features\n",
    "        features = get_spatial_features(subj, obj)\n",
    "        if not features.keys():\n",
    "            continue\n",
    "        names = sorted(list(features.keys()))\n",
    "        relationships = [\n",
    "            name for name in names for _ in range(len(features[name]))\n",
    "        ]\n",
    "        features = np.concatenate([features[name] for name in names], axis=0)\n",
    "\n",
    "        # Clustering\n",
    "        clustering = DBSCAN(eps=1, min_samples=1, metric='euclidean').fit(features)\n",
    "        clusters = defaultdict(list)\n",
    "        neg_label = 0\n",
    "        for cnt, label in enumerate(clustering.labels_):\n",
    "            if label > -1:\n",
    "                clusters[str(label)].append(relationships[cnt])\n",
    "            else:\n",
    "                neg_label -= 1\n",
    "                clusters[str(neg_label)].append(relationships[cnt])\n",
    "\n",
    "        # Cluster counters\n",
    "        cluster_ids = sorted([int(key) for key in clusters.keys()])\n",
    "        distinct_relationships = sorted(list(set(relationships)))\n",
    "        rel_counter = Counter(relationships)\n",
    "        cluster_counters = []\n",
    "        for cid in cluster_ids:\n",
    "            counter = Counter(clusters[str(cid)])\n",
    "            cluster_counters.append([counter[rel] for rel in distinct_relationships])\n",
    "\n",
    "        # Two predicates tend to be similar if they co-exist in a cluster\n",
    "        rel_sims = {}\n",
    "        for r1, rel1 in enumerate(distinct_relationships):\n",
    "            for r2, rel2 in enumerate(distinct_relationships):\n",
    "                if rel1 == rel2:\n",
    "                    rel_sims[(rel1, rel2)] = 1.0\n",
    "                    continue\n",
    "                rel_sims[(rel1, rel2)] = 0\n",
    "                for cluster in cluster_counters:\n",
    "                    if rel1 != rel2 and cluster[r1] and cluster[r2]:\n",
    "                        rel_sims[(rel1, rel2)] += min(cluster[r1], cluster[r2])\n",
    "                rel_sims[(rel1, rel2)] /= min(rel_counter[rel1], rel_counter[rel2])\n",
    "        for (pred0, pred1) in rel_sims:\n",
    "            rel0 = '_'.join([subj, pred0, obj])\n",
    "            rel1 = '_'.join([subj, pred1, obj])\n",
    "            if rel0 not in similarities:\n",
    "                similarities[rel0] = (-np.ones(len(PREDICATES))).tolist()\n",
    "                similarities[rel0][INV_PRED[pred0]] = 1.0\n",
    "                similarities[rel0][-1] = 0.0\n",
    "            if rel1 not in similarities:\n",
    "                similarities[rel1] = (-np.ones(len(PREDICATES))).tolist()\n",
    "                similarities[rel1][INV_PRED[pred1]] = 1.0\n",
    "                similarities[rel1][-1] = 0.0\n",
    "            similarities[rel0][INV_PRED[pred1]] = rel_sims[(pred0, pred1)]\n",
    "            similarities[rel1][INV_PRED[pred0]] = rel_sims[(pred0, pred1)]\n",
    "spatial_similarities = dict(similarities)\n",
    "spatial_similarities['person_on_horse']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Similarities using co-occurences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.5,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " 0.0,\n",
       " 1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " 0.3333333333333333,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.4230769230769231,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.6666666666666666,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " -1.0,\n",
       " 0.0,\n",
       " -1.0,\n",
       " 0.0]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def _same_pair_annos(anno):\n",
    "    \"\"\"Return a list of tuples referring to a common pair.\"\"\"\n",
    "    relations = anno['relations']\n",
    "    objects = anno['objects']['names']\n",
    "    pairs = np.stack(\n",
    "        (relations['subj_ids'], relations['obj_ids']), axis=1\n",
    "    )\n",
    "    common_pairs = (pairs[..., None] == pairs.T[None, ...]).all(1) * 1\n",
    "    nz_i, nz_j = common_pairs.nonzero()\n",
    "    return [\n",
    "        (\n",
    "            '_'.join([\n",
    "                objects[relations['subj_ids'][ind_i]],\n",
    "                relations['names'][ind_i],\n",
    "                objects[relations['obj_ids'][ind_j]]\n",
    "            ]),\n",
    "            '_'.join([\n",
    "                objects[relations['subj_ids'][ind_i]],\n",
    "                relations['names'][ind_j],\n",
    "                objects[relations['obj_ids'][ind_j]]\n",
    "            ])\n",
    "        )\n",
    "        for ind_i, ind_j in zip(nz_i, nz_j)\n",
    "        if ind_i <= ind_j\n",
    "        # and relations['names'][ind_i] != relations['names'][ind_j]\n",
    "    ]\n",
    "\n",
    "\n",
    "same_pair_counter = Counter([\n",
    "    (pair[0], pair[1])\n",
    "    if pair[0] < pair[1]\n",
    "    else (pair[1], pair[0])\n",
    "    for anno in ANNOS.values()\n",
    "    for pair in _same_pair_annos(anno)\n",
    "])\n",
    "rel_counter = Counter([\n",
    "    '_'.join([subj, pred, obj])\n",
    "    for anno in ANNOS.values()\n",
    "    for subj, pred, obj in zip(\n",
    "        anno['objects']['names'][anno['relations']['subj_ids']],\n",
    "        anno['relations']['names'],\n",
    "        anno['objects']['names'][anno['relations']['obj_ids']]\n",
    "    )\n",
    "])\n",
    "for pair in same_pair_counter:\n",
    "    same_pair_counter[pair] /= min(rel_counter[pair[0]], rel_counter[pair[1]])\n",
    "\n",
    "similarities = {}\n",
    "for pair in same_pair_counter:\n",
    "    pred0 = pair[0].split('_')[1]\n",
    "    pred1 = pair[1].split('_')[1]\n",
    "    subj, _, obj = pair[0].split('_')\n",
    "    if any(rel not in similarities for rel in pair):\n",
    "        sim_vec = (-np.ones(len(PREDICATES))).tolist()\n",
    "        for pred in PREDICATES:\n",
    "            if '_'.join([subj, pred, obj]) in rel_counter:\n",
    "                sim_vec[INV_PRED[pred]] = 0.0\n",
    "        sim_vec[INV_PRED[PREDICATES[-1]]] = 0.0\n",
    "    if pair[0] not in similarities:\n",
    "        similarities[pair[0]] = list(sim_vec)\n",
    "        similarities[pair[0]][INV_PRED[pred0]] = 1.0\n",
    "    if pair[1] not in similarities:\n",
    "        similarities[pair[1]] = list(sim_vec)\n",
    "        similarities[pair[1]][INV_PRED[pred1]] = 1.0\n",
    "    similarities[pair[0]][INV_PRED[pred1]] = min(same_pair_counter[pair], 1.0)\n",
    "    similarities[pair[1]][INV_PRED[pred0]] = min(same_pair_counter[pair], 1.0)\n",
    "pred_similarities = dict(similarities)\n",
    "pred_similarities['person_on_horse']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "has 0.8674698795180723\n",
      "in 0.875\n",
      "near 1.0\n",
      "wear 1.0\n",
      "with 1.0\n",
      "__background__ 0.0\n"
     ]
    }
   ],
   "source": [
    "vector = spatial_similarities['person_wear_jeans']\n",
    "for v, val in enumerate(vector):\n",
    "    if val > -1:\n",
    "        print(PREDICATES[v], val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "above 0.5\n",
      "adjacent to 0.0\n",
      "behind 0.0\n",
      "below 0.0\n",
      "beside 0.0\n",
      "by 0.0\n",
      "has 1.0\n",
      "hold 0.0\n",
      "in the front of 0.0\n",
      "look 0.0\n",
      "near 0.0\n",
      "next to 0.0\n",
      "on 1.0\n",
      "on the left of 0.0\n",
      "on the top of 0.3333333333333333\n",
      "ride 0.4230769230769231\n",
      "sit on 0.6666666666666666\n",
      "stand next to 0.0\n",
      "taller than 0.0\n",
      "touch 0.0\n",
      "walk 0.0\n",
      "wear 0.0\n",
      "__background__ 0.0\n"
     ]
    }
   ],
   "source": [
    "vector = pred_similarities['person_on_horse']\n",
    "# vector = spacy_similarities[0, 0, INV_PRED['wear']]\n",
    "for v, val in enumerate(vector):\n",
    "    if val > -1:\n",
    "        print(PREDICATES[v], val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert all(key in spatial_similarities for key in RELATIONSHIPS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('prerequisites/sgg_annos/' + 'VRD' + '_spatial_similarities.json', 'w') as fid:\n",
    "    json.dump(spatial_similarities, fid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('prerequisites/sgg_annos/' + 'VRD' + '_pred_cooccurences.json', 'w') as fid:\n",
    "    json.dump(pred_similarities, fid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plausibilities to be added "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}