tutorials/ranking/drmm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matchzoo version 1.0\n",
      "`ranking_task` initialized with metrics [normalized_discounted_cumulative_gain@3(0.0), normalized_discounted_cumulative_gain@5(0.0), mean_average_precision(0.0)]\n",
      "data loading ...\n",
      "data loaded as `train_pack_raw` `dev_pack_raw` `test_pack_raw`\n"
     ]
    }
   ],
   "source": [
    "%run init.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranking_task = mz.tasks.Ranking(losses=mz.losses.RankCrossEntropyLoss(num_neg=10))\n",
    "ranking_task.metrics = [\n",
    "    mz.metrics.NormalizedDiscountedCumulativeGain(k=3),\n",
    "    mz.metrics.NormalizedDiscountedCumulativeGain(k=5),\n",
    "    mz.metrics.MeanAveragePrecision()\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "preprocessor = mz.models.DRMM.get_default_preprocessor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 7113.33it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:04<00:00, 3978.25it/s]\n",
      "Processing text_right with append: 100%|██████████| 18841/18841 [00:00<00:00, 490493.51it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 18841/18841 [00:00<00:00, 109302.57it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 132131.62it/s]\n",
      "Processing text_left with extend: 100%|██████████| 2118/2118 [00:00<00:00, 524195.19it/s]\n",
      "Processing text_right with extend: 100%|██████████| 18841/18841 [00:00<00:00, 784459.51it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 418401/418401 [00:00<00:00, 2734112.41it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 8663.39it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:04<00:00, 4394.28it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 67567.99it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 184305.72it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 108278.18it/s]\n",
      "Processing length_left with len: 100%|██████████| 2118/2118 [00:00<00:00, 542009.51it/s]\n",
      "Processing length_right with len: 100%|██████████| 18841/18841 [00:00<00:00, 874372.16it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 122/122 [00:00<00:00, 8109.56it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 1115/1115 [00:00<00:00, 4648.20it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 122271.73it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 73121.62it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 129164.22it/s]\n",
      "Processing length_left with len: 100%|██████████| 122/122 [00:00<00:00, 196733.98it/s]\n",
      "Processing length_right with len: 100%|██████████| 1115/1115 [00:00<00:00, 599186.29it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 237/237 [00:00<00:00, 9027.54it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2300/2300 [00:00<00:00, 4620.73it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 127314.83it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 159738.08it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 91012.78it/s]\n",
      "Processing length_left with len: 100%|██████████| 237/237 [00:00<00:00, 181164.58it/s]\n",
      "Processing length_right with len: 100%|██████████| 2300/2300 [00:00<00:00, 689556.77it/s]\n"
     ]
    }
   ],
   "source": [
    "train_pack_processed = preprocessor.fit_transform(train_pack_raw)\n",
    "dev_pack_processed = preprocessor.transform(dev_pack_raw)\n",
    "test_pack_processed = preprocessor.transform(test_pack_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'filter_unit': <matchzoo.preprocessors.units.frequency_filter.FrequencyFilter at 0x7f543427e278>,\n",
       " 'vocab_unit': <matchzoo.preprocessors.units.vocabulary.Vocabulary at 0x7f5434656f28>,\n",
       " 'vocab_size': 30058,\n",
       " 'embedding_input_dim': 30058}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_embedding = mz.datasets.embeddings.load_glove_embedding(dimension=300)\n",
    "term_index = preprocessor.context['vocab_unit'].state['term_index']\n",
    "embedding_matrix = glove_embedding.build_matrix(term_index)\n",
    "l2_norm = np.sqrt((embedding_matrix * embedding_matrix).sum(axis=1))\n",
    "embedding_matrix = embedding_matrix / l2_norm[:, np.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "histgram_callback = mz.dataloader.callbacks.Histogram(\n",
    "    embedding_matrix, bin_size=30, hist_mode='LCH'\n",
    ")\n",
    "\n",
    "trainset = mz.dataloader.Dataset(\n",
    "    data_pack=train_pack_processed,\n",
    "    mode='pair',\n",
    "    num_dup=5,\n",
    "    num_neg=10,\n",
    "    callbacks=[histgram_callback]\n",
    ")\n",
    "testset = mz.dataloader.Dataset(\n",
    "    data_pack=test_pack_processed,\n",
    "    callbacks=[histgram_callback]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "padding_callback = mz.models.DRMM.get_default_padding_callback()\n",
    "\n",
    "trainloader = mz.dataloader.DataLoader(\n",
    "    device='cpu',\n",
    "    dataset=trainset,\n",
    "    batch_size=20,\n",
    "    stage='train',\n",
    "    resample=True,\n",
    "    callback=padding_callback\n",
    ")\n",
    "testloader = mz.dataloader.DataLoader(\n",
    "    dataset=testset,\n",
    "    batch_size=20,\n",
    "    stage='dev',\n",
    "    callback=padding_callback\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DRMM(\n",
      "  (embedding): Embedding(30058, 300)\n",
      "  (attention): Attention(\n",
      "    (linear): Linear(in_features=300, out_features=1, bias=False)\n",
      "  )\n",
      "  (mlp): Sequential(\n",
      "    (0): Sequential(\n",
      "      (0): Linear(in_features=30, out_features=10, bias=True)\n",
      "      (1): Tanh()\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Linear(in_features=10, out_features=1, bias=True)\n",
      "      (1): Tanh()\n",
      "    )\n",
      "  )\n",
      "  (out): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable params:  9018023\n"
     ]
    }
   ],
   "source": [
    "model = mz.models.DRMM()\n",
    "\n",
    "model.params['task'] = ranking_task\n",
    "model.params['mask_value'] = 0\n",
    "model.params['embedding'] = embedding_matrix\n",
    "model.params['hist_bin_size'] = 30\n",
    "model.params['mlp_num_layers'] = 1\n",
    "model.params['mlp_num_units'] = 10\n",
    "model.params['mlp_num_fan_out'] = 1\n",
    "model.params['mlp_activation_func'] = 'tanh'\n",
    "\n",
    "model.build()\n",
    "\n",
    "print(model)\n",
    "print('Trainable params: ', sum(p.numel() for p in model.parameters() if p.requires_grad))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adadelta(model.parameters())\n",
    "\n",
    "trainer = mz.trainers.Trainer(\n",
    "    device='cpu',\n",
    "    model=model,\n",
    "    optimizer=optimizer,\n",
    "    trainloader=trainloader,\n",
    "    validloader=testloader,\n",
    "    validate_interval=None,\n",
    "    epochs=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1502fe2052c444f69cb2a62f96eaeeb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=255), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-255 Loss-2.318]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5425 - normalized_discounted_cumulative_gain@5(0.0): 0.6073 - mean_average_precision(0.0): 0.561\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c156eb897204d6f855d3d627c9192a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=255), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-510 Loss-2.113]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5565 - normalized_discounted_cumulative_gain@5(0.0): 0.6226 - mean_average_precision(0.0): 0.5737\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbfa28c57f634fd19b1f7cd876c86328",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=255), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-765 Loss-2.032]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5546 - normalized_discounted_cumulative_gain@5(0.0): 0.6222 - mean_average_precision(0.0): 0.569\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29c243f5f19247c9a5ce2db7798b0286",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=255), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-1020 Loss-1.979]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5597 - normalized_discounted_cumulative_gain@5(0.0): 0.6275 - mean_average_precision(0.0): 0.5791\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d51c12648b5484db875f47d091f62bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=255), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-1275 Loss-1.934]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5531 - normalized_discounted_cumulative_gain@5(0.0): 0.6211 - mean_average_precision(0.0): 0.5761\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ba264e44b314355ab39c3fbdf09ef3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=255), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-1530 Loss-1.879]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5632 - normalized_discounted_cumulative_gain@5(0.0): 0.6278 - mean_average_precision(0.0): 0.5865\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a039b5240c744e4a141aea8cafc5881",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=255), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-1785 Loss-1.818]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5458 - normalized_discounted_cumulative_gain@5(0.0): 0.6146 - mean_average_precision(0.0): 0.5676\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80c905a2bb6f404fbbd0a9dd6cd9e2a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=255), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-2040 Loss-1.778]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5558 - normalized_discounted_cumulative_gain@5(0.0): 0.6136 - mean_average_precision(0.0): 0.5735\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "448a9d857a4745629ddac84e130a6c63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=255), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-2295 Loss-1.722]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5515 - normalized_discounted_cumulative_gain@5(0.0): 0.6078 - mean_average_precision(0.0): 0.5716\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "545044c58a044ad0ac1f3e3b7803a08a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=255), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-2550 Loss-1.684]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5578 - normalized_discounted_cumulative_gain@5(0.0): 0.622 - mean_average_precision(0.0): 0.5804\n",
      "\n",
      "Cost time: 8711.306358098984s\n"
     ]
    }
   ],
   "source": [
    "trainer.run()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}