process_amazon_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# from dataset.visa_data import VisaTranasactionDataset\n",
    "# import importlib, sys\n",
    "# importlib.reload(sys.modules['dataset.visa_data'])\n",
    "# from dataset.visa_data import VisaTranasactionDataset\n",
    "from dataset.amazon_time_static import AmazonWithTimePosAndStaticSplitDataset\n",
    "from dataset.amazon_time_pos import AmazonWithTimePosDataset\n",
    "from dataset.amazon_static import AmazonWithStaticSplitDataset\n",
    "from dataset.amazon import AmazonDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "root=\"./data/amazon_movie/\"\n",
    "fname=\"Movies_and_TV_5\"\n",
    "val_fname=\"\"\n",
    "test_fname=\"\"\n",
    "fextension=\"\"\n",
    "preload_fextension=\"\"\n",
    "reviewer_level_cached=True\n",
    "vocab_cached=False\n",
    "external_vocab_path=\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not fextension:\n",
    "    if vocab_cached:\n",
    "        external_vocab_path=root+\"vocab_ob\"\n",
    "    fextension=\"\"\n",
    "else:\n",
    "    if vocab_cached:\n",
    "        external_vocab_path=root+\"vocab_ob_\"+ fextension\n",
    "if not preload_fextension:\n",
    "    preload_fextension=\"\"\n",
    "if not external_vocab_path:\n",
    "    external_vocab_path=\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "assert fextension in [\"static\", \"time\", \"static-test\", \"time-test\",\n",
    "                      \"time-pos\", \"time-pos-test\", \"\", \"test\", \"static-only\", \"static-only-test\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pad_seq_first is True for training data, False for validation and test data.\n",
    "\n",
    "For validation and test data, we only pad review sequences if the len < seq_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_class = ''\n",
    "if fextension in ['static', 'static-test']:\n",
    "    dataset_class = 'AmazonWithTimePosAndStaticSplitDataset'\n",
    "elif fextension in [\"static-only\", \"static-only-test\"]:\n",
    "    dataset_class = 'AmazonWithStaticSplitDataset'\n",
    "elif fextension in [\"time-pos\", \"time-pos-test\"]:\n",
    "    dataset_class = 'AmazonWithTimePosDataset'\n",
    "elif fextension in [\"\", \"test\"]:\n",
    "    dataset_class = 'AmazonDataset'\n",
    "dataset = eval(dataset_class)(cls_task=True,\n",
    "                              reviewer_ids=None,\n",
    "                              seq_len=10,\n",
    "                              root=root,\n",
    "                              fname=fname,\n",
    "                              reviewer_level_cached=reviewer_level_cached,\n",
    "                              vocab_cached=vocab_cached,\n",
    "                              external_vocab_path=external_vocab_path,\n",
    "                              preload_vocab_dir=root,\n",
    "                              save_vocab_dir=root,\n",
    "                              preload_fextension=preload_fextension,\n",
    "                              fextension=fextension,\n",
    "                              nrows=None,\n",
    "                              flatten=False,\n",
    "                              stride=1,\n",
    "                              return_labels=True,\n",
    "                              label_category='last_label',\n",
    "                              pad_seq_first=True,\n",
    "                              get_rids=True,\n",
    "                              long_and_sort=True,\n",
    "                              binary_task=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not external_vocab_path:\n",
    "    external_vocab_path=dataset.vocab_path\n",
    "vocab_cached=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dataset = eval(dataset_class)(cls_task=True,\n",
    "                                  reviewer_ids=None,\n",
    "                                  seq_len=10,\n",
    "                                  root=root,\n",
    "                                  fname=val_fname,\n",
    "                                  reviewer_level_cached=reviewer_level_cached,\n",
    "                                  vocab_cached=vocab_cached,\n",
    "                                  external_vocab_path=external_vocab_path,\n",
    "                                  preload_vocab_dir=root,\n",
    "                                  save_vocab_dir=root,\n",
    "                                  preload_fextension=preload_fextension,\n",
    "                                  fextension=fextension,\n",
    "                                  nrows=None,\n",
    "                                  flatten=False,\n",
    "                                  stride=1,\n",
    "                                  return_labels=True,\n",
    "                                  label_category='last_label',\n",
    "                                  pad_seq_first=False,\n",
    "                                  get_rids=True,\n",
    "                                  long_and_sort=True,\n",
    "                                  binary_task=True)\n",
    "\n",
    "test_dataset = eval(dataset_class)(cls_task=True,\n",
    "                                  reviewer_ids=None,\n",
    "                                  seq_len=10,\n",
    "                                  root=root,\n",
    "                                  fname=test_fname,\n",
    "                                  reviewer_level_cached=reviewer_level_cached,\n",
    "                                  vocab_cached=vocab_cached,\n",
    "                                  external_vocab_path=external_vocab_path,\n",
    "                                  preload_vocab_dir=root,\n",
    "                                  save_vocab_dir=root,\n",
    "                                  preload_fextension=preload_fextension,\n",
    "                                  fextension=fextension,\n",
    "                                  nrows=None,\n",
    "                                  flatten=False,\n",
    "                                  stride=1,\n",
    "                                  return_labels=True,\n",
    "                                  label_category='last_label',\n",
    "                                  pad_seq_first=False,\n",
    "                                  get_rids=True,\n",
    "                                  long_and_sort=True,\n",
    "                                  binary_task=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    print(len(dataset.vocab), len(dataset.dynamic_vocab), len(dataset.time_feature_vocab), len(dataset.static_vocab))\n",
    "except:\n",
    "    print(len(dataset.vocab))\n",
    "print(len(val_dataset.vocab))\n",
    "print(len(test_dataset.vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dataset.ncols, val_dataset.ncols, test_dataset.ncols)\n",
    "print(len(dataset), len(val_dataset), len(test_dataset))\n",
    "print(dataset[0])\n",
    "print('dataset len', [i.shape for i in dataset[0]])\n",
    "print(val_dataset[0])\n",
    "print('val_dataset len', [i.shape for i in val_dataset[0]])\n",
    "print(test_dataset[0])\n",
    "print('test_dataset len', [i.shape for i in test_dataset[0]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.7-visa",
   "language": "python",
   "name": "py3.7-visa"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}