preprocess_amazon_liang.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6e79cd24-568f-4179-9684-b00b49d52cf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import ast\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gzip"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2019cca-a2f7-4788-b7f4-7f8d3e741f3c",
   "metadata": {},
   "source": [
    "## Read json dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2edb52ba-0455-4ad0-9c74-4add8350096d",
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "file_name = \"amazon/Movies_and_TV_5.json.gz\"\n",
    "seq_len = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9046eee9-a8e6-4816-834b-7766cbc23727",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse(path):\n",
    "    g = gzip.open(path, 'rb')\n",
    "    for l in g:\n",
    "        yield json.loads(l)\n",
    "\n",
    "def getDF(path):\n",
    "    i = 0\n",
    "    df = {}\n",
    "    for d in parse(path):\n",
    "        df[i] = d\n",
    "        i += 1\n",
    "    return pd.DataFrame.from_dict(df, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "47ec1a0b-ad28-4f90-903c-f89bf4c82afa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3410019, 12)\n"
     ]
    }
   ],
   "source": [
    "data_table = getDF(file_name)\n",
    "print(data_table.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f2350201-2d30-4383-999d-f6893dca70b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall</th>\n",
       "      <th>verified</th>\n",
       "      <th>reviewTime</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>style</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>unixReviewTime</th>\n",
       "      <th>vote</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>11 9, 2012</td>\n",
       "      <td>A2M1CU2IRZG0K9</td>\n",
       "      <td>0005089549</td>\n",
       "      <td>{'Format:': ' VHS Tape'}</td>\n",
       "      <td>Terri</td>\n",
       "      <td>So sorry I didn't purchase this years ago when...</td>\n",
       "      <td>Amazing!</td>\n",
       "      <td>1352419200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>12 30, 2011</td>\n",
       "      <td>AFTUJYISOFHY6</td>\n",
       "      <td>0005089549</td>\n",
       "      <td>{'Format:': ' VHS Tape'}</td>\n",
       "      <td>Melissa D. Abercrombie</td>\n",
       "      <td>Believe me when I tell you that you will recei...</td>\n",
       "      <td>Great Gospel VHS of the Cathedrals!</td>\n",
       "      <td>1325203200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>04 21, 2005</td>\n",
       "      <td>A3JVF9Y53BEOGC</td>\n",
       "      <td>000503860X</td>\n",
       "      <td>{'Format:': ' DVD'}</td>\n",
       "      <td>Anthony Thompson</td>\n",
       "      <td>I have seen X live many times, both in the ear...</td>\n",
       "      <td>A great document of a great band</td>\n",
       "      <td>1114041600</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>04 6, 2005</td>\n",
       "      <td>A12VPEOEZS1KTC</td>\n",
       "      <td>000503860X</td>\n",
       "      <td>{'Format:': ' DVD'}</td>\n",
       "      <td>JadeRain</td>\n",
       "      <td>I was so excited for this!  Finally, a live co...</td>\n",
       "      <td>YES!!  X LIVE!!</td>\n",
       "      <td>1112745600</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>12 3, 2010</td>\n",
       "      <td>ATLZNVLYKP9AZ</td>\n",
       "      <td>000503860X</td>\n",
       "      <td>{'Format:': ' DVD'}</td>\n",
       "      <td>T. Fisher</td>\n",
       "      <td>X is one of the best punk bands ever. I don't ...</td>\n",
       "      <td>X have still got it</td>\n",
       "      <td>1291334400</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   overall  verified   reviewTime      reviewerID        asin  \\\n",
       "0      5.0      True   11 9, 2012  A2M1CU2IRZG0K9  0005089549   \n",
       "1      5.0      True  12 30, 2011   AFTUJYISOFHY6  0005089549   \n",
       "2      5.0      True  04 21, 2005  A3JVF9Y53BEOGC  000503860X   \n",
       "3      5.0      True   04 6, 2005  A12VPEOEZS1KTC  000503860X   \n",
       "4      5.0      True   12 3, 2010   ATLZNVLYKP9AZ  000503860X   \n",
       "\n",
       "                      style            reviewerName  \\\n",
       "0  {'Format:': ' VHS Tape'}                   Terri   \n",
       "1  {'Format:': ' VHS Tape'}  Melissa D. Abercrombie   \n",
       "2       {'Format:': ' DVD'}        Anthony Thompson   \n",
       "3       {'Format:': ' DVD'}                JadeRain   \n",
       "4       {'Format:': ' DVD'}               T. Fisher   \n",
       "\n",
       "                                          reviewText  \\\n",
       "0  So sorry I didn't purchase this years ago when...   \n",
       "1  Believe me when I tell you that you will recei...   \n",
       "2  I have seen X live many times, both in the ear...   \n",
       "3  I was so excited for this!  Finally, a live co...   \n",
       "4  X is one of the best punk bands ever. I don't ...   \n",
       "\n",
       "                               summary  unixReviewTime vote image  \n",
       "0                             Amazing!      1352419200  NaN   NaN  \n",
       "1  Great Gospel VHS of the Cathedrals!      1325203200  NaN   NaN  \n",
       "2     A great document of a great band      1114041600   11   NaN  \n",
       "3                      YES!!  X LIVE!!      1112745600    5   NaN  \n",
       "4                  X have still got it      1291334400    5   NaN  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5e29a062-7b95-44aa-be33-b299694e3123",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_table.drop_duplicates(subset=['overall', 'verified', 'reviewTime', 'reviewerID', 'asin',\n",
    "#        'reviewerName', 'reviewText', 'summary', 'unixReviewTime', 'vote'], inplace=True)\n",
    "# print(data_table.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "31fa48ef-e620-421b-8688-5b2bc9adfd71",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_table['reviewLen'] = data_table['reviewText'].str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e0c0c4ff-b908-47fd-b084-3bdd6095192f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_table.drop(columns=['style', 'reviewerName', 'reviewText', 'summary', 'image', 'vote'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0d38e892-df98-424e-916d-3aa41a3a976b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "112194\n"
     ]
    }
   ],
   "source": [
    "print(data_table.duplicated().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d32f71d7-4d43-4b10-bae6-8112e75fccc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_table.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71e43530-3b3c-483f-8adc-cc39c6bece75",
   "metadata": {},
   "source": [
    "## Filter out user with few ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7066b397-54d4-4f6a-ad73-aa13d896d473",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_cnt = data_table.groupby('reviewerID').agg(\n",
    " user_count=('overall', 'count')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01152fec-98ca-4ce5-8367-8955bd31d22c",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_table1 = data_table.join(user_cnt, on='reviewerID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3474e04e-751e-447f-ac02-b7e81b62b585",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_table1.drop(data_table1.loc[data_table1['user_count']<=4].index, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3129d3e-cc38-4630-918d-e6c256426205",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data_table1.shape)\n",
    "data_table1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac8891a8-e3f9-456b-b421-89b10959dd9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_table1.drop(columns=['user_count'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "971538d0-c053-4635-8822-5db817c51796",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_table1['reviewTimeDayOfWeek'] = pd.to_datetime(data_table1['unixReviewTime'], unit='s').dt.dayofweek\n",
    "data_table1['reviewTimeHour'] = pd.to_datetime(data_table1['unixReviewTime'], unit='s').dt.hour\n",
    "data_table1.sort_values(by=['reviewerID', 'unixReviewTime'],inplace=True)\n",
    "\n",
    "data_table1['rownumber'] = np.arange(data_table1.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2274f06d-6484-43f4-8558-d6e5c4c45496",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data_table1.shape)\n",
    "data_table1.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf8fe31c-c51d-4843-b3c7-dba08b7858ac",
   "metadata": {},
   "source": [
    "## Split data into train, val, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bae26d51-0374-4739-9c76-602f03ab8c47",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_data(x):\n",
    "    return x.index[:-2]\n",
    "\n",
    "def get_val_data(x):\n",
    "    return x.index[-1 - seq_len:-1]\n",
    "\n",
    "def get_test_data(x):\n",
    "    return x.index[-seq_len:]\n",
    "\n",
    "def get_all_data(x):\n",
    "    d = {}\n",
    "    d['train_idx'] = get_train_data(x)\n",
    "    d['val_idx'] = get_val_data(x)\n",
    "    d['test_idx'] = get_test_data(x)\n",
    "    return pd.Series(d, index=['train_idx', 'val_idx', 'test_idx'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f17d8ff-5e66-489f-abf8-fc0e1f8b1c82",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_group = data_table1.groupby('reviewerID')\n",
    "user_group.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b02aed9-b1d4-416f-89fc-0c0bd754c16b",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_idx = user_group['reviewerID'].apply(get_all_data)\n",
    "all_idx.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c137933-8c62-4473-9682-8575b30540a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_idx = np.concatenate(all_idx.loc[:,'train_idx'])\n",
    "val_idx = np.concatenate(all_idx.loc[:,'val_idx'])\n",
    "test_idx = np.concatenate(all_idx.loc[:,'test_idx'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e90483c3-03a5-46fa-9a5a-b943a97cf48d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_table0 = data_table1.loc[train_idx]\n",
    "val_table0 = data_table1.loc[val_idx]\n",
    "test_table0 = data_table1.loc[test_idx]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85e9d018-60dc-40ee-ab93-4973ed72df1c",
   "metadata": {},
   "source": [
    "## Calculate global statistics or static table using training data only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d92dbb87-3a33-4dff-8c0a-7fb021810148",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_high_per(x):\n",
    "    return (x>=4).sum()/len(x)\n",
    "\n",
    "def get_low_per(x):\n",
    "    return (x<=3).sum()/len(x)\n",
    "\n",
    "def get_pos_reviewer(x):\n",
    "    #return (x>=4).sum() > (x<=3).sum()\n",
    "    if (x>=4).sum() > (x<=3).sum():\n",
    "        return 'isPos'\n",
    "    else:\n",
    "        return 'isNeg'\n",
    "\n",
    "def get_top_item(x):\n",
    "    x_ov = x.apply(lambda x: x.split('_')[0])\n",
    "    x_as = x.apply(lambda x: x.split('_')[1])\n",
    "    top_idx = x_ov.sort_values(ascending=False).iloc[0:3].index\n",
    "    top_item = x_as[top_idx].tolist()\n",
    "    return top_item\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "aa2f3cd7-5b03-46ba-9f2b-978972bd0719",
   "metadata": {},
   "outputs": [],
   "source": [
    "static_table = train_table0.groupby('reviewerID').agg(\n",
    "    avg_rating=('overall', 'mean'),\n",
    "    as_count=('overall', 'count'),\n",
    "    low_per=('overall', lambda x: get_low_per(x)),\n",
    "    high_per=('overall', lambda x: get_high_per(x)),\n",
    "    pos_reviewer=('overall', lambda x: get_pos_reviewer(x)),\n",
    "    # top_item=('o_a', lambda x: get_top_item(x)),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbf8400b-29df-4cad-87e8-8176faeef956",
   "metadata": {},
   "outputs": [],
   "source": [
    "static_table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a366f526-f50f-4771-b230-258e99b27f50",
   "metadata": {},
   "outputs": [],
   "source": [
    "static_table.isnull().any()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "482b9042-ff49-45a6-9d00-b4fd425c0aac",
   "metadata": {},
   "source": [
    "## Global default: reviewer rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b58aefd8-12da-41fd-ae90-d1a73dfde1e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_avg_rating = train_table0['overall'].mean()\n",
    "print(df_avg_rating)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f1c75ff-1990-4f00-ae12-b32a6acc202a",
   "metadata": {},
   "source": [
    "## Asin rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6de6d2a9-baf6-4a8c-8225-ce933881020a",
   "metadata": {},
   "outputs": [],
   "source": [
    "asin_rating = train_table0.groupby('asin').agg(\n",
    " asin_rating=('overall', 'mean'),\n",
    " asin_cnt=('overall', 'count'),\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7faf50ad-4212-43a7-9089-ca236076434a",
   "metadata": {},
   "outputs": [],
   "source": [
    "asin_rating.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ada4c95-6597-4a80-ae52-97ee940caa2e",
   "metadata": {},
   "source": [
    "## Global default: asin rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be0192c4-0086-4bcd-ae83-2434720a5dde",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_asin_rating = train_table0['overall'].mean()\n",
    "print(df_asin_rating)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c88adc32-66ca-4a82-894a-d539c79d6540",
   "metadata": {},
   "source": [
    "## Join tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0d47256-51f4-4554-962c-16dc6e9ef9c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_table1 = train_table0.join(static_table, on='reviewerID')\n",
    "val_table1 = val_table0.join(static_table, on='reviewerID')\n",
    "test_table1 = test_table0.join(static_table, on='reviewerID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "920cc83a-4b7d-46a2-b0ae-1076ad43a68e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_table2 = train_table1.join(asin_rating, on='asin')\n",
    "val_table2 = val_table1.join(asin_rating, on='asin')\n",
    "test_table2 = test_table1.join(asin_rating, on='asin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6bb4398-944d-4632-900c-88b3b628bf94",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train_table2.shape, val_table2.shape, test_table2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "176b4136-f225-4668-b317-25fd56cdc9e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_table2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cb16759-1429-4402-b575-e4ec77da1d66",
   "metadata": {},
   "source": [
    "## Filling default values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "631b440d-3f04-40b5-8148-b20d0ba0f0db",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train_table2.isnull().any())\n",
    "print(val_table2.isnull().any())\n",
    "print(test_table2.isnull().any())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cdd6dfb-f839-494d-8cd0-eeb97dd9163e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_values = {\"avg_rating\": df_avg_rating, \"asin_rating \": df_asin_rating}\n",
    "train_table2.fillna(value=df_values, inplace=True)\n",
    "#dy: what about val and test table?\n",
    "val_table2.fillna(value=df_values, inplace=True)\n",
    "test_table2.fillna(value=df_values, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21cc130d-7604-444b-9b7f-e86a18b72d5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_cols = ['as_count', 'asin_cnt']\n",
    "for df in [train_table2, val_table2, test_table2]:\n",
    "    df.drop(columns=drop_cols, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1b17b302-ffcb-486d-81ae-d6be3e03fd86",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2814911, 17) (1996181, 17) (2214747, 17)\n"
     ]
    }
   ],
   "source": [
    "print(train_table2.shape, val_table2.shape, test_table2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7ef6635d-44e0-4d13-b10f-e3b615a35f07",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "amazon/Movies_and_TV_5_train.csv amazon/Movies_and_TV_5_val.csv amazon/Movies_and_TV_5_test.csv amazon/Movies_and_TV_5.csv\n"
     ]
    }
   ],
   "source": [
    "train_file_name = file_name.split('.')[0] + '_train.csv'\n",
    "val_file_name = file_name.split('.')[0] + '_val.csv'\n",
    "test_file_name = file_name.split('.')[0] + '_test.csv'\n",
    "all_file_name = file_name.split('.')[0] + '.csv'\n",
    "print(train_file_name, val_file_name, test_file_name, all_file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "6f9f30d4-76a3-47bc-ab0a-5fb25cfaa717",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_table2.to_csv(train_file_name, index=False)\n",
    "val_table2.to_csv(val_file_name, index=False)\n",
    "test_table2.to_csv(test_file_name, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.7-visa",
   "language": "python",
   "name": "py3.7-visa"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}