autox/autox_recommend/data_process/Netflix-data-process.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## import包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "046298b9-7ef7-4e68-aef2-a1fe316be5a0",
    "_uuid": "3bc39967a41f9ec3989f971c49916b822b0806b7",
    "execution": {
     "iopub.execute_input": "2022-05-18T13:59:37.776967Z",
     "iopub.status.busy": "2022-05-18T13:59:37.776569Z",
     "iopub.status.idle": "2022-05-18T13:59:38.079855Z",
     "shell.execute_reply": "2022-05-18T13:59:38.078460Z",
     "shell.execute_reply.started": "2022-05-18T13:59:37.776906Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "import gc\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "be4477f1-7a11-48f4-8147-262a6198609f",
    "_uuid": "665b9a4bceca7bb318e39f1a5825170b18c6cc63"
   },
   "source": [
    "## 处理后的数据包含:\n",
    "1. 交互表(必须, user-item的交互记录, 包括训练集和测试集, 测试集中所有记录的时间都在训练集之后);\n",
    "2. item表(可选)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读数据\n",
    "数据地址：https://www.kaggle.com/datasets/netflix-inc/netflix-prize-data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:00:06.231257Z",
     "iopub.status.busy": "2022-05-18T14:00:06.230815Z",
     "iopub.status.idle": "2022-05-18T14:00:06.245693Z",
     "shell.execute_reply": "2022-05-18T14:00:06.244424Z",
     "shell.execute_reply.started": "2022-05-18T14:00:06.231192Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['combined_data_3.txt',\n",
       " 'movie_titles.csv',\n",
       " 'combined_data_4.txt',\n",
       " 'combined_data_1.txt',\n",
       " 'README',\n",
       " 'probe.txt',\n",
       " 'combined_data_2.txt',\n",
       " 'qualifying.txt']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = './netflix'\n",
    "os.listdir(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "0343ba37-0654-469c-98e5-812ecbaca528",
    "_uuid": "2a5476e11ee4539c129f2da35fccdacf2c296765",
    "execution": {
     "iopub.execute_input": "2022-05-18T14:04:34.515583Z",
     "iopub.status.busy": "2022-05-18T14:04:34.515174Z",
     "iopub.status.idle": "2022-05-18T14:06:19.012678Z",
     "shell.execute_reply": "2022-05-18T14:06:19.011432Z",
     "shell.execute_reply.started": "2022-05-18T14:04:34.515481Z"
    }
   },
   "outputs": [],
   "source": [
    "col_names = ['userId', 'rating', 'time']\n",
    "ratings1 = pd.read_csv(f'{path}/combined_data_1.txt', header = None, names = col_names)\n",
    "ratings2 = pd.read_csv(f'{path}/combined_data_2.txt', header = None, names = col_names)\n",
    "ratings3 = pd.read_csv(f'{path}/combined_data_3.txt', header = None, names = col_names)\n",
    "ratings4 = pd.read_csv(f'{path}/combined_data_4.txt', header = None, names = col_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据处理\n",
    "### 交互表\n",
    "原始数据格式是下面这样的格式，每个movieId单独一行，该行后的记录全是与这个movie交互的用户记录，处理时需将movieId对应的行填充入记录后再删除。\n",
    "\n",
    "MovieID1:\n",
    "\n",
    "CustomerID11,Date11\n",
    "\n",
    "CustomerID12,Date12\n",
    "\n",
    "…\n",
    "\n",
    "MovieID2:\n",
    "\n",
    "CustomerID21,Date21\n",
    "\n",
    "CustomerID22,Date22"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:06:19.015229Z",
     "iopub.status.busy": "2022-05-18T14:06:19.014940Z",
     "iopub.status.idle": "2022-05-18T14:06:34.549660Z",
     "shell.execute_reply": "2022-05-18T14:06:34.547075Z",
     "shell.execute_reply.started": "2022-05-18T14:06:19.015178Z"
    }
   },
   "outputs": [],
   "source": [
    "ratings = pd.concat([ratings1, ratings2, ratings3, ratings4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:06:39.420745Z",
     "iopub.status.busy": "2022-05-18T14:06:39.420344Z",
     "iopub.status.idle": "2022-05-18T14:06:41.181225Z",
     "shell.execute_reply": "2022-05-18T14:06:41.179605Z",
     "shell.execute_reply.started": "2022-05-18T14:06:39.420679Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del ratings1, ratings2, ratings3, ratings4\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:06:43.113468Z",
     "iopub.status.busy": "2022-05-18T14:06:43.113070Z",
     "iopub.status.idle": "2022-05-18T14:06:43.147677Z",
     "shell.execute_reply": "2022-05-18T14:06:43.146053Z",
     "shell.execute_reply.started": "2022-05-18T14:06:43.113407Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>rating</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1:</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1488844</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2005-09-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>822109</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2005-05-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>885013</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2005-10-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>30878</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2005-12-26</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    userId  rating        time\n",
       "0       1:     NaN         NaN\n",
       "1  1488844     3.0  2005-09-06\n",
       "2   822109     5.0  2005-05-13\n",
       "3   885013     4.0  2005-10-19\n",
       "4    30878     4.0  2005-12-26"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((100480507, 3), (23168232, 3))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.loc[ratings['rating'].notnull()].shape, ratings.loc[ratings['rating'] == 5].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:09:19.994871Z",
     "iopub.status.busy": "2022-05-18T14:09:19.994499Z",
     "iopub.status.idle": "2022-05-18T14:09:26.263780Z",
     "shell.execute_reply": "2022-05-18T14:09:26.262795Z",
     "shell.execute_reply.started": "2022-05-18T14:09:19.994810Z"
    }
   },
   "outputs": [],
   "source": [
    "ratings = ratings.loc[(ratings['rating'].isnull()) | (ratings['rating']==5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:09:29.163933Z",
     "iopub.status.busy": "2022-05-18T14:09:29.163494Z",
     "iopub.status.idle": "2022-05-18T14:09:30.251665Z",
     "shell.execute_reply": "2022-05-18T14:09:30.249616Z",
     "shell.execute_reply.started": "2022-05-18T14:09:29.163873Z"
    }
   },
   "outputs": [],
   "source": [
    "ratings.drop('rating', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:09:33.615923Z",
     "iopub.status.busy": "2022-05-18T14:09:33.615491Z",
     "iopub.status.idle": "2022-05-18T14:10:01.659669Z",
     "shell.execute_reply": "2022-05-18T14:10:01.658701Z",
     "shell.execute_reply.started": "2022-05-18T14:09:33.615857Z"
    }
   },
   "outputs": [],
   "source": [
    "ratings['movieId'] = ratings['userId'].apply(lambda x: x.replace(':', '') if ':' in x else np.nan)\n",
    "ratings['movieId'].fillna(method='ffill', inplace=True)\n",
    "ratings.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:10:02.279922Z",
     "iopub.status.busy": "2022-05-18T14:10:02.279066Z",
     "iopub.status.idle": "2022-05-18T14:10:02.292991Z",
     "shell.execute_reply": "2022-05-18T14:10:02.291916Z",
     "shell.execute_reply.started": "2022-05-18T14:10:02.279402Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>time</th>\n",
       "      <th>movieId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>822109</td>\n",
       "      <td>2005-05-13</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2207774</td>\n",
       "      <td>2005-06-06</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>372233</td>\n",
       "      <td>2005-11-23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>814701</td>\n",
       "      <td>2005-09-29</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>662870</td>\n",
       "      <td>2005-08-24</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     userId        time movieId\n",
       "2    822109  2005-05-13       1\n",
       "12  2207774  2005-06-06       1\n",
       "20   372233  2005-11-23       1\n",
       "28   814701  2005-09-29       1\n",
       "30   662870  2005-08-24       1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.status.idle": "2022-05-18T14:10:14.920019Z",
     "shell.execute_reply": "2022-05-18T14:10:14.918804Z",
     "shell.execute_reply.started": "2022-05-18T14:10:05.235483Z"
    }
   },
   "outputs": [],
   "source": [
    "ratings['time'] = pd.to_datetime(ratings['time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:10:14.922820Z",
     "iopub.status.busy": "2022-05-18T14:10:14.922507Z",
     "iopub.status.idle": "2022-05-18T14:10:15.028669Z",
     "shell.execute_reply": "2022-05-18T14:10:15.027584Z",
     "shell.execute_reply.started": "2022-05-18T14:10:14.922763Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Timestamp('1999-11-11 00:00:00'), Timestamp('2005-12-31 00:00:00'))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings['time'].min(), ratings['time'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:10:15.030366Z",
     "iopub.status.busy": "2022-05-18T14:10:15.030077Z",
     "iopub.status.idle": "2022-05-18T14:10:16.436643Z",
     "shell.execute_reply": "2022-05-18T14:10:16.435492Z",
     "shell.execute_reply.started": "2022-05-18T14:10:15.030318Z"
    }
   },
   "outputs": [],
   "source": [
    "data_used_time = datetime.datetime.strptime('2005-12-25 00:00:00', '%Y-%m-%d %H:%M:%S')\n",
    "\n",
    "train = ratings.loc[ratings['time'] < data_used_time]\n",
    "test = ratings.loc[ratings['time'] >= data_used_time]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:10:16.438452Z",
     "iopub.status.busy": "2022-05-18T14:10:16.438068Z",
     "iopub.status.idle": "2022-05-18T14:10:16.542922Z",
     "shell.execute_reply": "2022-05-18T14:10:16.541808Z",
     "shell.execute_reply.started": "2022-05-18T14:10:16.438349Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Timestamp('1999-11-11 00:00:00'), Timestamp('2005-12-24 00:00:00'))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['time'].min(), train['time'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:10:16.545628Z",
     "iopub.status.busy": "2022-05-18T14:10:16.545358Z",
     "iopub.status.idle": "2022-05-18T14:10:16.553226Z",
     "shell.execute_reply": "2022-05-18T14:10:16.552433Z",
     "shell.execute_reply.started": "2022-05-18T14:10:16.545581Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Timestamp('2005-12-25 00:00:00'), Timestamp('2005-12-31 00:00:00'))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['time'].min(), test['time'].max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## item表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:21:28.720624Z",
     "iopub.status.busy": "2022-05-18T14:21:28.720237Z",
     "iopub.status.idle": "2022-05-18T14:21:28.767387Z",
     "shell.execute_reply": "2022-05-18T14:21:28.765822Z",
     "shell.execute_reply.started": "2022-05-18T14:21:28.720565Z"
    }
   },
   "outputs": [],
   "source": [
    "movies = pd.read_csv(f'{path}/movie_titles.csv', \n",
    "                     encoding = 'ISO-8859-1', \n",
    "                     header = None,\n",
    "                     names = ['movieId', 'releaseYear', 'title'],\n",
    "                     dtype = {'movieId': 'str', 'releaseYear': 'str'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:21:53.643399Z",
     "iopub.status.busy": "2022-05-18T14:21:53.642863Z",
     "iopub.status.idle": "2022-05-18T14:21:53.655575Z",
     "shell.execute_reply": "2022-05-18T14:21:53.654597Z",
     "shell.execute_reply.started": "2022-05-18T14:21:53.643328Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>releaseYear</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2003</td>\n",
       "      <td>Dinosaur Planet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2004</td>\n",
       "      <td>Isle of Man TT 2004 Review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1997</td>\n",
       "      <td>Character</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1994</td>\n",
       "      <td>Paula Abdul's Get Up &amp; Dance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2004</td>\n",
       "      <td>The Rise and Fall of ECW</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  movieId releaseYear                         title\n",
       "0       1        2003               Dinosaur Planet\n",
       "1       2        2004    Isle of Man TT 2004 Review\n",
       "2       3        1997                     Character\n",
       "3       4        1994  Paula Abdul's Get Up & Dance\n",
       "4       5        2004      The Rise and Fall of ECW"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 存档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:24:18.900973Z",
     "iopub.status.busy": "2022-05-18T14:24:18.900576Z",
     "iopub.status.idle": "2022-05-18T14:24:18.910049Z",
     "shell.execute_reply": "2022-05-18T14:24:18.908733Z",
     "shell.execute_reply.started": "2022-05-18T14:24:18.900910Z"
    }
   },
   "outputs": [],
   "source": [
    "output_path = './Netflix_AutoX/'\n",
    "os.makedirs(output_path, exist_ok = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-18T14:36:30.181333Z",
     "iopub.status.busy": "2022-05-18T14:36:30.180687Z",
     "iopub.status.idle": "2022-05-18T14:36:30.184730Z",
     "shell.execute_reply": "2022-05-18T14:36:30.183923Z",
     "shell.execute_reply.started": "2022-05-18T14:36:30.181279Z"
    }
   },
   "outputs": [],
   "source": [
    "train.to_csv(output_path + 'inter_df.csv', index = False)\n",
    "test.to_csv(output_path + 'test.csv', index = False)\n",
    "movies.to_csv(output_path + 'item_df.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}