002. Transformations on RAW data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3961b4cd",
   "metadata": {},
   "source": [
    "# Transformations on RAW data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "051f969c",
   "metadata": {},
   "source": [
    "## Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "77f546ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import fnmatch\n",
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "336e7b20",
   "metadata": {},
   "source": [
    "## Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "02c43311",
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_FOLDER_READ='01.Raw data/'\n",
    "PATH_FOLDER_WRITE='02.Clean data/'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "453c773e",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d4ba81ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleanNotes(s):\n",
    "    s=re.sub(r\"\\[(.*?)\\]\",'',s)\n",
    "    s.strip()\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4071c3e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleanDates(s):\n",
    "    s=re.search(r'\\((.*?)\\)',s).group(1)\n",
    "    s.strip()\n",
    "    return s"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dcacdff6",
   "metadata": {},
   "source": [
    "## Loop all Json file of folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "90f9815c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_c = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "18654d55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episodes_RAW_S10.json\n",
      "Episodes_RAW_S06.json\n",
      "Episodes_RAW_S26.json\n",
      "Episodes_RAW_S30.json\n",
      "Episodes_RAW_S31.json\n",
      "Episodes_RAW_S27.json\n",
      "Episodes_RAW_S07.json\n",
      "Episodes_RAW_S11.json\n",
      "Episodes_RAW_S16.json\n",
      "Episodes_RAW_S01.json\n",
      "Episodes_RAW_S17.json\n",
      "Episodes_RAW_S21.json\n",
      "Episodes_RAW_S34.json\n",
      "Episodes_RAW_S22.json\n",
      "Episodes_RAW_S18.json\n",
      "Episodes_RAW_S02.json\n",
      "Episodes_RAW_S14.json\n",
      "Episodes_RAW_S15.json\n",
      "Episodes_RAW_S03.json\n",
      "Episodes_RAW_S19.json\n",
      "Episodes_RAW_S23.json\n",
      "Episodes_RAW_S28.json\n",
      "Episodes_RAW_S04.json\n",
      "Episodes_RAW_S12.json\n",
      "Episodes_RAW_S32.json\n",
      "Episodes_RAW_S24.json\n",
      "Episodes_RAW_S08.json\n",
      "Episodes_RAW_S09.json\n",
      "Episodes_RAW_S25.json\n",
      "Episodes_RAW_S33.json\n",
      "Episodes_RAW_S13.json\n",
      "Episodes_RAW_S05.json\n",
      "Episodes_RAW_S29.json\n"
     ]
    }
   ],
   "source": [
    "for file in os.listdir(PATH_FOLDER_READ):\n",
    "    if fnmatch.fnmatch(file.upper(), '*.JSON'):\n",
    "        print(file)\n",
    "        \n",
    "        # Read file\n",
    "        with open(PATH_FOLDER_READ+file, 'r') as fp:\n",
    "            data = json.load(fp)\n",
    "        \n",
    "        # Transform data into a DF\n",
    "        df = pd.DataFrame.from_dict(data[0])\n",
    "\n",
    "        # Headr on first row (ref https://stackoverflow.com/a/31328974/1008588)\n",
    "        new_header = df.iloc[0]\n",
    "        df = df[1:]\n",
    "        df.columns = new_header\n",
    "        \n",
    "        # Clean Wikipedia notes reference (clean up of anything between square brackets)\n",
    "        df.rename(columns=lambda c: re.sub(r\"\\[(.*?)\\]\",'',c), inplace=True)\n",
    "        \n",
    "        # Strip spaces in columns header\n",
    "        df.rename(columns=lambda c: c.strip(), inplace=True)\n",
    "        \n",
    "        # Add Season number as new column\n",
    "        df['No. Season'] = int(file[14:16])\n",
    "        \n",
    "        # Concat with existing DF df_c\n",
    "        df_c= pd.concat([df_c, df])\n",
    "df=df_c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c998e9a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# reset index\n",
    "df.reset_index(inplace = True, drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "14732952",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No.overall</th>\n",
       "      <th>No. inseason</th>\n",
       "      <th>Title</th>\n",
       "      <th>Directed by</th>\n",
       "      <th>Written by</th>\n",
       "      <th>Original air date</th>\n",
       "      <th>Prod.code</th>\n",
       "      <th>U.S. viewers(millions)</th>\n",
       "      <th>No. Season</th>\n",
       "      <th>Screenplay by</th>\n",
       "      <th>Release date(U.S.)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>204</td>\n",
       "      <td>1</td>\n",
       "      <td>\"Lard of the Dance\"</td>\n",
       "      <td>Dominic Polcino</td>\n",
       "      <td>Jane O'Brien</td>\n",
       "      <td>August 23, 1998 (1998-08-23)</td>\n",
       "      <td>5F20</td>\n",
       "      <td>7[210]</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>205</td>\n",
       "      <td>2</td>\n",
       "      <td>\"The Wizard of Evergreen Terrace\"</td>\n",
       "      <td>Mark Kirkland</td>\n",
       "      <td>John Swartzwelder</td>\n",
       "      <td>September 20, 1998 (1998-09-20)</td>\n",
       "      <td>5F21</td>\n",
       "      <td>7.95[211]</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>206</td>\n",
       "      <td>3</td>\n",
       "      <td>\"Bart the Mother\"</td>\n",
       "      <td>Steven Dean Moore</td>\n",
       "      <td>David X. Cohen</td>\n",
       "      <td>September 27, 1998 (1998-09-27)</td>\n",
       "      <td>5F22</td>\n",
       "      <td>7.35[212]</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>207</td>\n",
       "      <td>4</td>\n",
       "      <td>\"Treehouse of Horror IX\"</td>\n",
       "      <td>Steven Dean Moore</td>\n",
       "      <td>Donick CaryLarry DoyleDavid X. Cohen</td>\n",
       "      <td>October 25, 1998 (1998-10-25)</td>\n",
       "      <td>AABF01</td>\n",
       "      <td>8.5[213]</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>208</td>\n",
       "      <td>5</td>\n",
       "      <td>\"When You Dish Upon a Star\"</td>\n",
       "      <td>Pete Michels</td>\n",
       "      <td>Richard Appel</td>\n",
       "      <td>November 8, 1998 (1998-11-08)</td>\n",
       "      <td>5F19</td>\n",
       "      <td>9[214]</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "0 No.overall No. inseason                              Title  \\\n",
       "0        204            1                \"Lard of the Dance\"   \n",
       "1        205            2  \"The Wizard of Evergreen Terrace\"   \n",
       "2        206            3                  \"Bart the Mother\"   \n",
       "3        207            4           \"Treehouse of Horror IX\"   \n",
       "4        208            5        \"When You Dish Upon a Star\"   \n",
       "\n",
       "0        Directed by                            Written by  \\\n",
       "0    Dominic Polcino                          Jane O'Brien   \n",
       "1      Mark Kirkland                     John Swartzwelder   \n",
       "2  Steven Dean Moore                        David X. Cohen   \n",
       "3  Steven Dean Moore  Donick CaryLarry DoyleDavid X. Cohen   \n",
       "4       Pete Michels                         Richard Appel   \n",
       "\n",
       "0                Original air date Prod.code U.S. viewers(millions)  \\\n",
       "0     August 23, 1998 (1998-08-23)      5F20                 7[210]   \n",
       "1  September 20, 1998 (1998-09-20)      5F21              7.95[211]   \n",
       "2  September 27, 1998 (1998-09-27)      5F22              7.35[212]   \n",
       "3    October 25, 1998 (1998-10-25)    AABF01               8.5[213]   \n",
       "4    November 8, 1998 (1998-11-08)      5F19                 9[214]   \n",
       "\n",
       "0  No. Season Screenplay by Release date(U.S.)  \n",
       "0          10           NaN                NaN  \n",
       "1          10           NaN                NaN  \n",
       "2          10           NaN                NaN  \n",
       "3          10           NaN                NaN  \n",
       "4          10           NaN                NaN  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dde05db3",
   "metadata": {},
   "source": [
    "## Manage specific cases"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "981b4afc",
   "metadata": {},
   "source": [
    "### Delete The Simpsons movie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "38e8ede2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No.overall</th>\n",
       "      <th>No. inseason</th>\n",
       "      <th>Title</th>\n",
       "      <th>Directed by</th>\n",
       "      <th>Written by</th>\n",
       "      <th>Original air date</th>\n",
       "      <th>Prod.code</th>\n",
       "      <th>U.S. viewers(millions)</th>\n",
       "      <th>No. Season</th>\n",
       "      <th>Screenplay by</th>\n",
       "      <th>Release date(U.S.)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>408</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Simpsons Movie</td>\n",
       "      <td>David Silverman</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19</td>\n",
       "      <td>James L. Brooks, Matt Groening, Al Jean, Ian M...</td>\n",
       "      <td>July 27, 2007 (2007-07-27)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "0   No.overall No. inseason               Title      Directed by Written by  \\\n",
       "408        NaN          NaN  The Simpsons Movie  David Silverman        NaN   \n",
       "\n",
       "0   Original air date Prod.code U.S. viewers(millions)  No. Season  \\\n",
       "408               NaN       NaN                    NaN          19   \n",
       "\n",
       "0                                        Screenplay by  \\\n",
       "408  James L. Brooks, Matt Groening, Al Jean, Ian M...   \n",
       "\n",
       "0            Release date(U.S.)  \n",
       "408  July 27, 2007 (2007-07-27)  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['Title'] == 'The Simpsons Movie']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "49d1c4e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(df[df['Title'] == 'The Simpsons Movie'].index)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2fa3263d",
   "metadata": {},
   "source": [
    "## Drop unuseful columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7bcb9b1",
   "metadata": {},
   "source": [
    "Drop these columns as they were only used in the Simpsons movie table."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6291a397",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(['Screenplay by', 'Release date(U.S.)'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6cf3f01d",
   "metadata": {},
   "source": [
    "### Episodes made of two parts: 681 and 682"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "67c5e65a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No.overall</th>\n",
       "      <th>No. inseason</th>\n",
       "      <th>Title</th>\n",
       "      <th>Directed by</th>\n",
       "      <th>Written by</th>\n",
       "      <th>Original air date</th>\n",
       "      <th>Prod.code</th>\n",
       "      <th>U.S. viewers(millions)</th>\n",
       "      <th>No. Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>681682</td>\n",
       "      <td>1920</td>\n",
       "      <td>\"Warrin' Priests\" (Parts One and Two)</td>\n",
       "      <td>Bob AndersonMatthew Nastuk</td>\n",
       "      <td>Pete Holmes</td>\n",
       "      <td>April 26, 2020 (2020-04-26)May 3, 2020 (2020-0...</td>\n",
       "      <td>ZABF12ZABF13</td>\n",
       "      <td>1.35[286]1.36[287]</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "0   No.overall No. inseason                                  Title  \\\n",
       "111     681682         1920  \"Warrin' Priests\" (Parts One and Two)   \n",
       "\n",
       "0                   Directed by   Written by  \\\n",
       "111  Bob AndersonMatthew Nastuk  Pete Holmes   \n",
       "\n",
       "0                                    Original air date     Prod.code  \\\n",
       "111  April 26, 2020 (2020-04-26)May 3, 2020 (2020-0...  ZABF12ZABF13   \n",
       "\n",
       "0   U.S. viewers(millions)  No. Season  \n",
       "111     1.35[286]1.36[287]          31  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['No.overall'] == '681682']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f68b5c16",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(df[df['No.overall'] == '681682'].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "75c454f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "newRow={'No.overall':'681',\n",
    "    'No. inseason':'19',\n",
    "    'Title':'\"Warrin\\' Priests\" (Parts One)',\n",
    "    'Directed by': 'Bob Anderson',\n",
    "    'Written by':'Pete Holmes',\n",
    "    'Original air date':'April 26, 2020 (2020-04-26)',\n",
    "    'Prod.code':'ZABF12',\n",
    "    'U.S. viewers(millions)':'1.35[287]',\n",
    "    'No. Season':'31'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3f4b69f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "dn=pd.DataFrame(newRow, index=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c7f37b24",
   "metadata": {},
   "outputs": [],
   "source": [
    "df= pd.concat([df, dn])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2498f797",
   "metadata": {},
   "outputs": [],
   "source": [
    "newRow={'No.overall':'682',\n",
    "    'No. inseason':'20',\n",
    "    'Title':'\"Warrin\\' Priests\" (Parts Two)',\n",
    "    'Directed by':'Matthew Nastuk',\n",
    "    'Written by':'Pete Holmes',\n",
    "    'Original air date':'May 03, 2020 (2020-05-03)',\n",
    "    'Prod.code':'ZABF13',\n",
    "    'U.S. viewers(millions)':'1.36[288]',\n",
    "    'No. Season':'31'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "eebe66f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dn=pd.DataFrame(newRow, index=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5ab2d880",
   "metadata": {},
   "outputs": [],
   "source": [
    "df= pd.concat([df, dn])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa8d88b2",
   "metadata": {},
   "source": [
    "### Episodes made of two parts: 608 and 609"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "93302050",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No.overall</th>\n",
       "      <th>No. inseason</th>\n",
       "      <th>Title</th>\n",
       "      <th>Directed by</th>\n",
       "      <th>Written by</th>\n",
       "      <th>Original air date</th>\n",
       "      <th>Prod.code</th>\n",
       "      <th>U.S. viewers(millions)</th>\n",
       "      <th>No. Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>442</th>\n",
       "      <td>608609</td>\n",
       "      <td>1213</td>\n",
       "      <td>\"The Great Phatsby\" (Vol. I &amp; Vol. II)</td>\n",
       "      <td>Chris ClementsTimothy Bailey</td>\n",
       "      <td>Dan GreaneyMatt Selman (part 2 only)</td>\n",
       "      <td>January 15, 2017 (2017-01-15)</td>\n",
       "      <td>WABF04WABF05</td>\n",
       "      <td>6.90[214]</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    No.overall No. inseason                                   Title  \\\n",
       "442     608609         1213  \"The Great Phatsby\" (Vol. I & Vol. II)   \n",
       "\n",
       "                      Directed by                            Written by  \\\n",
       "442  Chris ClementsTimothy Bailey  Dan GreaneyMatt Selman (part 2 only)   \n",
       "\n",
       "                 Original air date     Prod.code U.S. viewers(millions)  \\\n",
       "442  January 15, 2017 (2017-01-15)  WABF04WABF05              6.90[214]   \n",
       "\n",
       "    No. Season  \n",
       "442         28  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['No.overall'] == '608609']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "65b7e154",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(df[df['No.overall'] == '608609'].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cdcc1329",
   "metadata": {},
   "outputs": [],
   "source": [
    "newRow={'No.overall':'608',\n",
    "    'No. inseason':'12',\n",
    "    'Title':'\"The Great Phatsby\" (Vol. I)',\n",
    "    'Directed by':'Chris Clements',\n",
    "    'Written by':'Dan Greaney',\n",
    "    'Original air date':'January 15, 2017 (2017-01-15)',\n",
    "    'Prod.code':'WABF04',\n",
    "    'U.S. viewers(millions)':'6.90[214]',\n",
    "    'No. Season':'28'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d47833f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "dn=pd.DataFrame(newRow, index=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "335ba2df",
   "metadata": {},
   "outputs": [],
   "source": [
    "df= pd.concat([df, dn])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f70d37d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "newRow={'No.overall':'609',\n",
    "    'No. inseason':'12',\n",
    "    'Title':'\"The Great Phatsby\" (Vol. II)',\n",
    "    'Directed by':'Timothy Bailey',\n",
    "    'Written by':'Matt Selman',\n",
    "    'Original air date':'January 15, 2017 (2017-01-15)',\n",
    "    'Prod.code':'WABF04',\n",
    "    'U.S. viewers(millions)':'6.90[214]',\n",
    "    'No. Season':'28'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "51964f5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dn=pd.DataFrame(newRow, index=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3170b84b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df= pd.concat([df, dn])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8d6c5c3",
   "metadata": {},
   "source": [
    "### Episodes made of two parts: 712 and 713"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2c075350",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No.overall</th>\n",
       "      <th>No. inseason</th>\n",
       "      <th>Title</th>\n",
       "      <th>Directed by</th>\n",
       "      <th>Written by</th>\n",
       "      <th>Original air date</th>\n",
       "      <th>Prod.code</th>\n",
       "      <th>U.S. viewers(millions)</th>\n",
       "      <th>No. Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>712713</td>\n",
       "      <td>67</td>\n",
       "      <td>\"A Serious Flanders\" (Parts 1 &amp; 2)</td>\n",
       "      <td>Debbie Bruce MahanMatthew Faughnan</td>\n",
       "      <td>Cesar Mazariegos</td>\n",
       "      <td>November 7, 2021 (2021-11-07)November 14, 2021...</td>\n",
       "      <td>QABF21QABF22</td>\n",
       "      <td>3.47[320]1.66[321]</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    No.overall No. inseason                               Title  \\\n",
       "616     712713           67  \"A Serious Flanders\" (Parts 1 & 2)   \n",
       "\n",
       "                            Directed by        Written by  \\\n",
       "616  Debbie Bruce MahanMatthew Faughnan  Cesar Mazariegos   \n",
       "\n",
       "                                     Original air date     Prod.code  \\\n",
       "616  November 7, 2021 (2021-11-07)November 14, 2021...  QABF21QABF22   \n",
       "\n",
       "    U.S. viewers(millions) No. Season  \n",
       "616     3.47[320]1.66[321]         33  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['No.overall'] == '712713']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f74dac3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(df[df['No.overall'] == '712713'].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6a7cc2ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "newRow={'No.overall':'712',\n",
    "    'No. inseason':'6',\n",
    "    'Title':'\"A Serious Flanders\" (Parts 1)\"',\n",
    "    'Directed by':'Debbie Bruce Mahan',\n",
    "    'Written by':'Cesar Mazariegos',\n",
    "    'Original air date':'November 7, 2021 (2021-11-07)',\n",
    "    'Prod.code':'QABF21',\n",
    "    'U.S. viewers(millions)':'3.47[320]',\n",
    "    'No. Season':'33'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "af0c54fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "dn=pd.DataFrame(newRow, index=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "f9334a90",
   "metadata": {},
   "outputs": [],
   "source": [
    "df= pd.concat([df, dn])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fb7270c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "newRow={'No.overall':'713',\n",
    "    'No. inseason':'7',\n",
    "    'Title':'\"A Serious Flanders\" (Parts 2)\"',\n",
    "    'Directed by':'Matthew Faughnan',\n",
    "    'Written by':'Cesar Mazariegos',\n",
    "    'Original air date':'November 14, 2021 (2021-11-14)',\n",
    "    'Prod.code':'QABF22',\n",
    "    'U.S. viewers(millions)':'1.66[321]',\n",
    "    'No. Season':'33'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8553dd85",
   "metadata": {},
   "outputs": [],
   "source": [
    "dn=pd.DataFrame(newRow, index=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "cd1d3557",
   "metadata": {},
   "outputs": [],
   "source": [
    "df= pd.concat([df, dn])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b79ed413",
   "metadata": {},
   "outputs": [],
   "source": [
    "# reset index\n",
    "df.reset_index(inplace = True, drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43e407fc",
   "metadata": {},
   "source": [
    "## Manage Wikipedia notes inside cells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "ce286cbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No.overall</th>\n",
       "      <th>No. inseason</th>\n",
       "      <th>Title</th>\n",
       "      <th>Directed by</th>\n",
       "      <th>Written by</th>\n",
       "      <th>Original air date</th>\n",
       "      <th>Prod.code</th>\n",
       "      <th>U.S. viewers(millions)</th>\n",
       "      <th>No. Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>204</td>\n",
       "      <td>1</td>\n",
       "      <td>\"Lard of the Dance\"</td>\n",
       "      <td>Dominic Polcino</td>\n",
       "      <td>Jane O'Brien</td>\n",
       "      <td>August 23, 1998 (1998-08-23)</td>\n",
       "      <td>5F20</td>\n",
       "      <td>7[210]</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>205</td>\n",
       "      <td>2</td>\n",
       "      <td>\"The Wizard of Evergreen Terrace\"</td>\n",
       "      <td>Mark Kirkland</td>\n",
       "      <td>John Swartzwelder</td>\n",
       "      <td>September 20, 1998 (1998-09-20)</td>\n",
       "      <td>5F21</td>\n",
       "      <td>7.95[211]</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>206</td>\n",
       "      <td>3</td>\n",
       "      <td>\"Bart the Mother\"</td>\n",
       "      <td>Steven Dean Moore</td>\n",
       "      <td>David X. Cohen</td>\n",
       "      <td>September 27, 1998 (1998-09-27)</td>\n",
       "      <td>5F22</td>\n",
       "      <td>7.35[212]</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>207</td>\n",
       "      <td>4</td>\n",
       "      <td>\"Treehouse of Horror IX\"</td>\n",
       "      <td>Steven Dean Moore</td>\n",
       "      <td>Donick CaryLarry DoyleDavid X. Cohen</td>\n",
       "      <td>October 25, 1998 (1998-10-25)</td>\n",
       "      <td>AABF01</td>\n",
       "      <td>8.5[213]</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>208</td>\n",
       "      <td>5</td>\n",
       "      <td>\"When You Dish Upon a Star\"</td>\n",
       "      <td>Pete Michels</td>\n",
       "      <td>Richard Appel</td>\n",
       "      <td>November 8, 1998 (1998-11-08)</td>\n",
       "      <td>5F19</td>\n",
       "      <td>9[214]</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  No.overall No. inseason                              Title  \\\n",
       "0        204            1                \"Lard of the Dance\"   \n",
       "1        205            2  \"The Wizard of Evergreen Terrace\"   \n",
       "2        206            3                  \"Bart the Mother\"   \n",
       "3        207            4           \"Treehouse of Horror IX\"   \n",
       "4        208            5        \"When You Dish Upon a Star\"   \n",
       "\n",
       "         Directed by                            Written by  \\\n",
       "0    Dominic Polcino                          Jane O'Brien   \n",
       "1      Mark Kirkland                     John Swartzwelder   \n",
       "2  Steven Dean Moore                        David X. Cohen   \n",
       "3  Steven Dean Moore  Donick CaryLarry DoyleDavid X. Cohen   \n",
       "4       Pete Michels                         Richard Appel   \n",
       "\n",
       "                 Original air date Prod.code U.S. viewers(millions) No. Season  \n",
       "0     August 23, 1998 (1998-08-23)      5F20                 7[210]         10  \n",
       "1  September 20, 1998 (1998-09-20)      5F21              7.95[211]         10  \n",
       "2  September 27, 1998 (1998-09-27)      5F22              7.35[212]         10  \n",
       "3    October 25, 1998 (1998-10-25)    AABF01               8.5[213]         10  \n",
       "4    November 8, 1998 (1998-11-08)      5F19                 9[214]         10  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "5a438578",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['U.S. viewers(millions)']=df['U.S. viewers(millions)'].apply(str)\n",
    "df['U.S. viewers(millions)']=df['U.S. viewers(millions)'].apply(cleanNotes)\n",
    "\n",
    "df['Directed by']=df['Directed by'].apply(str)\n",
    "df['Directed by']=df['Directed by'].apply(cleanNotes)\n",
    "\n",
    "df['Written by']=df['Written by'].apply(str)\n",
    "df['Written by']=df['Written by'].apply(cleanNotes)\n",
    "\n",
    "df['Prod.code']=df['Prod.code'].apply(str)\n",
    "df['Prod.code']=df['Prod.code'].apply(cleanNotes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "5b11444e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No.overall</th>\n",
       "      <th>No. inseason</th>\n",
       "      <th>Title</th>\n",
       "      <th>Directed by</th>\n",
       "      <th>Written by</th>\n",
       "      <th>Original air date</th>\n",
       "      <th>Prod.code</th>\n",
       "      <th>U.S. viewers(millions)</th>\n",
       "      <th>No. Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>204</td>\n",
       "      <td>1</td>\n",
       "      <td>\"Lard of the Dance\"</td>\n",
       "      <td>Dominic Polcino</td>\n",
       "      <td>Jane O'Brien</td>\n",
       "      <td>August 23, 1998 (1998-08-23)</td>\n",
       "      <td>5F20</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>205</td>\n",
       "      <td>2</td>\n",
       "      <td>\"The Wizard of Evergreen Terrace\"</td>\n",
       "      <td>Mark Kirkland</td>\n",
       "      <td>John Swartzwelder</td>\n",
       "      <td>September 20, 1998 (1998-09-20)</td>\n",
       "      <td>5F21</td>\n",
       "      <td>7.95</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>206</td>\n",
       "      <td>3</td>\n",
       "      <td>\"Bart the Mother\"</td>\n",
       "      <td>Steven Dean Moore</td>\n",
       "      <td>David X. Cohen</td>\n",
       "      <td>September 27, 1998 (1998-09-27)</td>\n",
       "      <td>5F22</td>\n",
       "      <td>7.35</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>207</td>\n",
       "      <td>4</td>\n",
       "      <td>\"Treehouse of Horror IX\"</td>\n",
       "      <td>Steven Dean Moore</td>\n",
       "      <td>Donick CaryLarry DoyleDavid X. Cohen</td>\n",
       "      <td>October 25, 1998 (1998-10-25)</td>\n",
       "      <td>AABF01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>208</td>\n",
       "      <td>5</td>\n",
       "      <td>\"When You Dish Upon a Star\"</td>\n",
       "      <td>Pete Michels</td>\n",
       "      <td>Richard Appel</td>\n",
       "      <td>November 8, 1998 (1998-11-08)</td>\n",
       "      <td>5F19</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  No.overall No. inseason                              Title  \\\n",
       "0        204            1                \"Lard of the Dance\"   \n",
       "1        205            2  \"The Wizard of Evergreen Terrace\"   \n",
       "2        206            3                  \"Bart the Mother\"   \n",
       "3        207            4           \"Treehouse of Horror IX\"   \n",
       "4        208            5        \"When You Dish Upon a Star\"   \n",
       "\n",
       "         Directed by                            Written by  \\\n",
       "0    Dominic Polcino                          Jane O'Brien   \n",
       "1      Mark Kirkland                     John Swartzwelder   \n",
       "2  Steven Dean Moore                        David X. Cohen   \n",
       "3  Steven Dean Moore  Donick CaryLarry DoyleDavid X. Cohen   \n",
       "4       Pete Michels                         Richard Appel   \n",
       "\n",
       "                 Original air date Prod.code U.S. viewers(millions) No. Season  \n",
       "0     August 23, 1998 (1998-08-23)      5F20                      7         10  \n",
       "1  September 20, 1998 (1998-09-20)      5F21                   7.95         10  \n",
       "2  September 27, 1998 (1998-09-27)      5F22                   7.35         10  \n",
       "3    October 25, 1998 (1998-10-25)    AABF01                    8.5         10  \n",
       "4    November 8, 1998 (1998-11-08)      5F19                      9         10  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6703a359",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Original air date']=df['Original air date'].apply(str)\n",
    "df['Original air date']=df['Original air date'].apply(cleanDates)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be75c072",
   "metadata": {},
   "source": [
    "## Remove double quotes from column Title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "afb71e94",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Title']=df['Title'].str.replace('\"', '')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8499abe",
   "metadata": {},
   "source": [
    "## Fix datatype of numeric columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "d967b767",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['No.overall'] = pd.to_numeric(df['No.overall'])\n",
    "df['No. inseason'] = pd.to_numeric(df['No. inseason'])\n",
    "df['No. Season'] = pd.to_numeric(df['No. Season'])\n",
    "\n",
    "df['U.S. viewers(millions)'] = df['U.S. viewers(millions)'].replace(['TBD'], None)\n",
    "df['U.S. viewers(millions)'] = df['U.S. viewers(millions)'].replace(['N/A'], None)\n",
    "df['U.S. viewers(millions)'] = pd.to_numeric(df['U.S. viewers(millions)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5553713",
   "metadata": {},
   "source": [
    "## Final check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "b76f01b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No.overall</th>\n",
       "      <th>No. inseason</th>\n",
       "      <th>Title</th>\n",
       "      <th>Directed by</th>\n",
       "      <th>Written by</th>\n",
       "      <th>Original air date</th>\n",
       "      <th>Prod.code</th>\n",
       "      <th>U.S. viewers(millions)</th>\n",
       "      <th>No. Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>204</td>\n",
       "      <td>1</td>\n",
       "      <td>Lard of the Dance</td>\n",
       "      <td>Dominic Polcino</td>\n",
       "      <td>Jane O'Brien</td>\n",
       "      <td>1998-08-23</td>\n",
       "      <td>5F20</td>\n",
       "      <td>7.00</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>205</td>\n",
       "      <td>2</td>\n",
       "      <td>The Wizard of Evergreen Terrace</td>\n",
       "      <td>Mark Kirkland</td>\n",
       "      <td>John Swartzwelder</td>\n",
       "      <td>1998-09-20</td>\n",
       "      <td>5F21</td>\n",
       "      <td>7.95</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>206</td>\n",
       "      <td>3</td>\n",
       "      <td>Bart the Mother</td>\n",
       "      <td>Steven Dean Moore</td>\n",
       "      <td>David X. Cohen</td>\n",
       "      <td>1998-09-27</td>\n",
       "      <td>5F22</td>\n",
       "      <td>7.35</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>207</td>\n",
       "      <td>4</td>\n",
       "      <td>Treehouse of Horror IX</td>\n",
       "      <td>Steven Dean Moore</td>\n",
       "      <td>Donick CaryLarry DoyleDavid X. Cohen</td>\n",
       "      <td>1998-10-25</td>\n",
       "      <td>AABF01</td>\n",
       "      <td>8.50</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>208</td>\n",
       "      <td>5</td>\n",
       "      <td>When You Dish Upon a Star</td>\n",
       "      <td>Pete Michels</td>\n",
       "      <td>Richard Appel</td>\n",
       "      <td>1998-11-08</td>\n",
       "      <td>5F19</td>\n",
       "      <td>9.00</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   No.overall  No. inseason                            Title  \\\n",
       "0         204             1                Lard of the Dance   \n",
       "1         205             2  The Wizard of Evergreen Terrace   \n",
       "2         206             3                  Bart the Mother   \n",
       "3         207             4           Treehouse of Horror IX   \n",
       "4         208             5        When You Dish Upon a Star   \n",
       "\n",
       "         Directed by                            Written by Original air date  \\\n",
       "0    Dominic Polcino                          Jane O'Brien        1998-08-23   \n",
       "1      Mark Kirkland                     John Swartzwelder        1998-09-20   \n",
       "2  Steven Dean Moore                        David X. Cohen        1998-09-27   \n",
       "3  Steven Dean Moore  Donick CaryLarry DoyleDavid X. Cohen        1998-10-25   \n",
       "4       Pete Michels                         Richard Appel        1998-11-08   \n",
       "\n",
       "  Prod.code  U.S. viewers(millions)  No. Season  \n",
       "0      5F20                    7.00          10  \n",
       "1      5F21                    7.95          10  \n",
       "2      5F22                    7.35          10  \n",
       "3    AABF01                    8.50          10  \n",
       "4      5F19                    9.00          10  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "6128d4e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No.overall</th>\n",
       "      <th>No. inseason</th>\n",
       "      <th>Title</th>\n",
       "      <th>Directed by</th>\n",
       "      <th>Written by</th>\n",
       "      <th>Original air date</th>\n",
       "      <th>Prod.code</th>\n",
       "      <th>U.S. viewers(millions)</th>\n",
       "      <th>No. Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>694</th>\n",
       "      <td>682</td>\n",
       "      <td>20</td>\n",
       "      <td>Warrin' Priests (Parts Two)</td>\n",
       "      <td>Matthew Nastuk</td>\n",
       "      <td>Pete Holmes</td>\n",
       "      <td>2020-05-03</td>\n",
       "      <td>ZABF13</td>\n",
       "      <td>1.36</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>695</th>\n",
       "      <td>608</td>\n",
       "      <td>12</td>\n",
       "      <td>The Great Phatsby (Vol. I)</td>\n",
       "      <td>Chris Clements</td>\n",
       "      <td>Dan Greaney</td>\n",
       "      <td>2017-01-15</td>\n",
       "      <td>WABF04</td>\n",
       "      <td>6.90</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>696</th>\n",
       "      <td>609</td>\n",
       "      <td>12</td>\n",
       "      <td>The Great Phatsby (Vol. II)</td>\n",
       "      <td>Timothy Bailey</td>\n",
       "      <td>Matt Selman</td>\n",
       "      <td>2017-01-15</td>\n",
       "      <td>WABF04</td>\n",
       "      <td>6.90</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>697</th>\n",
       "      <td>712</td>\n",
       "      <td>6</td>\n",
       "      <td>A Serious Flanders (Parts 1)</td>\n",
       "      <td>Debbie Bruce Mahan</td>\n",
       "      <td>Cesar Mazariegos</td>\n",
       "      <td>2021-11-07</td>\n",
       "      <td>QABF21</td>\n",
       "      <td>3.47</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>713</td>\n",
       "      <td>7</td>\n",
       "      <td>A Serious Flanders (Parts 2)</td>\n",
       "      <td>Matthew Faughnan</td>\n",
       "      <td>Cesar Mazariegos</td>\n",
       "      <td>2021-11-14</td>\n",
       "      <td>QABF22</td>\n",
       "      <td>1.66</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     No.overall  No. inseason                         Title  \\\n",
       "694         682            20   Warrin' Priests (Parts Two)   \n",
       "695         608            12    The Great Phatsby (Vol. I)   \n",
       "696         609            12   The Great Phatsby (Vol. II)   \n",
       "697         712             6  A Serious Flanders (Parts 1)   \n",
       "698         713             7  A Serious Flanders (Parts 2)   \n",
       "\n",
       "            Directed by        Written by Original air date Prod.code  \\\n",
       "694      Matthew Nastuk       Pete Holmes        2020-05-03    ZABF13   \n",
       "695      Chris Clements       Dan Greaney        2017-01-15    WABF04   \n",
       "696      Timothy Bailey       Matt Selman        2017-01-15    WABF04   \n",
       "697  Debbie Bruce Mahan  Cesar Mazariegos        2021-11-07    QABF21   \n",
       "698    Matthew Faughnan  Cesar Mazariegos        2021-11-14    QABF22   \n",
       "\n",
       "     U.S. viewers(millions)  No. Season  \n",
       "694                    1.36          31  \n",
       "695                    6.90          28  \n",
       "696                    6.90          28  \n",
       "697                    3.47          33  \n",
       "698                    1.66          33  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "04e29d1d",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['1998-08-23', '1998-09-20', '1998-09-27', '1998-10-25',\n",
       "       '1998-11-08', '1998-11-15', '1998-11-22', '1998-12-06',\n",
       "       '1998-12-20', '1999-01-10', '1999-01-17', '1999-01-31',\n",
       "       '1999-02-07', '1999-02-14', '1999-02-21', '1999-02-28',\n",
       "       '1999-03-28', '1999-04-04', '1999-04-11', '1999-04-25',\n",
       "       '1999-05-02', '1999-05-09', '1999-05-16', '1994-09-04',\n",
       "       '1994-09-11', '1994-09-25', '1994-10-02', '1994-10-09',\n",
       "       '1994-10-30', '1994-11-06', '1994-11-13', '1994-11-27',\n",
       "       '1994-12-04', '1994-12-18', '1995-01-08', '1995-01-22',\n",
       "       '1995-02-05', '1995-02-12', '1995-02-19', '1995-02-26',\n",
       "       '1995-03-05', '1995-03-19', '1995-04-09', '1995-04-16',\n",
       "       '1995-04-30', '1995-05-07', '1995-05-14', '1995-05-21',\n",
       "       '2014-09-28', '2014-10-05', '2014-10-12', '2014-10-19',\n",
       "       '2014-11-02', '2014-11-09', '2014-11-16', '2014-11-23',\n",
       "       '2014-12-07', '2015-01-04', '2015-01-11', '2015-01-25',\n",
       "       '2015-02-08', '2015-02-15', '2015-03-01', '2015-03-08',\n",
       "       '2015-03-15', '2015-04-19', '2015-04-26', '2015-05-03',\n",
       "       '2015-05-10', '2015-05-17', '2018-09-30', '2018-10-07',\n",
       "       '2018-10-14', '2018-10-21', '2018-11-04', '2018-11-11',\n",
       "       '2018-11-18', '2018-11-25', '2018-12-02', '2018-12-09',\n",
       "       '2019-01-06', '2019-01-13', '2019-02-10', '2019-02-17',\n",
       "       '2019-03-03', '2019-03-10', '2019-03-17', '2019-03-24',\n",
       "       '2019-03-31', '2019-04-07', '2019-04-28', '2019-05-05',\n",
       "       '2019-05-12', '2019-09-29', '2019-10-06', '2019-10-13',\n",
       "       '2019-10-20', '2019-11-03', '2019-11-10', '2019-11-17',\n",
       "       '2019-11-24', '2019-12-01', '2019-12-15', '2020-01-05',\n",
       "       '2020-02-16', '2020-02-23', '2020-03-01', '2020-03-08',\n",
       "       '2020-03-15', '2020-03-22', '2020-04-19', '2020-05-10',\n",
       "       '2020-05-17', '2015-09-27', '2015-10-04', '2015-10-11',\n",
       "       '2015-10-18', '2015-10-25', '2015-11-08', '2015-11-22',\n",
       "       '2015-12-06', '2015-12-13', '2016-01-03', '2016-01-10',\n",
       "       '2016-01-17', '2016-02-14', '2016-02-21', '2016-03-06',\n",
       "       '2016-03-13', '2016-04-03', '2016-04-10', '2016-04-24',\n",
       "       '2016-05-08', '2016-05-15', '2016-05-22', '1995-09-17',\n",
       "       '1995-09-24', '1995-10-01', '1995-10-08', '1995-10-15',\n",
       "       '1995-10-29', '1995-11-05', '1995-11-19', '1995-11-26',\n",
       "       '1995-12-03', '1995-12-17', '1996-01-07', '1996-01-14',\n",
       "       '1996-02-04', '1996-02-11', '1996-02-18', '1996-02-25',\n",
       "       '1996-03-17', '1996-03-24', '1996-03-31', '1996-04-14',\n",
       "       '1996-04-28', '1996-05-05', '1996-05-19', '1999-09-26',\n",
       "       '1999-10-03', '1999-10-24', '1999-10-31', '1999-11-07',\n",
       "       '1999-11-14', '1999-11-21', '1999-11-28', '1999-12-19',\n",
       "       '2000-01-09', '2000-01-16', '2000-01-23', '2000-02-06',\n",
       "       '2000-02-13', '2000-02-20', '2000-02-27', '2000-03-19',\n",
       "       '2000-04-09', '2000-04-30', '2000-05-07', '2000-05-14',\n",
       "       '2000-05-21', '2004-11-07', '2004-11-14', '2004-11-21',\n",
       "       '2004-12-05', '2004-12-12', '2005-01-16', '2005-01-30',\n",
       "       '2005-02-06', '2005-02-13', '2005-02-20', '2005-03-06',\n",
       "       '2005-03-13', '2005-03-20', '2005-04-03', '2005-04-17',\n",
       "       '2005-05-01', '2005-05-08', '2005-05-15', '1989-12-17',\n",
       "       '1990-01-14', '1990-01-21', '1990-01-28', '1990-02-04',\n",
       "       '1990-02-11', '1990-02-18', '1990-02-25', '1990-03-18',\n",
       "       '1990-03-25', '1990-04-15', '1990-04-29', '1990-05-13',\n",
       "       '2005-09-11', '2005-09-18', '2005-09-25', '2005-11-06',\n",
       "       '2005-11-13', '2005-11-20', '2005-11-27', '2005-12-11',\n",
       "       '2005-12-18', '2006-01-08', '2006-01-29', '2006-02-26',\n",
       "       '2006-03-12', '2006-03-19', '2006-03-26', '2006-04-02',\n",
       "       '2006-04-09', '2006-04-23', '2006-04-30', '2006-05-07',\n",
       "       '2006-05-14', '2006-05-21', '2009-09-27', '2009-10-04',\n",
       "       '2009-10-11', '2009-10-18', '2009-11-15', '2009-11-22',\n",
       "       '2009-11-29', '2009-12-13', '2010-01-03', '2010-01-10',\n",
       "       '2010-01-31', '2010-02-14', '2010-02-21', '2010-03-14',\n",
       "       '2010-03-21', '2010-03-28', '2010-04-11', '2010-04-18',\n",
       "       '2010-04-25', '2010-05-02', '2010-05-09', '2010-05-16',\n",
       "       '2010-05-23', '2022-09-25', '2022-10-02', '2022-10-09',\n",
       "       '2022-10-16', '2022-10-23', '2022-10-30', '2022-11-13',\n",
       "       '2022-11-20', '2022-11-27', '2022-12-04', '2022-12-11',\n",
       "       '2023-01-01', '2010-09-26', '2010-10-03', '2010-10-10',\n",
       "       '2010-11-07', '2010-11-14', '2010-11-21', '2010-11-28',\n",
       "       '2010-12-05', '2010-12-12', '2011-01-09', '2011-01-16',\n",
       "       '2011-01-23', '2011-02-13', '2011-02-20', '2011-03-06',\n",
       "       '2011-03-13', '2011-03-27', '2011-04-10', '2011-05-01',\n",
       "       '2011-05-08', '2011-05-15', '2011-05-22', '2006-09-10',\n",
       "       '2006-09-17', '2006-09-24', '2006-11-05', '2006-11-12',\n",
       "       '2006-11-19', '2006-11-26', '2006-12-10', '2006-12-17',\n",
       "       '2007-01-07', '2007-01-28', '2007-02-11', '2007-02-18',\n",
       "       '2007-03-04', '2007-03-11', '2007-03-25', '2007-04-22',\n",
       "       '2007-04-29', '2007-05-06', '2007-05-13', '2007-05-20',\n",
       "       '1990-10-11', '1990-10-18', '1990-10-25', '1990-11-01',\n",
       "       '1990-11-08', '1990-11-15', '1990-11-22', '1990-12-06',\n",
       "       '1990-12-20', '1991-01-10', '1991-01-24', '1991-01-31',\n",
       "       '1991-02-07', '1991-02-14', '1991-02-21', '1991-03-07',\n",
       "       '1991-03-28', '1991-04-11', '1991-04-25', '1991-05-02',\n",
       "       '1991-05-09', '1991-07-11', '2002-11-03', '2002-11-10',\n",
       "       '2002-11-17', '2002-11-24', '2002-12-01', '2002-12-15',\n",
       "       '2003-01-05', '2003-01-12', '2003-02-02', '2003-02-09',\n",
       "       '2003-02-16', '2003-03-02', '2003-03-09', '2003-03-16',\n",
       "       '2003-03-30', '2003-04-13', '2003-04-27', '2003-05-04',\n",
       "       '2003-05-11', '2003-05-18', '2003-11-02', '2003-11-09',\n",
       "       '2003-11-16', '2003-11-23', '2003-11-30', '2003-12-07',\n",
       "       '2003-12-14', '2004-01-04', '2004-01-11', '2004-01-25',\n",
       "       '2004-02-08', '2004-02-15', '2004-02-22', '2004-03-14',\n",
       "       '2004-03-21', '2004-03-28', '2004-04-18', '2004-04-25',\n",
       "       '2004-05-02', '2004-05-09', '2004-05-16', '2004-05-23',\n",
       "       '1991-09-19', '1991-09-26', '1991-10-03', '1991-10-10',\n",
       "       '1991-10-17', '1991-10-24', '1991-10-31', '1991-11-07',\n",
       "       '1991-11-14', '1991-11-21', '1991-12-05', '1991-12-26',\n",
       "       '1992-01-09', '1992-01-23', '1992-02-06', '1992-02-13',\n",
       "       '1992-02-20', '1992-02-27', '1992-03-12', '1992-03-26',\n",
       "       '1992-04-09', '1992-04-23', '1992-05-07', '1992-08-27',\n",
       "       '2011-09-25', '2011-10-02', '2011-10-30', '2011-11-06',\n",
       "       '2011-11-13', '2011-11-20', '2011-11-27', '2011-12-04',\n",
       "       '2011-12-11', '2012-01-08', '2012-01-15', '2012-01-29',\n",
       "       '2012-02-12', '2012-02-19', '2012-03-04', '2012-03-11',\n",
       "       '2012-03-18', '2012-04-15', '2012-04-29', '2012-05-06',\n",
       "       '2012-05-13', '2012-05-20', '2016-09-25', '2016-10-02',\n",
       "       '2016-10-09', '2016-10-16', '2016-10-23', '2016-11-06',\n",
       "       '2016-11-13', '2016-11-20', '2016-12-04', '2016-12-11',\n",
       "       '2017-01-08', '2017-02-12', '2017-02-19', '2017-03-05',\n",
       "       '2017-03-12', '2017-03-19', '2017-04-02', '2017-04-30',\n",
       "       '2017-05-07', '2017-05-21', '1992-09-24', '1992-10-01',\n",
       "       '1992-10-08', '1992-10-15', '1992-10-29', '1992-11-03',\n",
       "       '1992-11-05', '1992-11-12', '1992-11-19', '1992-12-03',\n",
       "       '1992-12-17', '1993-01-14', '1993-01-21', '1993-02-04',\n",
       "       '1993-02-11', '1993-02-18', '1993-03-11', '1993-04-01',\n",
       "       '1993-04-15', '1993-04-29', '1993-05-06', '1993-05-13',\n",
       "       '2000-11-01', '2000-11-05', '2000-11-12', '2000-11-19',\n",
       "       '2000-11-26', '2000-12-03', '2000-12-10', '2000-12-17',\n",
       "       '2001-01-07', '2001-01-14', '2001-02-04', '2001-02-11',\n",
       "       '2001-02-18', '2001-02-25', '2001-03-04', '2001-03-11',\n",
       "       '2001-04-01', '2001-04-29', '2001-05-06', '2001-05-13',\n",
       "       '2001-05-20', '2020-09-27', '2020-10-04', '2020-10-11',\n",
       "       '2020-11-01', '2020-11-08', '2020-11-15', '2020-11-22',\n",
       "       '2020-11-29', '2020-12-06', '2020-12-13', '2021-01-03',\n",
       "       '2021-02-21', '2021-02-28', '2021-03-07', '2021-03-14',\n",
       "       '2021-03-21', '2021-03-28', '2021-04-11', '2021-04-18',\n",
       "       '2021-05-09', '2021-05-16', '2021-05-23', '2012-09-30',\n",
       "       '2012-10-07', '2012-11-04', '2012-11-11', '2012-11-18',\n",
       "       '2012-11-25', '2012-12-09', '2012-12-16', '2013-01-06',\n",
       "       '2013-01-13', '2013-01-27', '2013-02-10', '2013-02-17',\n",
       "       '2013-03-03', '2013-03-10', '2013-03-17', '2013-04-14',\n",
       "       '2013-04-28', '2013-05-05', '2013-05-12', '2013-05-19',\n",
       "       '1996-10-27', '1996-11-03', '1996-11-10', '1996-11-17',\n",
       "       '1996-11-24', '1996-12-01', '1996-12-15', '1996-12-29',\n",
       "       '1997-01-05', '1997-01-12', '1997-01-19', '1997-02-02',\n",
       "       '1997-02-07', '1997-02-09', '1997-02-16', '1997-02-23',\n",
       "       '1997-03-02', '1997-03-16', '1997-04-06', '1997-04-13',\n",
       "       '1997-04-20', '1997-04-27', '1997-05-04', '1997-05-11',\n",
       "       '1997-05-18', '1997-09-21', '1997-09-28', '1997-10-19',\n",
       "       '1997-10-26', '1997-11-02', '1997-11-09', '1997-11-16',\n",
       "       '1997-11-23', '1997-12-07', '1997-12-21', '1998-01-04',\n",
       "       '1998-01-11', '1998-02-08', '1998-02-15', '1998-02-22',\n",
       "       '1998-03-01', '1998-03-08', '1998-03-22', '1998-03-29',\n",
       "       '1998-04-05', '1998-04-19', '1998-04-26', '1998-05-03',\n",
       "       '1998-05-10', '1998-05-17', '2013-09-29', '2013-10-06',\n",
       "       '2013-11-03', '2013-11-10', '2013-11-17', '2013-11-24',\n",
       "       '2013-12-08', '2013-12-15', '2014-01-05', '2014-01-12',\n",
       "       '2014-01-26', '2014-03-09', '2014-03-16', '2014-03-23',\n",
       "       '2014-03-30', '2014-04-06', '2014-04-13', '2014-04-27',\n",
       "       '2014-05-04', '2014-05-11', '2014-05-18', '2021-09-26',\n",
       "       '2021-10-03', '2021-10-10', '2021-10-17', '2021-10-24',\n",
       "       '2021-11-21', '2021-11-28', '2021-12-19', '2022-01-02',\n",
       "       '2022-02-27', '2022-03-06', '2022-03-13', '2022-03-20',\n",
       "       '2022-03-27', '2022-04-10', '2022-04-24', '2022-05-01',\n",
       "       '2022-05-08', '2022-05-15', '2022-05-22', '2001-11-06',\n",
       "       '2001-11-11', '2001-11-18', '2001-12-02', '2001-12-09',\n",
       "       '2001-12-16', '2002-01-06', '2002-01-20', '2002-01-27',\n",
       "       '2002-02-10', '2002-02-17', '2002-02-24', '2002-03-10',\n",
       "       '2002-03-17', '2002-03-31', '2002-04-07', '2002-04-21',\n",
       "       '2002-04-28', '2002-05-05', '2002-05-12', '2002-05-19',\n",
       "       '2002-05-22', '1993-09-30', '1993-10-07', '1993-10-14',\n",
       "       '1993-10-21', '1993-10-28', '1993-11-04', '1993-11-11',\n",
       "       '1993-11-18', '1993-12-09', '1993-12-16', '1994-01-06',\n",
       "       '1994-02-03', '1994-02-10', '1994-02-17', '1994-02-24',\n",
       "       '1994-03-17', '1994-03-31', '1994-04-14', '1994-04-28',\n",
       "       '1994-05-05', '1994-05-12', '1994-05-19', '2017-10-01',\n",
       "       '2017-10-08', '2017-10-15', '2017-10-22', '2017-11-05',\n",
       "       '2017-11-12', '2017-11-19', '2017-12-03', '2017-12-10',\n",
       "       '2018-01-07', '2018-01-14', '2018-03-18', '2018-03-25',\n",
       "       '2018-04-01', '2018-04-08', '2018-04-15', '2018-04-22',\n",
       "       '2018-04-29', '2018-05-06', '2018-05-13', '2018-05-20',\n",
       "       '2020-04-26', '2020-05-03', '2017-01-15', '2021-11-07',\n",
       "       '2021-11-14'], dtype=object)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Original air date'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "3c34dbf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['No. Season'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "6a8a919e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['No.overall'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "b8dd4fbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['No. inseason'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "1bf41843",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Original air date'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e0bfad2",
   "metadata": {},
   "source": [
    "## Write cleaned data to Json file"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "365865a7",
   "metadata": {},
   "source": [
    "### Full data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "eae7322b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_json(PATH_FOLDER_WRITE+'dataset_DDB.json', orient='records', indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cced5cc3",
   "metadata": {},
   "source": [
    "### Sample data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "9f554208",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.sample(10).to_json(PATH_FOLDER_WRITE+'dataset_DDB_sample.json', orient='records', indent=4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "my_env",
   "language": "python",
   "name": "my_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}