IU-ML-Unsupervised-Unit1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "af5a4f2c",
   "metadata": {},
   "source": [
    "# Machine Learning - Unsupervised Learning and Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17069e5e",
   "metadata": {},
   "source": [
    "## Unit 1. Introduction to Unsupervised Machine Learning and Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8955ab4",
   "metadata": {},
   "source": [
    "Lecturer: Tai Le Quy\n",
    "\n",
    "Thanks Prof. Dr. Christian Müller-Kett\n",
    "\n",
    "This notebook is based on Python version on https://github.com/iubh/DLBDSMLUSL01"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e8da664",
   "metadata": {},
   "source": [
    "### 1. Example 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0bad5f46",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "43459dbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create sample data\n",
    "Student_R = {'Student_ID':['S1', 'S2', 'S3'], \n",
    "    'Birth_date': [datetime.date(1996,7,14), \n",
    "                   datetime.date(1997,8,22), \n",
    "                   datetime.date(1998,5,11)]}\n",
    "\n",
    "Student_R = pd.DataFrame(Student_R, columns = ['Student_ID','Birth_date'])\n",
    "\n",
    "Courses = { 'Student_ID':['S1', 'S2', 'S3', 'S1', 'S2', 'S3'], 'Grades':[18, 11, 12, 15, 19, 10]}\n",
    "\n",
    "Courses = pd.DataFrame (Courses, columns = ['Student_ID', 'Grades'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ecd96a43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Student_ID</th>\n",
       "      <th>Birth_date</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>S1</td>\n",
       "      <td>1996-07-14</td>\n",
       "      <td>1996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>S2</td>\n",
       "      <td>1997-08-22</td>\n",
       "      <td>1997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>S3</td>\n",
       "      <td>1998-05-11</td>\n",
       "      <td>1998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Student_ID  Birth_date  year\n",
       "0         S1  1996-07-14  1996\n",
       "1         S2  1997-08-22  1997\n",
       "2         S3  1998-05-11  1998"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Student_R"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e7f8acb7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Student_ID</th>\n",
       "      <th>Grades</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>S1</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>S2</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>S3</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>S1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>S2</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>S3</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Student_ID  Grades\n",
       "0         S1      18\n",
       "1         S2      11\n",
       "2         S3      12\n",
       "3         S1      15\n",
       "4         S2      19\n",
       "5         S3      10"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Courses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b8d8b337",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Student_ID  Birth_date  year\n",
      "0         S1  1996-07-14  1996\n",
      "1         S2  1997-08-22  1997\n",
      "2         S3  1998-05-11  1998\n"
     ]
    }
   ],
   "source": [
    "#%% extracting the year from the birth date\n",
    "Student_R['year'] = pd.DatetimeIndex(Student_R['Birth_date']).year\n",
    "print(Student_R.head())\n",
    "# console output:\n",
    "#   Student_ID  Birth_date  year\n",
    "# 0         S1  1996-07-14  1996\n",
    "# 1         S2  1997-08-22  1997\n",
    "# 2         S3  1998-05-11  1998"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d9e30bfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% creation of features by aggregation of grouped values\n",
    "goper = Courses.groupby('Student_ID')['Grades'].\\\n",
    "    agg(['mean','max','min'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ddb07ab8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            mean_grade  max_grade  min_grade\n",
      "Student_ID                                  \n",
      "S1                16.5         18         15\n",
      "S2                15.0         19         11\n",
      "S3                11.0         12         10\n"
     ]
    }
   ],
   "source": [
    "# rename columns\n",
    "goper.columns = ['mean_grade','max_grade','min_grade']\n",
    "print(goper.head())\n",
    "# console output:\n",
    "#             mean_grade  max_grade  min_grade\n",
    "# Student_ID                                  \n",
    "# S1                16.5         18         15\n",
    "# S2                15.0         19         11\n",
    "# S3                11.0         12         10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd017afd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a8da1a07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Student_ID</th>\n",
       "      <th>Birth_date</th>\n",
       "      <th>year</th>\n",
       "      <th>mean_grade</th>\n",
       "      <th>max_grade</th>\n",
       "      <th>min_grade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>S1</td>\n",
       "      <td>1996-07-14</td>\n",
       "      <td>1996</td>\n",
       "      <td>16.5</td>\n",
       "      <td>18</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>S2</td>\n",
       "      <td>1997-08-22</td>\n",
       "      <td>1997</td>\n",
       "      <td>15.0</td>\n",
       "      <td>19</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>S3</td>\n",
       "      <td>1998-05-11</td>\n",
       "      <td>1998</td>\n",
       "      <td>11.0</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Student_ID  Birth_date  year  mean_grade  max_grade  min_grade\n",
       "0         S1  1996-07-14  1996        16.5         18         15\n",
       "1         S2  1997-08-22  1997        15.0         19         11\n",
       "2         S3  1998-05-11  1998        11.0         12         10"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%% merge with the Student_R dataframe\n",
    "R = Student_R.merge(goper, left_on = 'Student_ID', \\\n",
    "    right_index=True, how = 'left'). \\\n",
    "        head()\n",
    "# show the dataframe\n",
    "R\n",
    "#   Student_ID  Birth_date  year  mean_grade  max_grade  min_grade\n",
    "# 0         S1  1996-07-14  1996        16.5         18         15\n",
    "# 1         S2  1997-08-22  1997        15.0         19         11\n",
    "# 2         S3  1998-05-11  1998        11.0         12         10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f43b4e5",
   "metadata": {},
   "source": [
    "### 2. Example 2\n",
    "In this example, we will see how we can use Python to extract added value within a dataset based on dates.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f456a567",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the required packages\n",
    "import pandas as pd\n",
    "from scipy.stats import spearmanr\n",
    "from datetime import date\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e9b10aa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% load the data\n",
    "dat = pd.read_csv('data_visitors.csv', delimiter=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d8165651",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>visitors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>2735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-01-02</td>\n",
       "      <td>2084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-01-03</td>\n",
       "      <td>647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>1098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-01-05</td>\n",
       "      <td>1778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>2021-12-27</td>\n",
       "      <td>536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>2021-12-28</td>\n",
       "      <td>277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>2021-12-29</td>\n",
       "      <td>842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>2021-12-30</td>\n",
       "      <td>1163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>2021-12-31</td>\n",
       "      <td>2200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>365 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           date  visitors\n",
       "0    2021-01-01      2735\n",
       "1    2021-01-02      2084\n",
       "2    2021-01-03       647\n",
       "3    2021-01-04      1098\n",
       "4    2021-01-05      1778\n",
       "..          ...       ...\n",
       "360  2021-12-27       536\n",
       "361  2021-12-28       277\n",
       "362  2021-12-29       842\n",
       "363  2021-12-30      1163\n",
       "364  2021-12-31      2200\n",
       "\n",
       "[365 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%% glimpse at the data\n",
    "dat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "36bfcfaf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.0010369613036009904, pvalue=0.9842482530265125)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%% correlate the date and the visitors\n",
    "spearmanr(dat['date'], dat['visitors'])\n",
    "#Calculate a Spearman correlation coefficient with associated p-value.\n",
    "# The Spearman rank-order correlation coefficient is a nonparametric measure of the monotonicity of the relationship\n",
    "# between two datasets. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation\n",
    "# Correlations of -1 or +1 imply an exact monotonic relationship. \n",
    "# Positive correlations imply that as x increases, so does y. \n",
    "# Negative correlations imply that as x increases, y decreases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cdf9cc2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% define the date as such\n",
    "dat['date'] = pd.to_datetime(dat['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9b0a9b4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% extract the weekdays from the date (Monday is 0 and Sunday is 6)\n",
    "dat['weekday'] = dat['date'].apply(lambda date: date.weekday())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3d14efc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>visitors</th>\n",
       "      <th>weekday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>2735</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-01-02</td>\n",
       "      <td>2084</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-01-03</td>\n",
       "      <td>647</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>1098</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-01-05</td>\n",
       "      <td>1778</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>2021-12-27</td>\n",
       "      <td>536</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>2021-12-28</td>\n",
       "      <td>277</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>2021-12-29</td>\n",
       "      <td>842</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>2021-12-30</td>\n",
       "      <td>1163</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>2021-12-31</td>\n",
       "      <td>2200</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>365 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          date  visitors  weekday\n",
       "0   2021-01-01      2735        4\n",
       "1   2021-01-02      2084        5\n",
       "2   2021-01-03       647        6\n",
       "3   2021-01-04      1098        0\n",
       "4   2021-01-05      1778        1\n",
       "..         ...       ...      ...\n",
       "360 2021-12-27       536        0\n",
       "361 2021-12-28       277        1\n",
       "362 2021-12-29       842        2\n",
       "363 2021-12-30      1163        3\n",
       "364 2021-12-31      2200        4\n",
       "\n",
       "[365 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%% glimpse at the data\n",
    "dat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "34e20e5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% extract the weekends\n",
    "dat['weekend'] = dat['weekday'].apply(lambda weekday: 0 if (weekday < 5) else 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ee1ee58e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>visitors</th>\n",
       "      <th>weekday</th>\n",
       "      <th>weekend</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>2735</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-01-02</td>\n",
       "      <td>2084</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-01-03</td>\n",
       "      <td>647</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-01-04</td>\n",
       "      <td>1098</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-01-05</td>\n",
       "      <td>1778</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>2021-12-27</td>\n",
       "      <td>536</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>2021-12-28</td>\n",
       "      <td>277</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>2021-12-29</td>\n",
       "      <td>842</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>2021-12-30</td>\n",
       "      <td>1163</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>2021-12-31</td>\n",
       "      <td>2200</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>365 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          date  visitors  weekday  weekend\n",
       "0   2021-01-01      2735        4        0\n",
       "1   2021-01-02      2084        5        1\n",
       "2   2021-01-03       647        6        1\n",
       "3   2021-01-04      1098        0        0\n",
       "4   2021-01-05      1778        1        0\n",
       "..         ...       ...      ...      ...\n",
       "360 2021-12-27       536        0        0\n",
       "361 2021-12-28       277        1        0\n",
       "362 2021-12-29       842        2        0\n",
       "363 2021-12-30      1163        3        0\n",
       "364 2021-12-31      2200        4        0\n",
       "\n",
       "[365 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%% glimpse at the data\n",
    "dat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "36b6f2c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.20095637220308932, pvalue=0.00011081519702575257)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%% correlate the weekends and the visitors\n",
    "spearmanr(dat['weekend'], dat['visitors'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a3ce3e4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEICAYAAACuxNj9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVqklEQVR4nO3dfbBc9X3f8fcnPMWxIcHlYmQJWTIR7gAhcnXD4BI8eOwC8SQBPHYiNTE4ZSJgcA1tUwxppnYz1YyLDSS0NYkcKBATiBziwCRQB7uxsWsIloiCECAjzNPVA8gwMUocK5b49o89t17E3nsu+O7uFft+zezs2e95uN+rkfhwzu+356SqkCRpOj8y7AYkSXOfYSFJamVYSJJaGRaSpFaGhSSplWEhSWrVt7BIcmSSv0rycJKNSS5q6m9McleSR5v3Q7v2uSzJ5iSbkpzWVV+WZEOz7uok6VffkqSXS7++Z5FkHjCvqu5PcjCwDjgT+BDwfFV9IsmlwKFV9dEkxwA3AycAbwa+CBxdVXuS3AdcBNwL3AFcXVV3TvfzDzvssFq0aFFffjdJeq1at27dt6tqbO/6/v36gVW1DdjWLO9M8jAwHzgDOKXZ7Abgy8BHm/otVbULeDzJZuCEJE8Ah1TVPQBJbqQTOtOGxaJFi1i7du3s/lKS9BqX5Mle9YGMWSRZBLwd+GvgTU2QTAbK4c1m84Gnu3abaGrzm+W965KkAel7WCR5A3ArcHFVvTDdpj1qNU29189amWRtkrU7dux45c1Kknrqa1gkOYBOUNxUVX/alJ9pxjMmxzWebeoTwJFduy8Atjb1BT3qL1NVq6tqvKrGx8ZedslNkvQq9XM2VIBrgYer6squVbcD5zTL5wC3ddWXJzkoyWJgCXBfc6lqZ5ITm2Oe3bWPJGkA+jbADZwEfBDYkGR9U/tN4BPAmiTnAk8BHwCoqo1J1gAPAbuBC6tqT7PfBcD1wOvoDGxPO7gtSZpdfZs6O2zj4+PlbChJemWSrKuq8b3rfoNbktTKsJAkternmIUk9dUll1zC9u3bOeKII7j88suH3c5rmmEhaZ+1fft2tmzZMuw2RoKXoSRJrQwLSVIrw0KS1MqwkCS1MiwkSa0MC0lSK8NCktTKsJAktTIsJEmtDAtJUivDQpLUyrCQJLUyLCRJrQwLSVKrvoVFkuuSPJvkwa7aHydZ37yemHw2d5JFSf6xa93vde2zLMmGJJuTXJ0k/epZktRbP59ncT3wP4AbJwtV9cuTy0muAL7Ttf1jVbW0x3GuAVYC9wJ3AKcDd85+u5KkqfTtzKKq7gae77WuOTv4JeDm6Y6RZB5wSFXdU1VFJ3jOnOVWJUkthjVmcTLwTFU92lVbnORvknwlyclNbT4w0bXNRFOTJA3QsB6ruoKXnlVsAxZW1XNJlgF/luRYoNf4RE110CQr6VyyYuHChbPYriSNtoGfWSTZH3gf8MeTtaraVVXPNcvrgMeAo+mcSSzo2n0BsHWqY1fV6qoar6rxsbGxfrQvSSNpGJeh3gM8UlX///JSkrEk+zXLbwWWAN+qqm3AziQnNuMcZwO3DaFnSRpp/Zw6ezNwD/C2JBNJzm1WLeflA9vvBB5I8rfAnwDnV9Xk4PgFwB8Am+mccTgTSpIGrG9jFlW1Yor6h3rUbgVunWL7tcBxs9qcJOkV8RvckqRWhoUkqdWwps5K+iE89ds/NewW5oTdz78R2J/dzz/pnwmw8D9v6NuxPbOQJLUyLCRJrQwLSVIrw0KS1MqwkCS1MiwkSa0MC0lSK8NCktTKsJAktTIsJEmtDAtJUivDQpLUyrCQJLUyLCRJrQwLSVKrfj6D+7okzyZ5sKv28SRbkqxvXu/tWndZks1JNiU5rau+LMmGZt3VSdKvniVJvfXzzOJ64PQe9auqamnzugMgyTHAcuDYZp9PJ9mv2f4aYCWwpHn1OqYkqY/6FhZVdTfw/Aw3PwO4pap2VdXjwGbghCTzgEOq6p6qKuBG4My+NCxJmtIwxiw+nOSB5jLVoU1tPvB01zYTTW1+s7x3XZI0QIMOi2uAo4ClwDbgiqbeaxyipqn3lGRlkrVJ1u7YseOHbFWSNGmgYVFVz1TVnqp6EfgMcEKzagI4smvTBcDWpr6gR32q46+uqvGqGh8bG5vd5iVphA00LJoxiElnAZMzpW4Hlic5KMliOgPZ91XVNmBnkhObWVBnA7cNsmdJEuzfrwMnuRk4BTgsyQTwMeCUJEvpXEp6AjgPoKo2JlkDPATsBi6sqj3NoS6gM7PqdcCdzUuSNEB9C4uqWtGjfO00268CVvWorwWOm8XWJEmvkN/gliS16tuZhST122E/+iKwu3lXPxkWkvZZv3H83w27hZHhZShJUivDQpLUyrCQJLUyLCRJrQwLSVIrw0KS1MqwkCS1MiwkSa0MC0lSK8NCktTKsJAktTIsJEmtDAtJUivDQpLUyrCQJLXqW1gkuS7Js0ke7Kp9MskjSR5I8vkkP9HUFyX5xyTrm9fvde2zLMmGJJuTXJ0k/epZktRbP88srgdO36t2F3BcVR0PfBO4rGvdY1W1tHmd31W/BlgJLGleex9TktRnfQuLqrobeH6v2l9W1e7m473AgumOkWQecEhV3VNVBdwInNmHdiVJ0xjmmMW/Ae7s+rw4yd8k+UqSk5vafGCia5uJpiZJGqChPIM7yX8CdgM3NaVtwMKqei7JMuDPkhwL9BqfqGmOu5LOJSsWLlw4u01L0ggb+JlFknOAnwd+pbm0RFXtqqrnmuV1wGPA0XTOJLovVS0Atk517KpaXVXjVTU+NjbWr19BkkbOQMMiyenAR4FfrKrvdtXHkuzXLL+VzkD2t6pqG7AzyYnNLKizgdsG2bMkqY+XoZLcDJwCHJZkAvgYndlPBwF3NTNg721mPr0T+O0ku4E9wPlVNTk4fgGdmVWvozPG0T3OIUkagL6FRVWt6FG+doptbwVunWLdWuC4WWxNr8All1zC9u3bOeKII7j88suH3Y6kIRnKALf2Hdu3b2fLli3DbkPSkHm7D0lSK8NCktTKsJAktWoNiyQnJXl9s/yrSa5M8pb+tyZJmitmcmZxDfDdJD8NXAI8SeceTZKkETGTsNjdfNP6DOB3q+p3gYP725YkaS6ZydTZnUkuA34VeGfzTesD+tuWJGkumcmZxS8Du4Bzq2o7nbu+frKvXUmS5pRpzyyas4jPVtV7JmtV9RSOWUjSSJk2LKpqT5LvJvnxqvrOoJqaC5b9R/MQ4OBv72Q/4Klv7/TPBFj3ybOH3YI0FDMZs/gesCHJXcA/TBar6iN960qSNKfMJCz+onlJkkZUa1hU1Q1JDqTzMCKATVX1/f62JUmaS1rDIskpwA3AE3Qec3pkknOq6u6+diZJmjNmchnqCuDUqtoEkORo4GZgWT8bkyTNHTP5nsUBk0EBUFXfxC/lSdJImcmZxdok1wJ/2Hz+FWBd/1qSJM01MzmzuADYCHwEuAh4CDivback1yV5NsmDXbU3JrkryaPN+6Fd6y5LsjnJpiSnddWXJdnQrLs6zcO7JUmDM5OwOL+qrqyq91XVWVV1FZ0AaXM9cPpetUuBL1XVEuBLzWeSHAMsB45t9vl08+1x6Nz1diWwpHntfUxJUp/NJCzO6VH7UNtOzWyp5/cqn0FnZhXN+5ld9VuqaldVPQ5sBk5IMg84pKruae58e2PXPpKkAZlyzCLJCuBfA4uT3N616mDguVf5895UVdsAqmpbksOb+nzg3q7tJpra95vlvetT9bySzlkICxcufJUtqtuLB77+Je+SRtN0A9xfB7YBh9GZPjtpJ/DALPfRaxyipqn3VFWrgdUA4+PjU26nmfuHJacOuwVJc8CUYVFVT9J5Kt47ZvHnPZNkXnNWMQ94tqlPAEd2bbcA2NrUF/SoS5IGaMoxiyRfa953Jnmh67UzyQuv8ufdzg/GQM4BbuuqL09yUJLFdAay72suWe1McmIzC+rsrn0kSQMy3ZnFzzbvr+oRqkluBk4BDksyAXwM+ASwJsm5wFPAB5qfsTHJGjrTcncDF1bVnuZQF9CZWfU64M7mJUkaoJncG+ooYKKqdjX3iToeuLGq/m66/apqxRSr3j3F9quAVT3qa4Hj2vqUJPXPTKbO3grsSfKTwLXAYuCP+tqVJGlOmUlYvFhVu4GzgN+pqn8HzOtvW5KkuWQmYfH95jsX5wB/3tS8kaAkjZCZhMWv0Zk+u6qqHm9mK322v21JkuaSmTwp7yE6NxGc/Pw4nVlNkqQRMd3tPtZU1S8l2cBLvzUdoKrq+L53J0maE6Y7s7ioef9fwH3A0/1vR5I0F005ZjF5wz86Nw78fTrjFD8PfK+5FYgkaUS0DnBX1X+pqmOBC4E3A19J8sW+dyZJmjNmMhtq0rPAdjq3Jz+8ZVtJ0mtIa1gkuSDJl+k82e4w4Ncd3Jak0dI6dRZ4C3BxVa3vcy+SpDlqJt+zuHQQjUiS5q5XMmYhSRpRhoUkqZVhIUlqZVhIkloNPCySvC3J+q7XC0kuTvLxJFu66u/t2ueyJJuTbEpy2qB7lqRRN5Ops7OqqjYBSwGS7AdsAT5P51boV1XVp7q3T3IMsBw4ls43yL+Y5OiuZ3RLkvps2Jeh3g081nKvqTOAW6pqV3N79M3ACQPpTpIEDD8slgM3d33+cJIHklyX5NCmNp+X3vF2oqlJkgZkaGGR5EDgF4HPNaVrgKPoXKLaBlwxuWmP3atHjSQrk6xNsnbHjh2z27AkjbBhnln8HHB/VT0DUFXPVNWeqnoR+Aw/uNQ0ARzZtd8CYGuvA1bV6qoar6rxsbGxPrYuSaNlmGGxgq5LUEnmda07C3iwWb4dWJ7koOb530voPIxJkjQgA58NBZDkx4B/BZzXVb48yVI6l5iemFxXVRuTrAEeAnYDFzoTSpIGayhhUVXfBf7ZXrUPTrP9KmBVv/uSJPU27NlQkqR9gGEhSWplWEiSWhkWkqRWhoUkqZVhIUlqZVhIkloZFpKkVoaFJKmVYSFJamVYSJJaGRaSpFaGhSSplWEhSWplWEiSWhkWkqRWhoUkqZVhIUlqNZSwSPJEkg1J1idZ29TemOSuJI8274d2bX9Zks1JNiU5bRg9S9IoG+aZxbuqamlVjTefLwW+VFVLgC81n0lyDLAcOBY4Hfh0kv2G0bAkjaq5dBnqDOCGZvkG4Myu+i1VtauqHgc2AycMvj1JGl3DCosC/jLJuiQrm9qbqmobQPN+eFOfDzzdte9EU5MkDcj+Q/q5J1XV1iSHA3cleWSabdOjVj037ATPSoCFCxf+8F1KkoAhnVlU1dbm/Vng83QuKz2TZB5A8/5ss/kEcGTX7guArVMcd3VVjVfV+NjYWL/al6SRM/CwSPL6JAdPLgOnAg8CtwPnNJudA9zWLN8OLE9yUJLFwBLgvsF2LUmjbRiXod4EfD7J5M//o6r630m+AaxJci7wFPABgKramGQN8BCwG7iwqvYMoW9JGlkDD4uq+hbw0z3qzwHvnmKfVcCqPrcmSZrCXJo6K0maowwLSVIrw0KS1MqwkCS1MiwkSa0MC0lSK8NCktTKsJAktTIsJEmtDAtJUivDQpLUyrCQJLUyLCRJrQwLSVIrw0KS1MqwkCS1MiwkSa0MC0lSq4GHRZIjk/xVkoeTbExyUVP/eJItSdY3r/d27XNZks1JNiU5bdA9S9KoG/gzuIHdwH+oqvuTHAysS3JXs+6qqvpU98ZJjgGWA8cCbwa+mOToqtoz0K4laYQN/MyiqrZV1f3N8k7gYWD+NLucAdxSVbuq6nFgM3BC/zuVJE0a6phFkkXA24G/bkofTvJAkuuSHNrU5gNPd+02wfThIkmaZUMLiyRvAG4FLq6qF4BrgKOApcA24IrJTXvsXlMcc2WStUnW7tixY/ablqQRNZSwSHIAnaC4qar+FKCqnqmqPVX1IvAZfnCpaQI4smv3BcDWXsetqtVVNV5V42NjY/37BSRpxAxjNlSAa4GHq+rKrvq8rs3OAh5slm8Hlic5KMliYAlw36D6lSQNZzbUScAHgQ1J1je13wRWJFlK5xLTE8B5AFW1Mcka4CE6M6kudCaUJA3WwMOiqr5G73GIO6bZZxWwqm9NSZKm5Te4JUmtDAtJUivDQpLUyrCQJLUyLCRJrQwLSVIrw0KS1MqwkCS1MiwkSa0MC0lSK8NCktTKsJAktTIsJEmtDAtJUivDQpLUyrCQJLUyLCRJrQwLSVKrfSYskpyeZFOSzUkuHXY/kjRK9omwSLIf8D+BnwOOAVYkOWa4XUnS6NgnwgI4AdhcVd+qqn8CbgHOGHJPkjQy9pWwmA883fV5oqlJkgZg/2E3MEPpUauXbZSsBFY2H/8+yaa+djU6DgO+Pewm5oJ86pxht6CX8+/npI/1+k/lK/aWXsV9JSwmgCO7Pi8Atu69UVWtBlYPqqlRkWRtVY0Puw+pF/9+Dsa+chnqG8CSJIuTHAgsB24fck+SNDL2iTOLqtqd5MPAF4D9gOuqauOQ25KkkbFPhAVAVd0B3DHsPkaUl/Y0l/n3cwBS9bJxYkmSXmJfGbOQJA2RYaFpeZsVzVVJrkvybJIHh93LKDAsNCVvs6I57nrg9GE3MSoMC03H26xozqqqu4Hnh93HqDAsNB1vsyIJMCw0vRndZkXSa59hoenM6DYrkl77DAtNx9usSAIMC02jqnYDk7dZeRhY421WNFckuRm4B3hbkokk5w67p9cyv8EtSWrlmYUkqZVhIUlqZVhIkloZFpKkVoaFJKmVYSENSJIvJ5nVZ0UnOSXJn8/mMaVeDAtJUivDQppCkkuSfKRZvirJ/2mW353ks0lOTXJPkvuTfC7JG5r1y5J8Jcm6JF9IMm+v4/5IkhuS/Nck+yX5ZJJvJHkgyXnNNqc0ZyJ/kuSRJDclSbPu9Kb2NeB9A/1D0cgyLKSp3Q2c3CyPA29IcgDws8AG4LeA91TVvwDWAv++Wf/fgfdX1TLgOmBV1zH3B24CvllVvwWcC3ynqn4G+Bng15MsbrZ9O3AxnWeJvBU4KcmPAp8BfqHp7Yh+/OLS3vYfdgPSHLYOWJbkYGAXcD+d0DiZzj2yjgH+b/M//AfS3HoCOA64q6nvB2zrOubv07ltymSAnAocn+T9zecfB5YA/wTcV1UTAEnWA4uAvwcer6pHm/pngZWz/HtLL2NYSFOoqu8neQL4NeDrwAPAu4CjgMeBu6pqRfc+SX4K2FhV75jisF8H3pXkiqr6Hp3bwP/bqvrCXsc5hU5ATdrDD/69eo8eDZyXoaTp3Q38RvP+VeB8YD1wL53LQj8JkOTHkhwNbALGkryjqR+Q5Niu410L3AF8Lsn+dG7SeEFz+YokRyd5/TT9PAIsTnJU83nFNNtKs8awkKb3VWAecE9VPQN8D/hqVe0APgTcnOQBOuHxz5vHz74f+G9J/pZOsPzL7gNW1ZV0Lmn9IfAHwEPA/UkepHOZasoz/uZsZCXwF80A95Oz96tKU/Ous5KkVp5ZSJJaGRaSpFaGhSSplWEhSWplWEiSWhkWkqRWhoUkqZVhIUlq9f8ARc8Y41SIoFwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#%% visually glimpse at the data\n",
    "ax = sns.barplot(x=\"weekend\", y=\"visitors\", data=dat)\n",
    "# %%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84e50cbb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}