notebooks/01_make_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **Objetivo**\n",
    "- Ler todos os arquivos CSV brutos separados, processa-los e mesclá-los em um único conjunto de dados\n",
    "\n",
    "#### **Informações Complementares**\n",
    " - Os dados foram coletados usando um aplicativo de aquisição de dados instalado em um smart watch Apple.\n",
    "- O smart watch contém tanto um acelerômetro quanto um giroscópio, que foram usados para coletar dados sobre os movimentos de exercícios.\n",
    "- Os sensores foram configurados com as configurações padrão: o acelerômetro a 12.500Hz e o giroscópio a 25.000Hz.\n",
    "- Cinco participantes foram recrutados para realizar exercícios básicos com barra em séries de 5 (heavy) e 10 (medium) repetições (supino: bench, levantamento terra: dead, desenvolvimento de ombro (militar): ohp, remada alta: row e agachamento livre: squat).\n",
    "- O conjunto de dados bruto continham data/hora e valores x, y e z dos sensors (acelerômetro e giroscópio).\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from glob import glob\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "164"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --------------------------------------------------------------\n",
    "# List all data in data/raw/MetaMotion\n",
    "# --------------------------------------------------------------\n",
    "\n",
    "files = glob(\"../data/raw/*.csv\")\n",
    "len(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch (ms)</th>\n",
       "      <th>time (01:00)</th>\n",
       "      <th>elapsed (s)</th>\n",
       "      <th>x-axis (g)</th>\n",
       "      <th>y-axis (g)</th>\n",
       "      <th>z-axis (g)</th>\n",
       "      <th>participant</th>\n",
       "      <th>label</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1547219408431</td>\n",
       "      <td>2019-01-11T16:10:08.431</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.010</td>\n",
       "      <td>0.964</td>\n",
       "      <td>-0.087</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1547219408511</td>\n",
       "      <td>2019-01-11T16:10:08.511</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.961</td>\n",
       "      <td>-0.069</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1547219408591</td>\n",
       "      <td>2019-01-11T16:10:08.591</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.974</td>\n",
       "      <td>-0.087</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      epoch (ms)             time (01:00)  elapsed (s)  x-axis (g)  \\\n",
       "0  1547219408431  2019-01-11T16:10:08.431         0.00       0.010   \n",
       "1  1547219408511  2019-01-11T16:10:08.511         0.08       0.000   \n",
       "2  1547219408591  2019-01-11T16:10:08.591         0.16       0.001   \n",
       "\n",
       "   y-axis (g)  z-axis (g) participant  label category  \n",
       "0       0.964      -0.087           A  bench    heavy  \n",
       "1       0.961      -0.069           A  bench    heavy  \n",
       "2       0.974      -0.087           A  bench    heavy  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --------------------------------------------------------------\n",
    "# Extract features from filename\n",
    "# --------------------------------------------------------------\n",
    "\n",
    "# Extracting 3 values of file name:\n",
    "# participant,\n",
    "# label (exercise) and\n",
    "# categoty (light or heavy)\n",
    "\n",
    "f = files[0]\n",
    "f.split(\"-\")[0]\n",
    "f.split(\"-\")[1]\n",
    "f.split(\"-\")[2].split(\"_\")[0]\n",
    "\n",
    "data_path = \"../data/raw\\\\\"\n",
    "\n",
    "# Extrating the participant\n",
    "participant = f.split(\"-\")[0].replace(data_path, \"\")\n",
    "# Extracting the label\n",
    "label = f.split(\"-\")[1]\n",
    "\n",
    "# Removing number in any string\n",
    "def remove_numbers(string):\n",
    "    return re.sub(r\"\\d+\", \"\", string)\n",
    "\n",
    "\n",
    "category = remove_numbers(f.split(\"-\")[2].split(\"_\")[0])  # Extracting the category\n",
    "\n",
    "# Adding extra columns\n",
    "df = pd.read_csv(f)\n",
    "df[\"participant\"] = f.split(\"-\")[0].replace(data_path, \"\")\n",
    "df[\"label\"] = f.split(\"-\")[1]\n",
    "df[\"category\"] = remove_numbers(f.split(\"-\")[2].split(\"_\")[0])\n",
    "\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch (ms)</th>\n",
       "      <th>time (01:00)</th>\n",
       "      <th>elapsed (s)</th>\n",
       "      <th>x-axis (deg/s)</th>\n",
       "      <th>y-axis (deg/s)</th>\n",
       "      <th>z-axis (deg/s)</th>\n",
       "      <th>participant</th>\n",
       "      <th>label</th>\n",
       "      <th>category</th>\n",
       "      <th>set</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1547666043834</td>\n",
       "      <td>2019-01-16 20:14:03.834</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-10.549</td>\n",
       "      <td>1.646</td>\n",
       "      <td>-1.280</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1547666043874</td>\n",
       "      <td>2019-01-16 20:14:03.874</td>\n",
       "      <td>0.04</td>\n",
       "      <td>-8.963</td>\n",
       "      <td>1.524</td>\n",
       "      <td>-0.732</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1547666043914</td>\n",
       "      <td>2019-01-16 20:14:03.914</td>\n",
       "      <td>0.08</td>\n",
       "      <td>-2.378</td>\n",
       "      <td>-1.280</td>\n",
       "      <td>0.305</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      epoch (ms)             time (01:00)  elapsed (s)  x-axis (deg/s)  \\\n",
       "0  1547666043834  2019-01-16 20:14:03.834         0.00         -10.549   \n",
       "1  1547666043874  2019-01-16 20:14:03.874         0.04          -8.963   \n",
       "2  1547666043914  2019-01-16 20:14:03.914         0.08          -2.378   \n",
       "\n",
       "   y-axis (deg/s)  z-axis (deg/s) participant  label category  set  \n",
       "0           1.646          -1.280           E  squat    heavy   82  \n",
       "1           1.524          -0.732           E  squat    heavy   82  \n",
       "2          -1.280           0.305           E  squat    heavy   82  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch (ms)</th>\n",
       "      <th>time (01:00)</th>\n",
       "      <th>elapsed (s)</th>\n",
       "      <th>x-axis (deg/s)</th>\n",
       "      <th>y-axis (deg/s)</th>\n",
       "      <th>z-axis (deg/s)</th>\n",
       "      <th>participant</th>\n",
       "      <th>label</th>\n",
       "      <th>category</th>\n",
       "      <th>set</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>1547666063394</td>\n",
       "      <td>2019-01-16 20:14:23.394</td>\n",
       "      <td>19.56</td>\n",
       "      <td>5.854</td>\n",
       "      <td>-2.561</td>\n",
       "      <td>1.463</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490</th>\n",
       "      <td>1547666063434</td>\n",
       "      <td>2019-01-16 20:14:23.434</td>\n",
       "      <td>19.60</td>\n",
       "      <td>4.268</td>\n",
       "      <td>0.549</td>\n",
       "      <td>-0.305</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>1547666063474</td>\n",
       "      <td>2019-01-16 20:14:23.474</td>\n",
       "      <td>19.64</td>\n",
       "      <td>-1.098</td>\n",
       "      <td>1.768</td>\n",
       "      <td>-0.122</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        epoch (ms)             time (01:00)  elapsed (s)  x-axis (deg/s)  \\\n",
       "489  1547666063394  2019-01-16 20:14:23.394        19.56           5.854   \n",
       "490  1547666063434  2019-01-16 20:14:23.434        19.60           4.268   \n",
       "491  1547666063474  2019-01-16 20:14:23.474        19.64          -1.098   \n",
       "\n",
       "     y-axis (deg/s)  z-axis (deg/s) participant  label category  set  \n",
       "489          -2.561           1.463           E  squat    heavy   82  \n",
       "490           0.549          -0.305           E  squat    heavy   82  \n",
       "491           1.768          -0.122           E  squat    heavy   82  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# --------------------------------------------------------------\n",
    "# Read all files\n",
    "# --------------------------------------------------------------\n",
    "\n",
    "# Create df to store acc anf gyr\n",
    "acc_df = pd.DataFrame()\n",
    "gyr_df = pd.DataFrame()\n",
    "\n",
    "# Creating a set to increment alfter each file is read\n",
    "acc_set = 1\n",
    "gyr_set = 1\n",
    "\n",
    "# Lopping through all files\n",
    "for f in files:\n",
    "    # Reading the file\n",
    "    df = pd.read_csv(f)\n",
    "    # Extracting the participant\n",
    "    participant = f.split(\"-\")[0].replace(data_path, \"\")\n",
    "    # Extracting the label\n",
    "    label = f.split(\"-\")[1]\n",
    "    # Removing number in any string\n",
    "    category = remove_numbers(f.split(\"-\")[2].split(\"_\")[0])  # Extracting the category\n",
    "    df[\"participant\"] = participant\n",
    "    df[\"label\"] = label\n",
    "    df[\"category\"] = category\n",
    "    # If exist the word 'accelerometer' in the file name\n",
    "    if \"Accelerometer\" in f:\n",
    "        # Adding to acc_df\n",
    "        df[\"set\"] = acc_set\n",
    "        acc_df = pd.concat([acc_df, df])\n",
    "        acc_set += 1\n",
    "    if \"Gyroscope\" in f:\n",
    "        # Adding to gyr_df\n",
    "        df[\"set\"] = gyr_set\n",
    "        gyr_df = pd.concat([gyr_df, df])\n",
    "        gyr_set += 1\n",
    "\n",
    "display(df.head(3))\n",
    "display(df.tail(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 20334 entries, 0 to 241\n",
      "Data columns (total 10 columns):\n",
      " #   Column        Non-Null Count  Dtype  \n",
      "---  ------        --------------  -----  \n",
      " 0   epoch (ms)    20334 non-null  int64  \n",
      " 1   time (01:00)  20334 non-null  object \n",
      " 2   elapsed (s)   20334 non-null  float64\n",
      " 3   x-axis (g)    20334 non-null  float64\n",
      " 4   y-axis (g)    20334 non-null  float64\n",
      " 5   z-axis (g)    20334 non-null  float64\n",
      " 6   participant   20334 non-null  object \n",
      " 7   label         20334 non-null  object \n",
      " 8   category      20334 non-null  object \n",
      " 9   set           20334 non-null  int64  \n",
      "dtypes: float64(4), int64(2), object(4)\n",
      "memory usage: 1.7+ MB\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "None"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0     2019-01-16 19:14:03.834\n",
       "1     2019-01-16 19:14:03.874\n",
       "2     2019-01-16 19:14:03.914\n",
       "3     2019-01-16 19:14:03.954\n",
       "4     2019-01-16 19:14:03.994\n",
       "                ...          \n",
       "487   2019-01-16 19:14:23.314\n",
       "488   2019-01-16 19:14:23.354\n",
       "489   2019-01-16 19:14:23.394\n",
       "490   2019-01-16 19:14:23.434\n",
       "491   2019-01-16 19:14:23.474\n",
       "Name: epoch (ms), Length: 492, dtype: datetime64[ns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --------------------------------------------------------------\n",
    "# Working with datetimes\n",
    "# --------------------------------------------------------------\n",
    "display(acc_df.info())\n",
    "\n",
    "# Converting UNIX time to datetime\n",
    "pd.to_datetime(df[\"epoch (ms)\"], unit=\"ms\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      2\n",
       "1      2\n",
       "2      2\n",
       "3      2\n",
       "4      2\n",
       "      ..\n",
       "487    2\n",
       "488    2\n",
       "489    2\n",
       "490    2\n",
       "491    2\n",
       "Name: time (01:00), Length: 492, dtype: int32"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Conveting object to datetime\n",
    "pd.to_datetime(df[\"time (01:00)\"]).dt.weekday"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x-axis (g)</th>\n",
       "      <th>y-axis (g)</th>\n",
       "      <th>z-axis (g)</th>\n",
       "      <th>participant</th>\n",
       "      <th>label</th>\n",
       "      <th>category</th>\n",
       "      <th>set</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epoch (ms)</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-01-19 17:30:49.220</th>\n",
       "      <td>-0.051</td>\n",
       "      <td>-1.026</td>\n",
       "      <td>-0.088</td>\n",
       "      <td>E</td>\n",
       "      <td>row</td>\n",
       "      <td>medium</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-19 17:30:49.300</th>\n",
       "      <td>-0.054</td>\n",
       "      <td>-1.038</td>\n",
       "      <td>-0.093</td>\n",
       "      <td>E</td>\n",
       "      <td>row</td>\n",
       "      <td>medium</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-19 17:30:49.380</th>\n",
       "      <td>-0.052</td>\n",
       "      <td>-1.034</td>\n",
       "      <td>-0.095</td>\n",
       "      <td>E</td>\n",
       "      <td>row</td>\n",
       "      <td>medium</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-19 17:30:49.460</th>\n",
       "      <td>-0.063</td>\n",
       "      <td>-1.011</td>\n",
       "      <td>-0.095</td>\n",
       "      <td>E</td>\n",
       "      <td>row</td>\n",
       "      <td>medium</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-19 17:30:49.540</th>\n",
       "      <td>-0.052</td>\n",
       "      <td>-1.017</td>\n",
       "      <td>-0.090</td>\n",
       "      <td>E</td>\n",
       "      <td>row</td>\n",
       "      <td>medium</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-19 17:35:13.381</th>\n",
       "      <td>-0.060</td>\n",
       "      <td>-1.021</td>\n",
       "      <td>-0.058</td>\n",
       "      <td>E</td>\n",
       "      <td>row</td>\n",
       "      <td>medium</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-19 17:35:13.461</th>\n",
       "      <td>-0.035</td>\n",
       "      <td>-1.037</td>\n",
       "      <td>-0.026</td>\n",
       "      <td>E</td>\n",
       "      <td>row</td>\n",
       "      <td>medium</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-19 17:35:13.541</th>\n",
       "      <td>-0.045</td>\n",
       "      <td>-1.029</td>\n",
       "      <td>-0.033</td>\n",
       "      <td>E</td>\n",
       "      <td>row</td>\n",
       "      <td>medium</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-19 17:35:13.621</th>\n",
       "      <td>-0.039</td>\n",
       "      <td>-1.027</td>\n",
       "      <td>-0.039</td>\n",
       "      <td>E</td>\n",
       "      <td>row</td>\n",
       "      <td>medium</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-19 17:35:13.701</th>\n",
       "      <td>-0.049</td>\n",
       "      <td>-1.031</td>\n",
       "      <td>-0.049</td>\n",
       "      <td>E</td>\n",
       "      <td>row</td>\n",
       "      <td>medium</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>770 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         x-axis (g)  y-axis (g)  z-axis (g) participant label  \\\n",
       "epoch (ms)                                                                      \n",
       "2019-01-19 17:30:49.220      -0.051      -1.026      -0.088           E   row   \n",
       "2019-01-19 17:30:49.300      -0.054      -1.038      -0.093           E   row   \n",
       "2019-01-19 17:30:49.380      -0.052      -1.034      -0.095           E   row   \n",
       "2019-01-19 17:30:49.460      -0.063      -1.011      -0.095           E   row   \n",
       "2019-01-19 17:30:49.540      -0.052      -1.017      -0.090           E   row   \n",
       "...                             ...         ...         ...         ...   ...   \n",
       "2019-01-19 17:35:13.381      -0.060      -1.021      -0.058           E   row   \n",
       "2019-01-19 17:35:13.461      -0.035      -1.037      -0.026           E   row   \n",
       "2019-01-19 17:35:13.541      -0.045      -1.029      -0.033           E   row   \n",
       "2019-01-19 17:35:13.621      -0.039      -1.027      -0.039           E   row   \n",
       "2019-01-19 17:35:13.701      -0.049      -1.031      -0.049           E   row   \n",
       "\n",
       "                        category  set  \n",
       "epoch (ms)                             \n",
       "2019-01-19 17:30:49.220   medium   78  \n",
       "2019-01-19 17:30:49.300   medium   78  \n",
       "2019-01-19 17:30:49.380   medium   78  \n",
       "2019-01-19 17:30:49.460   medium   78  \n",
       "2019-01-19 17:30:49.540   medium   78  \n",
       "...                          ...  ...  \n",
       "2019-01-19 17:35:13.381   medium   80  \n",
       "2019-01-19 17:35:13.461   medium   80  \n",
       "2019-01-19 17:35:13.541   medium   80  \n",
       "2019-01-19 17:35:13.621   medium   80  \n",
       "2019-01-19 17:35:13.701   medium   80  \n",
       "\n",
       "[770 rows x 7 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Set time to index\n",
    "acc_df.index = pd.to_datetime(acc_df[\"epoch (ms)\"], unit=\"ms\")\n",
    "gyr_df.index = pd.to_datetime(gyr_df[\"epoch (ms)\"], unit=\"ms\")\n",
    "\n",
    "# Removing columns\n",
    "del acc_df[\"epoch (ms)\"]\n",
    "del acc_df[\"time (01:00)\"]\n",
    "del acc_df[\"elapsed (s)\"]\n",
    "\n",
    "del gyr_df[\"epoch (ms)\"]\n",
    "del gyr_df[\"time (01:00)\"]\n",
    "del gyr_df[\"elapsed (s)\"]\n",
    "\n",
    "# A single exemple\n",
    "acc_df.query('participant == \"E\"').query('label == \"row\" ').query(\n",
    "    'category == \"medium\" '\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x-axis (g)</th>\n",
       "      <th>y-axis (g)</th>\n",
       "      <th>z-axis (g)</th>\n",
       "      <th>participant</th>\n",
       "      <th>label</th>\n",
       "      <th>category</th>\n",
       "      <th>set</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epoch (ms)</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-01-11 15:10:08.431</th>\n",
       "      <td>0.010</td>\n",
       "      <td>0.964</td>\n",
       "      <td>-0.087</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-11 15:10:08.511</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.961</td>\n",
       "      <td>-0.069</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-11 15:10:08.591</th>\n",
       "      <td>0.001</td>\n",
       "      <td>0.974</td>\n",
       "      <td>-0.087</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-11 15:10:08.671</th>\n",
       "      <td>-0.012</td>\n",
       "      <td>0.971</td>\n",
       "      <td>-0.084</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-11 15:10:08.751</th>\n",
       "      <td>-0.013</td>\n",
       "      <td>0.954</td>\n",
       "      <td>-0.094</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-16 19:14:23.089</th>\n",
       "      <td>0.012</td>\n",
       "      <td>0.596</td>\n",
       "      <td>0.815</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-16 19:14:23.169</th>\n",
       "      <td>0.009</td>\n",
       "      <td>0.528</td>\n",
       "      <td>0.821</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-16 19:14:23.249</th>\n",
       "      <td>0.015</td>\n",
       "      <td>0.554</td>\n",
       "      <td>0.746</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-16 19:14:23.329</th>\n",
       "      <td>0.006</td>\n",
       "      <td>0.574</td>\n",
       "      <td>0.824</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-16 19:14:23.409</th>\n",
       "      <td>0.015</td>\n",
       "      <td>0.588</td>\n",
       "      <td>0.826</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20334 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         x-axis (g)  y-axis (g)  z-axis (g) participant  \\\n",
       "epoch (ms)                                                                \n",
       "2019-01-11 15:10:08.431       0.010       0.964      -0.087           A   \n",
       "2019-01-11 15:10:08.511       0.000       0.961      -0.069           A   \n",
       "2019-01-11 15:10:08.591       0.001       0.974      -0.087           A   \n",
       "2019-01-11 15:10:08.671      -0.012       0.971      -0.084           A   \n",
       "2019-01-11 15:10:08.751      -0.013       0.954      -0.094           A   \n",
       "...                             ...         ...         ...         ...   \n",
       "2019-01-16 19:14:23.089       0.012       0.596       0.815           E   \n",
       "2019-01-16 19:14:23.169       0.009       0.528       0.821           E   \n",
       "2019-01-16 19:14:23.249       0.015       0.554       0.746           E   \n",
       "2019-01-16 19:14:23.329       0.006       0.574       0.824           E   \n",
       "2019-01-16 19:14:23.409       0.015       0.588       0.826           E   \n",
       "\n",
       "                         label category  set  \n",
       "epoch (ms)                                    \n",
       "2019-01-11 15:10:08.431  bench    heavy    1  \n",
       "2019-01-11 15:10:08.511  bench    heavy    1  \n",
       "2019-01-11 15:10:08.591  bench    heavy    1  \n",
       "2019-01-11 15:10:08.671  bench    heavy    1  \n",
       "2019-01-11 15:10:08.751  bench    heavy    1  \n",
       "...                        ...      ...  ...  \n",
       "2019-01-16 19:14:23.089  squat    heavy   82  \n",
       "2019-01-16 19:14:23.169  squat    heavy   82  \n",
       "2019-01-16 19:14:23.249  squat    heavy   82  \n",
       "2019-01-16 19:14:23.329  squat    heavy   82  \n",
       "2019-01-16 19:14:23.409  squat    heavy   82  \n",
       "\n",
       "[20334 rows x 7 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x-axis (deg/s)</th>\n",
       "      <th>y-axis (deg/s)</th>\n",
       "      <th>z-axis (deg/s)</th>\n",
       "      <th>participant</th>\n",
       "      <th>label</th>\n",
       "      <th>category</th>\n",
       "      <th>set</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epoch (ms)</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-01-11 15:10:08.351</th>\n",
       "      <td>0.122</td>\n",
       "      <td>-5.488</td>\n",
       "      <td>-3.841</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-11 15:10:08.391</th>\n",
       "      <td>2.195</td>\n",
       "      <td>-9.695</td>\n",
       "      <td>-0.610</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-11 15:10:08.431</th>\n",
       "      <td>2.622</td>\n",
       "      <td>-8.110</td>\n",
       "      <td>-4.024</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-11 15:10:08.471</th>\n",
       "      <td>1.951</td>\n",
       "      <td>-4.695</td>\n",
       "      <td>-4.634</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-11 15:10:08.511</th>\n",
       "      <td>1.524</td>\n",
       "      <td>-2.561</td>\n",
       "      <td>-2.500</td>\n",
       "      <td>A</td>\n",
       "      <td>bench</td>\n",
       "      <td>heavy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-16 19:14:23.314</th>\n",
       "      <td>-1.707</td>\n",
       "      <td>-0.671</td>\n",
       "      <td>1.585</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-16 19:14:23.354</th>\n",
       "      <td>0.915</td>\n",
       "      <td>0.305</td>\n",
       "      <td>2.988</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-16 19:14:23.394</th>\n",
       "      <td>5.854</td>\n",
       "      <td>-2.561</td>\n",
       "      <td>1.463</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-16 19:14:23.434</th>\n",
       "      <td>4.268</td>\n",
       "      <td>0.549</td>\n",
       "      <td>-0.305</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-01-16 19:14:23.474</th>\n",
       "      <td>-1.098</td>\n",
       "      <td>1.768</td>\n",
       "      <td>-0.122</td>\n",
       "      <td>E</td>\n",
       "      <td>squat</td>\n",
       "      <td>heavy</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>41184 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         x-axis (deg/s)  y-axis (deg/s)  z-axis (deg/s)  \\\n",
       "epoch (ms)                                                                \n",
       "2019-01-11 15:10:08.351           0.122          -5.488          -3.841   \n",
       "2019-01-11 15:10:08.391           2.195          -9.695          -0.610   \n",
       "2019-01-11 15:10:08.431           2.622          -8.110          -4.024   \n",
       "2019-01-11 15:10:08.471           1.951          -4.695          -4.634   \n",
       "2019-01-11 15:10:08.511           1.524          -2.561          -2.500   \n",
       "...                                 ...             ...             ...   \n",
       "2019-01-16 19:14:23.314          -1.707          -0.671           1.585   \n",
       "2019-01-16 19:14:23.354           0.915           0.305           2.988   \n",
       "2019-01-16 19:14:23.394           5.854          -2.561           1.463   \n",
       "2019-01-16 19:14:23.434           4.268           0.549          -0.305   \n",
       "2019-01-16 19:14:23.474          -1.098           1.768          -0.122   \n",
       "\n",
       "                        participant  label category  set  \n",
       "epoch (ms)                                                \n",
       "2019-01-11 15:10:08.351           A  bench    heavy    1  \n",
       "2019-01-11 15:10:08.391           A  bench    heavy    1  \n",
       "2019-01-11 15:10:08.431           A  bench    heavy    1  \n",
       "2019-01-11 15:10:08.471           A  bench    heavy    1  \n",
       "2019-01-11 15:10:08.511           A  bench    heavy    1  \n",
       "...                             ...    ...      ...  ...  \n",
       "2019-01-16 19:14:23.314           E  squat    heavy   82  \n",
       "2019-01-16 19:14:23.354           E  squat    heavy   82  \n",
       "2019-01-16 19:14:23.394           E  squat    heavy   82  \n",
       "2019-01-16 19:14:23.434           E  squat    heavy   82  \n",
       "2019-01-16 19:14:23.474           E  squat    heavy   82  \n",
       "\n",
       "[41184 rows x 7 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# --------------------------------------------------------------\n",
    "# Turn into a single function (read_data_from_files)\n",
    "# --------------------------------------------------------------\n",
    "\n",
    "files = glob(\"../data/raw/*.csv\")\n",
    "data_path = \"../data/raw\\\\\"\n",
    "\n",
    "# Removing number in any string\n",
    "def remove_numbers(string):\n",
    "    return re.sub(r\"\\d+\", \"\", string)\n",
    "\n",
    "\n",
    "def read_data_from_files(files: list, data_path: str) -> tuple:\n",
    "    \"\"\"\n",
    "    Read data from a list of files and extract & clean accelerometer and gyroscope data.\n",
    "\n",
    "    Args:\n",
    "        files (list): List of file paths to read data from.\n",
    "        data_path (str): Path to the data directory.\n",
    "\n",
    "    Returns:\n",
    "        tuple: A tuple containing two DataFrames - `acc_df` (accelerometer data) and `gyr_df` (gyroscope data).\n",
    "    \"\"\"\n",
    "    # Create empty DataFrames to store accelerometer and gyroscope data\n",
    "    acc_df = pd.DataFrame()\n",
    "    gyr_df = pd.DataFrame()\n",
    "\n",
    "    # Set initial values for set counters\n",
    "    acc_set = 1\n",
    "    gyr_set = 1\n",
    "\n",
    "    # Loop through all files\n",
    "    for f in files:\n",
    "        # Reading the file\n",
    "        df = pd.read_csv(f)\n",
    "\n",
    "        # Extracting participant, label, and category from the file name\n",
    "        participant = f.split(\"-\")[0].replace(data_path, \"\")\n",
    "        label = f.split(\"-\")[1]\n",
    "        category = remove_numbers(f.split(\"-\")[2].split(\"_\")[0])\n",
    "\n",
    "        # Add participant, label, and category columns to the DataFrame\n",
    "        df[\"participant\"] = participant\n",
    "        df[\"label\"] = label\n",
    "        df[\"category\"] = category\n",
    "\n",
    "        # Check if the file contains accelerometer data\n",
    "        if \"Accelerometer\" in f:\n",
    "            # Add set number to the DataFrame\n",
    "            df[\"set\"] = acc_set\n",
    "            # Concatenate the DataFrame to the acc_df\n",
    "            acc_df = pd.concat([acc_df, df])\n",
    "            # Increment the acc_set counter\n",
    "            acc_set += 1\n",
    "\n",
    "        # Check if the file contains gyroscope data\n",
    "        if \"Gyroscope\" in f:\n",
    "            # Add set number to the DataFrame\n",
    "            df[\"set\"] = gyr_set\n",
    "            # Concatenate the DataFrame to the gyr_df\n",
    "            gyr_df = pd.concat([gyr_df, df])\n",
    "            # Increment the gyr_set counter\n",
    "            gyr_set += 1\n",
    "\n",
    "    # Set time as the DataFrame index\n",
    "    acc_df.index = pd.to_datetime(acc_df[\"epoch (ms)\"], unit=\"ms\")\n",
    "    gyr_df.index = pd.to_datetime(gyr_df[\"epoch (ms)\"], unit=\"ms\")\n",
    "\n",
    "    # Remove unnecessary columns from the DataFrames\n",
    "    columns_to_remove = [\"epoch (ms)\", \"time (01:00)\", \"elapsed (s)\"]\n",
    "    acc_df.drop(columns_to_remove, axis=1, inplace=True)\n",
    "    gyr_df.drop(columns_to_remove, axis=1, inplace=True)\n",
    "\n",
    "    return acc_df, gyr_df\n",
    "\n",
    "\n",
    "acc_df, gyr_df = read_data_from_files(files, data_path)\n",
    "\n",
    "display(acc_df)\n",
    "display(gyr_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 1119 entries, 2019-01-11 15:10:08.431000 to 2019-01-15 19:04:25.881000\n",
      "Data columns (total 10 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   acc_x        1119 non-null   float64\n",
      " 1   acc_y        1119 non-null   float64\n",
      " 2   acc_z        1119 non-null   float64\n",
      " 3   gyr_x        1119 non-null   float64\n",
      " 4   gyr_y        1119 non-null   float64\n",
      " 5   gyr_z        1119 non-null   float64\n",
      " 6   participant  1119 non-null   object \n",
      " 7   label        1119 non-null   object \n",
      " 8   category     1119 non-null   object \n",
      " 9   set          1119 non-null   float64\n",
      "dtypes: float64(7), object(3)\n",
      "memory usage: 96.2+ KB\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------------\n",
    "# Merging datasets in a single dataframe\n",
    "# --------------------------------------------------------------\n",
    "\n",
    "# Selecting just first 3 columns from acc_df\n",
    "data_merged = pd.concat([acc_df.iloc[:, :3], gyr_df], axis=1)\n",
    "\n",
    "# Rename columns\n",
    "data_merged.columns = [\n",
    "    \"acc_x\",\n",
    "    \"acc_y\",\n",
    "    \"acc_z\",\n",
    "    \"gyr_x\",\n",
    "    \"gyr_y\",\n",
    "    \"gyr_z\",\n",
    "    \"participant\",\n",
    "    \"label\",\n",
    "    \"category\",\n",
    "    \"set\",\n",
    "]\n",
    "\n",
    "# Dropping the nan values\n",
    "# There are two sensors measuring at diferent frequencies\n",
    "# The change that the sensors measurement exact at same time is small\n",
    "data_merged\n",
    "data_merged.dropna(inplace=False).info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 8180 entries, 2019-01-11 15:08:05.200000 to 2019-01-19 17:35:13.600000\n",
      "Data columns (total 10 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   acc_x        8180 non-null   float64\n",
      " 1   acc_y        8180 non-null   float64\n",
      " 2   acc_z        8180 non-null   float64\n",
      " 3   gyr_x        8180 non-null   float64\n",
      " 4   gyr_y        8180 non-null   float64\n",
      " 5   gyr_z        8180 non-null   float64\n",
      " 6   participant  8180 non-null   object \n",
      " 7   label        8180 non-null   object \n",
      " 8   category     8180 non-null   object \n",
      " 9   set          8180 non-null   float64\n",
      "dtypes: float64(7), object(3)\n",
      "memory usage: 703.0+ KB\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------------\n",
    "# Resample data (frequency conversion)\n",
    "# --------------------------------------------------------------\n",
    "\n",
    "# Accelerometer:    12.500HZ\n",
    "# Gyroscope:        25.000Hz\n",
    "\n",
    "aggregation = {\n",
    "    \"acc_x\": \"mean\",\n",
    "    \"acc_y\": \"mean\",\n",
    "    \"acc_z\": \"mean\",\n",
    "    \"gyr_x\": \"mean\",\n",
    "    \"gyr_y\": \"mean\",\n",
    "    \"gyr_z\": \"mean\",\n",
    "    \"participant\": \"last\",\n",
    "    \"label\": \"last\",\n",
    "    \"category\": \"last\",\n",
    "    \"set\": \"last\",\n",
    "}\n",
    "\n",
    "# Resample the DataFrame\n",
    "data_merged[:1000].resample(rule=\"200ms\").agg(aggregation, errors=\"ignore\")\n",
    "\n",
    "# Generates a list called df_by_days, where each element represents a DataFrame\n",
    "# for a specific day. The groupby() operation allows you to group the original DataFrame\n",
    "# data_merged by day, and then the list comprehension extracts and collects the\n",
    "# DataFrames into the df_by_days list\n",
    "df_by_days = [df_by_day for day, df_by_day in data_merged.groupby(pd.Grouper(freq=\"D\"))]\n",
    "\n",
    "# Takes the list of DataFrames df_by_days, performs resampling and aggregation on each\n",
    "# DataFrame, drops missing values, and then concatenates the resampled and aggregated\n",
    "# DataFrames into a single DataFrame called data_resampled.\n",
    "data_resampled = pd.concat(\n",
    "    [\n",
    "        df.resample(rule=\"200ms\").agg(aggregation, errors=\"ignore\").dropna()\n",
    "        for df in df_by_days\n",
    "    ]\n",
    ")\n",
    "\n",
    "data_resampled.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------------------------------------------------------\n",
    "# Turn into a single function (resample_and_aggregate_data)\n",
    "# --------------------------------------------------------------\n",
    "\n",
    "def resample_and_aggregate_data(\n",
    "    acc: pd.DataFrame,\n",
    "    gyr: pd.DataFrame,\n",
    "    columns_name: list,\n",
    "    rule: str,\n",
    "    aggregation: dict,\n",
    ") -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Resample and aggregate accelerometer and gyroscope data based on a specified rule and \n",
    "    aggregation dictionary.\n",
    "\n",
    "    Args:\n",
    "        acc (pd.DataFrame): The accelerometer data.\n",
    "        gyr (pd.DataFrame): The gyroscope data.\n",
    "        columns_name (list): The column names for the merged DataFrame.\n",
    "        rule (str): The resampling rule.\n",
    "        aggregation (dict): The aggregation dictionary specifying the columns and \n",
    "        aggregation methods.\n",
    "\n",
    "    Returns:\n",
    "        pd.DataFrame: The resampled and aggregated DataFrame.\n",
    "    \"\"\"\n",
    "    # Selecting just the first 3 columns from acc_df and combining with gyr_df\n",
    "    data_merged = pd.concat([acc.iloc[:, :3], gyr], axis=1)\n",
    "\n",
    "    # Rename columns\n",
    "    data_merged.columns = columns_name\n",
    "\n",
    "    # Group by day\n",
    "    df_by_days = [\n",
    "        df_by_day for day, df_by_day in data_merged.groupby(pd.Grouper(freq=\"D\"))\n",
    "    ]\n",
    "\n",
    "    # Resample and concate each day's DataFrame\n",
    "    data_resampled = pd.concat(\n",
    "        [\n",
    "            df.resample(rule=rule).agg(aggregation, errors=\"ignore\").dropna()\n",
    "            for df in df_by_days\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    data_resampled[\"set\"] = data_resampled[\"set\"].astype(\"int\")\n",
    "\n",
    "    return data_resampled\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 8180 entries, 2019-01-11 15:08:05.200000 to 2019-01-19 17:35:13.600000\n",
      "Data columns (total 10 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   acc_x        8180 non-null   float64\n",
      " 1   acc_y        8180 non-null   float64\n",
      " 2   acc_z        8180 non-null   float64\n",
      " 3   gyr_x        8180 non-null   float64\n",
      " 4   gyr_y        8180 non-null   float64\n",
      " 5   gyr_z        8180 non-null   float64\n",
      " 6   participant  8180 non-null   object \n",
      " 7   label        8180 non-null   object \n",
      " 8   category     8180 non-null   object \n",
      " 9   set          8180 non-null   int32  \n",
      "dtypes: float64(6), int32(1), object(3)\n",
      "memory usage: 671.0+ KB\n"
     ]
    }
   ],
   "source": [
    "columns_name = [\n",
    "    \"acc_x\",\n",
    "    \"acc_y\",\n",
    "    \"acc_z\",\n",
    "    \"gyr_x\",\n",
    "    \"gyr_y\",\n",
    "    \"gyr_z\",\n",
    "    \"participant\",\n",
    "    \"label\",\n",
    "    \"category\",\n",
    "    \"set\",\n",
    "]\n",
    "\n",
    "aggregation = {\n",
    "    \"acc_x\": \"mean\",\n",
    "    \"acc_y\": \"mean\",\n",
    "    \"acc_z\": \"mean\",\n",
    "    \"gyr_x\": \"mean\",\n",
    "    \"gyr_y\": \"mean\",\n",
    "    \"gyr_z\": \"mean\",\n",
    "    \"participant\": \"last\",\n",
    "    \"label\": \"last\",\n",
    "    \"category\": \"last\",\n",
    "    \"set\": \"last\",\n",
    "}\n",
    "\n",
    "rule = \"200ms\"\n",
    "\n",
    "data_resampled = resample_and_aggregate_data(\n",
    "    acc_df, gyr_df, columns_name, rule, aggregation\n",
    ")\n",
    "\n",
    "data_resampled.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Códigos orignais:** `../src/data/make_dataset.py`\n",
    "\n",
    "**Arquivo salvo em:** `../data/interim/01_data_resampled.pkl\"`\n",
    "\n",
    "**Próxima etapa:** `02_visualize.ipynb`"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.12 ('fitness-tracker')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4646813149621b46ca6d4f33c3fc66958fdef2358074d2ca1d1c681bb450ff99"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}