000_etl_survey_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun Feb 14 09:58:12 PST 2021\r\n"
     ]
    }
   ],
   "source": [
    "import numpy as np, matplotlib.pyplot as plt, pandas as pd\n",
    "pd.set_option('display.max_rows', 8)\n",
    "!date\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract, transform, and \"load\" data from Facebook Data for Good USA survey\n",
    "\n",
    "This code requires access to the IHME cluster to run; we have included it for completeness."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_etl(df):\n",
    "    t = pd.DataFrame()\n",
    "\n",
    "    t['age_group'] = df.D2\n",
    "    t['male'] = (df.D1 == 1).astype(int)\n",
    "    \n",
    "#     t['A1_hh_symptoms'] = df.A1\n",
    "    t['B2_individual_symptoms'] = df.B2\n",
    "    t['B2c_new_symptoms'] = df.B2c\n",
    "    t['B2b_duration_symptoms'] = df.B2b\n",
    "    \n",
    "    \n",
    "    t['test_last_14_days'] = (df.B10 == 1).astype(int)\n",
    "    t['test_positive'] = (df.B10a == 1).astype(int)\n",
    "    t['test_negative'] = (df.B10a == 2).astype(int)\n",
    "    t['test_unknown'] = (df.B10a == 3).astype(int)\n",
    "    t['test_reason'] = df.B10b\n",
    "    t['test_required'] = df.B10b.astype(str).str.contains('4')\n",
    "    \n",
    "    t['hcw'] = (df.Q64.isin([4,5]))\n",
    "    t = pd.concat([t, df.filter(regex='^[DQ]')], axis=1)\n",
    "\n",
    "    t['work_outside_home'] = (df.D10 == 1).astype(int)\n",
    "\n",
    "    t['activities_past24hrs'] = df.C13\n",
    "    t['masked_activities_past24hrs'] = df.C13a\n",
    "    t['mask_past5days'] = df.C14\n",
    "    \n",
    "    # race and ethnicity, which is in instrument, but not in data file\n",
    "    #t['D6'] = df.D6\n",
    "    #t['D7'] = df.D7\n",
    "    \n",
    "    t['weight'] = df.weight\n",
    "    return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ls -halt /ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_10_recordedby_2021_02_10.csv.gz\n",
      "CPU times: user 20 ms, sys: 2.88 ms, total: 22.9 ms\n",
      "Wall time: 22.9 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_group</th>\n",
       "      <th>male</th>\n",
       "      <th>B2_individual_symptoms</th>\n",
       "      <th>B2c_new_symptoms</th>\n",
       "      <th>B2b_duration_symptoms</th>\n",
       "      <th>test_last_14_days</th>\n",
       "      <th>test_positive</th>\n",
       "      <th>test_negative</th>\n",
       "      <th>test_unknown</th>\n",
       "      <th>test_reason</th>\n",
       "      <th>...</th>\n",
       "      <th>Q78</th>\n",
       "      <th>Q79</th>\n",
       "      <th>Q80</th>\n",
       "      <th>D10</th>\n",
       "      <th>D11</th>\n",
       "      <th>work_outside_home</th>\n",
       "      <th>activities_past24hrs</th>\n",
       "      <th>masked_activities_past24hrs</th>\n",
       "      <th>mask_past5days</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2,3,6,7,16,18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1653.229764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2,4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2540.198738</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   age_group  male B2_individual_symptoms  B2c_new_symptoms  \\\n",
       "0          6     0          2,3,6,7,16,18               NaN   \n",
       "1          4     0                      5               NaN   \n",
       "\n",
       "   B2b_duration_symptoms  test_last_14_days  test_positive  test_negative  \\\n",
       "0                    NaN                  0              0              0   \n",
       "1                    NaN                  0              0              0   \n",
       "\n",
       "   test_unknown  test_reason  ...  Q78  Q79  Q80  D10  D11  work_outside_home  \\\n",
       "0             0          NaN  ...  NaN  NaN  NaN  NaN  NaN                  0   \n",
       "1             0          NaN  ...  NaN  NaN  NaN  NaN  NaN                  0   \n",
       "\n",
       "   activities_past24hrs  masked_activities_past24hrs  mask_past5days  \\\n",
       "0                     8                          NaN               1   \n",
       "1                   2,4                          4.0               1   \n",
       "\n",
       "        weight  \n",
       "0  1653.229764  \n",
       "1  2540.198738  \n",
       "\n",
       "[2 rows x 47 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test ETL function on the most recent available data file\n",
    "\n",
    "import glob\n",
    "fnames = sorted(glob.glob('/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/'\n",
    "                   'AUTO_EXTRACTED/*/cvid_responses*.csv.gz'), reverse=True)\n",
    "fname = fnames[0]\n",
    "print(fname)\n",
    "df = pd.read_csv(fname, low_memory=False)\n",
    "%time my_etl(df.iloc[:1_000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse date out of filenames to be able to record when survey was offered\n",
    "\n",
    "def my_date(s):\n",
    "    s1 = s.split('cvid_responses_')[1]\n",
    "    s2 = s1.split('_recordedby_')[0]\n",
    "    s3 = s2.replace('_', '-')\n",
    "    return s3\n",
    "# results_us['date'] = results_us.index.map(my_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_10_recordedby_2021_02_10.csv.gz\n",
      "CPU times: user 11 ms, sys: 0 ns, total: 11 ms\n",
      "Wall time: 11 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_09_recordedby_2021_02_10.csv.gz\n",
      "CPU times: user 46.4 ms, sys: 1.16 ms, total: 47.6 ms\n",
      "Wall time: 47.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_08_recordedby_2021_02_10.csv.gz\n",
      "CPU times: user 46.9 ms, sys: 14.8 ms, total: 61.7 ms\n",
      "Wall time: 61.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_07_recordedby_2021_02_10.csv.gz\n",
      "CPU times: user 53.9 ms, sys: 683 µs, total: 54.6 ms\n",
      "Wall time: 54.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_06_recordedby_2021_02_10.csv.gz\n",
      "CPU times: user 48.9 ms, sys: 693 µs, total: 49.5 ms\n",
      "Wall time: 49.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_05_recordedby_2021_02_09.csv.gz\n",
      "CPU times: user 48.6 ms, sys: 0 ns, total: 48.6 ms\n",
      "Wall time: 48.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_04_recordedby_2021_02_08.csv.gz\n",
      "CPU times: user 62 ms, sys: 4.81 ms, total: 66.8 ms\n",
      "Wall time: 66.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_03_recordedby_2021_02_07.csv.gz\n",
      "CPU times: user 57.5 ms, sys: 0 ns, total: 57.5 ms\n",
      "Wall time: 57.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_02_recordedby_2021_02_06.csv.gz\n",
      "CPU times: user 43.6 ms, sys: 687 µs, total: 44.3 ms\n",
      "Wall time: 44.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_02_01_recordedby_2021_02_05.csv.gz\n",
      "CPU times: user 43.9 ms, sys: 737 µs, total: 44.6 ms\n",
      "Wall time: 44.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_31_recordedby_2021_02_04.csv.gz\n",
      "CPU times: user 53.6 ms, sys: 1.68 ms, total: 55.3 ms\n",
      "Wall time: 55.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_30_recordedby_2021_02_03.csv.gz\n",
      "CPU times: user 54.8 ms, sys: 6.79 ms, total: 61.6 ms\n",
      "Wall time: 61.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_29_recordedby_2021_02_02.csv.gz\n",
      "CPU times: user 51.6 ms, sys: 2.8 ms, total: 54.4 ms\n",
      "Wall time: 54.5 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_28_recordedby_2021_02_01.csv.gz\n",
      "CPU times: user 56.1 ms, sys: 1.99 ms, total: 58.1 ms\n",
      "Wall time: 58.2 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_27_recordedby_2021_01_31.csv.gz\n",
      "CPU times: user 55.9 ms, sys: 4.86 ms, total: 60.7 ms\n",
      "Wall time: 60.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_26_recordedby_2021_01_30.csv.gz\n",
      "CPU times: user 65.8 ms, sys: 899 µs, total: 66.7 ms\n",
      "Wall time: 66.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_25_recordedby_2021_01_29.csv.gz\n",
      "CPU times: user 58.6 ms, sys: 1.76 ms, total: 60.4 ms\n",
      "Wall time: 60.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_24_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 55.2 ms, sys: 3.7 ms, total: 58.9 ms\n",
      "Wall time: 59 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_23_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 55.8 ms, sys: 2.87 ms, total: 58.7 ms\n",
      "Wall time: 58.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_22_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 54.1 ms, sys: 22 µs, total: 54.2 ms\n",
      "Wall time: 54.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_21_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 58.7 ms, sys: 9.83 ms, total: 68.5 ms\n",
      "Wall time: 68.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_20_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 57.2 ms, sys: 1.03 ms, total: 58.2 ms\n",
      "Wall time: 58.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_19_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 58.7 ms, sys: 873 µs, total: 59.6 ms\n",
      "Wall time: 59.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_18_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 56.1 ms, sys: 8 µs, total: 56.1 ms\n",
      "Wall time: 56.2 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_17_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 62 ms, sys: 960 µs, total: 62.9 ms\n",
      "Wall time: 63.1 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_16_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 60.1 ms, sys: 0 ns, total: 60.1 ms\n",
      "Wall time: 60.2 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_15_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 53.7 ms, sys: 867 µs, total: 54.6 ms\n",
      "Wall time: 54.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_14_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 56.7 ms, sys: 0 ns, total: 56.7 ms\n",
      "Wall time: 56.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_13_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 60.6 ms, sys: 26 µs, total: 60.7 ms\n",
      "Wall time: 60.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_12_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 60.8 ms, sys: 895 µs, total: 61.7 ms\n",
      "Wall time: 61.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_11_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 53.6 ms, sys: 0 ns, total: 53.6 ms\n",
      "Wall time: 53.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_10_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 55.8 ms, sys: 12.9 ms, total: 68.7 ms\n",
      "Wall time: 68.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_09_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 57.5 ms, sys: 0 ns, total: 57.5 ms\n",
      "Wall time: 57.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_08_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 56.8 ms, sys: 2.91 ms, total: 59.7 ms\n",
      "Wall time: 59.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_07_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 52 ms, sys: 0 ns, total: 52 ms\n",
      "Wall time: 52.1 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_06_recordedby_2021_01_28.csv.gz\n",
      "CPU times: user 55.6 ms, sys: 0 ns, total: 55.6 ms\n",
      "Wall time: 55.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_05_recordedby_2021_01_09.csv.gz\n",
      "CPU times: user 74.6 ms, sys: 0 ns, total: 74.6 ms\n",
      "Wall time: 74.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_04_recordedby_2021_01_08.csv.gz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 83.3 ms, sys: 84 µs, total: 83.4 ms\n",
      "Wall time: 83.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_03_recordedby_2021_01_07.csv.gz\n",
      "CPU times: user 79.1 ms, sys: 19 µs, total: 79.1 ms\n",
      "Wall time: 79.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_02_recordedby_2021_01_06.csv.gz\n",
      "CPU times: user 75.2 ms, sys: 45 µs, total: 75.2 ms\n",
      "Wall time: 75.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2021_01_01_recordedby_2021_01_05.csv.gz\n",
      "CPU times: user 68.7 ms, sys: 12.9 ms, total: 81.6 ms\n",
      "Wall time: 81.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_31_recordedby_2021_01_04.csv.gz\n",
      "CPU times: user 69.7 ms, sys: 0 ns, total: 69.7 ms\n",
      "Wall time: 69.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_30_recordedby_2021_01_03.csv.gz\n",
      "CPU times: user 71.1 ms, sys: 14 ms, total: 85.1 ms\n",
      "Wall time: 85.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_29_recordedby_2021_01_02.csv.gz\n",
      "CPU times: user 58.1 ms, sys: 2.03 ms, total: 60.1 ms\n",
      "Wall time: 60.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_28_recordedby_2021_01_01.csv.gz\n",
      "CPU times: user 59.7 ms, sys: 2.99 ms, total: 62.7 ms\n",
      "Wall time: 62.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_27_recordedby_2020_12_31.csv.gz\n",
      "CPU times: user 57.9 ms, sys: 1.98 ms, total: 59.9 ms\n",
      "Wall time: 60 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_26_recordedby_2020_12_30.csv.gz\n",
      "CPU times: user 56.4 ms, sys: 0 ns, total: 56.4 ms\n",
      "Wall time: 56.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_25_recordedby_2020_12_29.csv.gz\n",
      "CPU times: user 51.8 ms, sys: 2.96 ms, total: 54.8 ms\n",
      "Wall time: 54.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_24_recordedby_2020_12_28.csv.gz\n",
      "CPU times: user 54.3 ms, sys: 97 µs, total: 54.4 ms\n",
      "Wall time: 54.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_23_recordedby_2020_12_27.csv.gz\n",
      "CPU times: user 58.5 ms, sys: 7.86 ms, total: 66.4 ms\n",
      "Wall time: 66.5 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_22_recordedby_2020_12_26.csv.gz\n",
      "CPU times: user 55.1 ms, sys: 3.76 ms, total: 58.8 ms\n",
      "Wall time: 59 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_21_recordedby_2020_12_25.csv.gz\n",
      "CPU times: user 52.7 ms, sys: 3.91 ms, total: 56.6 ms\n",
      "Wall time: 56.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_20_recordedby_2020_12_24.csv.gz\n",
      "CPU times: user 52.8 ms, sys: 0 ns, total: 52.8 ms\n",
      "Wall time: 53 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_19_recordedby_2020_12_23.csv.gz\n",
      "CPU times: user 52.5 ms, sys: 0 ns, total: 52.5 ms\n",
      "Wall time: 52.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_18_recordedby_2020_12_22.csv.gz\n",
      "CPU times: user 55.7 ms, sys: 0 ns, total: 55.7 ms\n",
      "Wall time: 55.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_17_recordedby_2020_12_21.csv.gz\n",
      "CPU times: user 53.8 ms, sys: 13 ms, total: 66.8 ms\n",
      "Wall time: 67 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_16_recordedby_2020_12_21.csv.gz\n",
      "CPU times: user 53.8 ms, sys: 966 µs, total: 54.8 ms\n",
      "Wall time: 55 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_15_recordedby_2020_12_19.csv.gz\n",
      "CPU times: user 65.1 ms, sys: 36 µs, total: 65.1 ms\n",
      "Wall time: 65.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_14_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 65.6 ms, sys: 0 ns, total: 65.6 ms\n",
      "Wall time: 65.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_13_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 51 ms, sys: 1.51 ms, total: 52.6 ms\n",
      "Wall time: 52.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_12_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 49.8 ms, sys: 0 ns, total: 49.8 ms\n",
      "Wall time: 50 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_11_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 49.1 ms, sys: 0 ns, total: 49.1 ms\n",
      "Wall time: 49.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_10_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 52.3 ms, sys: 95 µs, total: 52.4 ms\n",
      "Wall time: 52.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_09_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 50.1 ms, sys: 37 µs, total: 50.1 ms\n",
      "Wall time: 50.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_08_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 54.8 ms, sys: 952 µs, total: 55.7 ms\n",
      "Wall time: 55.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_07_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 95.7 ms, sys: 0 ns, total: 95.7 ms\n",
      "Wall time: 95.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_06_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 66.4 ms, sys: 0 ns, total: 66.4 ms\n",
      "Wall time: 66.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_05_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 52.6 ms, sys: 0 ns, total: 52.6 ms\n",
      "Wall time: 52.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_04_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 52.3 ms, sys: 92 µs, total: 52.4 ms\n",
      "Wall time: 52.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_03_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 54.4 ms, sys: 22.1 ms, total: 76.5 ms\n",
      "Wall time: 76.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_02_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 60 ms, sys: 0 ns, total: 60 ms\n",
      "Wall time: 60.1 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_12_01_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 71.2 ms, sys: 0 ns, total: 71.2 ms\n",
      "Wall time: 71.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_30_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 54.5 ms, sys: 3.87 ms, total: 58.3 ms\n",
      "Wall time: 58.5 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_29_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 55.4 ms, sys: 13.1 ms, total: 68.4 ms\n",
      "Wall time: 68.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_28_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 51.6 ms, sys: 902 µs, total: 52.5 ms\n",
      "Wall time: 52.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_27_recordedby_2020_12_18.csv.gz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 47.8 ms, sys: 3.96 ms, total: 51.7 ms\n",
      "Wall time: 51.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_26_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 47.4 ms, sys: 107 µs, total: 47.5 ms\n",
      "Wall time: 47.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_25_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 49.5 ms, sys: 4.12 ms, total: 53.6 ms\n",
      "Wall time: 53.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_24_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 51.7 ms, sys: 935 µs, total: 52.6 ms\n",
      "Wall time: 52.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_23_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 48.4 ms, sys: 0 ns, total: 48.4 ms\n",
      "Wall time: 48.5 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_22_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 54 ms, sys: 0 ns, total: 54 ms\n",
      "Wall time: 54.1 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_21_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 50.9 ms, sys: 0 ns, total: 50.9 ms\n",
      "Wall time: 51.1 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_20_recordedby_2020_12_18.csv.gz\n",
      "CPU times: user 48.7 ms, sys: 15.9 ms, total: 64.7 ms\n",
      "Wall time: 64.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_19_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 47.2 ms, sys: 56 µs, total: 47.2 ms\n",
      "Wall time: 47.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_18_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 50.7 ms, sys: 0 ns, total: 50.7 ms\n",
      "Wall time: 50.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_17_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 51.3 ms, sys: 1.84 ms, total: 53.1 ms\n",
      "Wall time: 53.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_16_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 47.1 ms, sys: 24 µs, total: 47.1 ms\n",
      "Wall time: 47.2 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_15_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 45.8 ms, sys: 0 ns, total: 45.8 ms\n",
      "Wall time: 45.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_14_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 41.1 ms, sys: 1.04 ms, total: 42.2 ms\n",
      "Wall time: 42.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_13_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 41.7 ms, sys: 0 ns, total: 41.7 ms\n",
      "Wall time: 41.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_12_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 42.3 ms, sys: 78 µs, total: 42.4 ms\n",
      "Wall time: 42.5 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_11_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 44.5 ms, sys: 125 µs, total: 44.6 ms\n",
      "Wall time: 44.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_10_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 47.5 ms, sys: 122 µs, total: 47.6 ms\n",
      "Wall time: 47.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_09_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 41.2 ms, sys: 49 µs, total: 41.2 ms\n",
      "Wall time: 41.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_08_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 37.1 ms, sys: 1.8 ms, total: 38.9 ms\n",
      "Wall time: 37.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_07_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 25.2 ms, sys: 47 µs, total: 25.2 ms\n",
      "Wall time: 25.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_06_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 38.4 ms, sys: 892 µs, total: 39.3 ms\n",
      "Wall time: 39.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_05_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 28.5 ms, sys: 1.13 ms, total: 29.7 ms\n",
      "Wall time: 29 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_04_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 25.2 ms, sys: 22 µs, total: 25.2 ms\n",
      "Wall time: 25.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_03_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 42.1 ms, sys: 2.83 ms, total: 45 ms\n",
      "Wall time: 45.1 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_02_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 44.9 ms, sys: 992 µs, total: 45.9 ms\n",
      "Wall time: 46 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_11_01_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 45 ms, sys: 8 µs, total: 45 ms\n",
      "Wall time: 45.1 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_31_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 43.1 ms, sys: 2.02 ms, total: 45.1 ms\n",
      "Wall time: 45.2 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_30_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 33.2 ms, sys: 66 µs, total: 33.3 ms\n",
      "Wall time: 32.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_29_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 32.4 ms, sys: 100 µs, total: 32.5 ms\n",
      "Wall time: 32.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_28_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 36.8 ms, sys: 0 ns, total: 36.8 ms\n",
      "Wall time: 36.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_27_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 37.3 ms, sys: 0 ns, total: 37.3 ms\n",
      "Wall time: 37.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_26_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 42.6 ms, sys: 138 µs, total: 42.7 ms\n",
      "Wall time: 42.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_25_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 45.8 ms, sys: 2.96 ms, total: 48.7 ms\n",
      "Wall time: 48.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_24_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 42.4 ms, sys: 0 ns, total: 42.4 ms\n",
      "Wall time: 42.5 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_23_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 42 ms, sys: 21 µs, total: 42.1 ms\n",
      "Wall time: 42.2 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_22_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 40.6 ms, sys: 0 ns, total: 40.6 ms\n",
      "Wall time: 40.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_21_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 41.8 ms, sys: 0 ns, total: 41.8 ms\n",
      "Wall time: 41.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_20_recordedby_2020_12_09.csv.gz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 46 ms, sys: 3.01 ms, total: 49 ms\n",
      "Wall time: 49.2 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_19_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 46 ms, sys: 1.02 ms, total: 47 ms\n",
      "Wall time: 47.2 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_18_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 45.1 ms, sys: 0 ns, total: 45.1 ms\n",
      "Wall time: 45.2 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_17_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 44.3 ms, sys: 75 µs, total: 44.4 ms\n",
      "Wall time: 44.5 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_16_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 41.7 ms, sys: 0 ns, total: 41.7 ms\n",
      "Wall time: 41.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_15_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 50.2 ms, sys: 3.06 ms, total: 53.3 ms\n",
      "Wall time: 53.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_14_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 48.8 ms, sys: 980 µs, total: 49.8 ms\n",
      "Wall time: 50 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_13_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 53.3 ms, sys: 0 ns, total: 53.3 ms\n",
      "Wall time: 53.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_12_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 45.5 ms, sys: 0 ns, total: 45.5 ms\n",
      "Wall time: 45.7 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_11_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 44.6 ms, sys: 6.84 ms, total: 51.4 ms\n",
      "Wall time: 51.5 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_10_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 48.3 ms, sys: 79 µs, total: 48.4 ms\n",
      "Wall time: 48.5 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_09_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 46.5 ms, sys: 0 ns, total: 46.5 ms\n",
      "Wall time: 46.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_08_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 47.4 ms, sys: 1.88 ms, total: 49.3 ms\n",
      "Wall time: 49.4 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_07_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 54.1 ms, sys: 38 µs, total: 54.1 ms\n",
      "Wall time: 54.3 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_06_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 55.8 ms, sys: 0 ns, total: 55.8 ms\n",
      "Wall time: 55.9 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_05_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 50.7 ms, sys: 0 ns, total: 50.7 ms\n",
      "Wall time: 50.8 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_04_recordedby_2020_12_09.csv.gz\n",
      "CPU times: user 47.5 ms, sys: 0 ns, total: 47.5 ms\n",
      "Wall time: 47.6 ms\n",
      "/ihme/limited_use/LIMITED_USE/PROJECT_FOLDERS/COVID19/SYMPTOM_SURVEY/US/AUTO_EXTRACTED/latest/cvid_responses_2020_10_03_recordedby_2020_12_09.csv.gz\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# loop over all data files from wave 4 of survey\n",
    "# and extract data from most recent file for each day\n",
    "\n",
    "data = {}\n",
    "for fname in fnames:\n",
    "    data_date = my_date(fname)\n",
    "    if not data_date in data.keys():\n",
    "        if data_date <= '2020-09-03':  # earlier waves of survey do not contain key questions for this analysis\n",
    "            continue\n",
    "        print(fname)\n",
    "        df_i = pd.read_csv(fname, low_memory=False)\n",
    "        %time data[data_date] = my_etl(df_i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in data.keys():\n",
    "    data[k]['date'] = k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_all = pd.concat(data.values())\n",
    "df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all.to_csv('data/fb_data_usa.csv.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7001656, 48)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}