set low memory to false; increase flush size

Author: vangj

csv2sqlite-streaming.ipynb

@@ -33,8 +33,8 @@
    "id": "5287cd41-7680-4ffc-be4e-cbf1098b7a86",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2022-04-01T22:30:15.776608Z",
-     "iopub.status.busy": "2022-04-01T22:30:15.776608Z"
+     "iopub.execute_input": "2022-04-01T22:54:59.484432Z",
+     "iopub.status.busy": "2022-04-01T22:54:59.483441Z"
     },
     "tags": []
    },
@@ -46,6 +46,14 @@
       "processing mimic-iv-1.0\\admissions.csv\n",
       "processing mimic-iv-1.0\\chartevents.csv\n"
      ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\jeev\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3357: DtypeWarning: Columns (6,8) have mixed types.Specify dtype option on import or set low_memory=False.\n",
+      "  if (await self.run_code(code, result,  async_=asy)):\n"
+     ]
     }
    ],
    "source": [
@@ -96,7 +104,7 @@
     "    if os.path.exists(file_path):\n",
     "        os.remove(file_path)\n",
     "\n",
-    "def create_db(input_dir, output_db, flush_size=1_000):\n",
+    "def create_db(input_dir, output_db, flush_size=1_000_000):\n",
     "    delete_file_if_exists(output_db)\n",
     "    pathlib.Path(output_db).touch()\n",
     "    \n",
@@ -123,14 +131,14 @@
     "                    lines.append(line)\n",
     "                    \n",
     "                    if len(lines) == flush_size:\n",
-    "                        df = pd.read_csv(StringIO(''.join(lines)))\n",
-    "                        df.to_sql(file_path.stem, conn, if_exists='append', index=False)\n",
+    "                        pd.read_csv(StringIO(''.join(lines)), low_memory=False) \\\n",
+    "                            .to_sql(file_path.stem, conn, if_exists='append', index=False)\n",
     "                        \n",
     "                        lines = []\n",
     "                        lines.append(headers)\n",
     "                \n",
-    "                df = pd.read_csv(StringIO(''.join(lines)))\n",
-    "                df.to_sql(file_path.stem, conn, if_exists='append', index=False)\n",
+    "                pd.read_csv(StringIO(''.join(lines)), low_memory=False) \\\n",
+    "                    to_sql(file_path.stem, conn, if_exists='append', index=False)\n",
     "    \n",
     "                \n",
     "create_db('./mimic-iv-1.0', 'mimic.db')\n",