finish

Author: vangj

.gitignore

@@ -1,4 +1,5 @@
 .ipynb_checkpoints/
 .DS_Store
 mimic-iv-1.0
-mimic*
+mimic*
+data.db

csv2sqlite-streaming.ipynb

@@ -33,21 +33,22 @@
    "id": "5287cd41-7680-4ffc-be4e-cbf1098b7a86",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2022-04-01T22:57:50.828795Z",
-     "iopub.status.busy": "2022-04-01T22:57:50.828795Z",
-     "iopub.status.idle": "2022-04-01T22:57:50.843795Z",
-     "shell.execute_reply": "2022-04-01T22:57:50.843795Z",
-     "shell.execute_reply.started": "2022-04-01T22:57:50.828795Z"
+     "iopub.execute_input": "2022-04-02T21:23:32.756771Z",
+     "iopub.status.busy": "2022-04-02T21:23:32.756771Z",
+     "iopub.status.idle": "2022-04-02T21:23:33.178843Z",
+     "shell.execute_reply": "2022-04-02T21:23:33.178843Z",
+     "shell.execute_reply.started": "2022-04-02T21:23:32.756771Z"
     },
     "tags": []
    },
    "outputs": [
     {
-     "ename": "SyntaxError",
-     "evalue": "invalid syntax (<ipython-input-1-391b4b0b0e5d>, line 82)",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-1-391b4b0b0e5d>\"\u001b[1;36m, line \u001b[1;32m82\u001b[0m\n\u001b[1;33m    to_sql(file_path.stem, conn, if_exists='append', index=False)\u001b[0m\n\u001b[1;37m    ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "processing data\\grade.csv\n",
+      "processing data\\person.csv\n",
+      "done\n"
      ]
     }
    ],
@@ -99,7 +100,7 @@
     "    if os.path.exists(file_path):\n",
     "        os.remove(file_path)\n",
     "\n",
-    "def create_db(input_dir, output_db, flush_size=1_000_000):\n",
+    "def create_db(input_dir, output_db, flush_size=1_000_000, encoding='utf-8'):\n",
     "    delete_file_if_exists(output_db)\n",
     "    pathlib.Path(output_db).touch()\n",
     "    \n",
@@ -115,7 +116,7 @@
     "            \n",
     "            cur.execute(sql)\n",
     "            \n",
-    "            with open(file_path, 'r') as fp:\n",
+    "            with open(file_path, 'r', encoding=encoding) as fp:\n",
     "                lines = []\n",
     "                lines.append(headers)\n",
     "                \n",
@@ -136,7 +137,7 @@
     "                    .to_sql(file_path.stem, conn, if_exists='append', index=False)\n",
     "    \n",
     "                \n",
-    "create_db('./mimic-iv-1.0', 'mimic.db')\n",
+    "create_db('./data', 'data.db')\n",
     "print('done')"
    ]
   },
@@ -150,22 +151,38 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "55291a90-d156-4ae5-8e3c-ea3e516c1019",
    "metadata": {
     "execution": {
-     "iopub.status.busy": "2022-04-01T22:57:50.844796Z",
-     "iopub.status.idle": "2022-04-01T22:57:50.844796Z",
-     "shell.execute_reply": "2022-04-01T22:57:50.844796Z"
+     "iopub.execute_input": "2022-04-02T21:23:33.179846Z",
+     "iopub.status.busy": "2022-04-02T21:23:33.179846Z",
+     "iopub.status.idle": "2022-04-02T21:23:33.193843Z",
+     "shell.execute_reply": "2022-04-02T21:23:33.193843Z",
+     "shell.execute_reply.started": "2022-04-02T21:23:33.179846Z"
     },
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(1, 'john', 'smith', 32, 155.5, 1, 1, 1, 99.9)\n",
+      "(1, 'john', 'smith', 32, 155.5, 1, 2, 1, 88.8)\n",
+      "(1, 'john', 'smith', 32, 155.5, 1, 3, 1, 92.2)\n",
+      "(2, 'jane', 'clinton', 28, 133.3, 0, 4, 2, 88.9)\n",
+      "(2, 'jane', 'clinton', 28, 133.3, 0, 5, 2, 92.2)\n",
+      "(2, 'jane', 'clinton', 28, 133.3, 0, 6, 2, 84.4)\n"
+     ]
+    }
+   ],
    "source": [
-    "with sqlite3.connect('mimic.db') as conn:\n",
+    "with sqlite3.connect('data.db') as conn:\n",
     "    sql = '''\n",
-    "    SELECT count(*) as total\n",
-    "    FROM admissions\n",
+    "    SELECT *\n",
+    "    FROM person p\n",
+    "        JOIN grade g ON p.id = g.person_id\n",
     "    '''\n",
     "    \n",
     "    cur = conn.cursor()\n",