2-cow-file-layouts-file-sizing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "44288d4b-1602-44ad-9881-f0e510a30d76",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Apache Hudi Core Conceptions (2) - COW: File Layouts & File Sizing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c53ac40-4d0b-46c7-88ee-a94345a0130f",
   "metadata": {},
   "source": [
    "*Author: [Laurence Geng](https://laurence.blog.csdn.net) @ [https://laurence.blog.csdn.net](https://laurence.blog.csdn.net)*"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d6261e3-fa15-410d-8472-29dd02f86b10",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 1. Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ce9885ac-50fe-4929-9b0d-85ff33f2cbcd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:17:28.659080Z",
     "iopub.status.busy": "2023-03-16T01:17:28.658820Z",
     "iopub.status.idle": "2023-03-16T01:17:32.933656Z",
     "shell.execute_reply": "2023-03-16T01:17:32.932672Z",
     "shell.execute_reply.started": "2023-03-16T01:17:28.659055Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--   1 emr-notebook hdfsadmingroup   61421977 2023-03-16 01:17 /tmp/hudi-spark-bundle.jar\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "# deploy hudi bundle jar\n",
    "hdfs dfs -copyFromLocal -f /usr/lib/hudi/hudi-spark-bundle.jar /tmp/hudi-spark-bundle.jar\n",
    "hdfs dfs -ls /tmp/hudi-spark-bundle.jar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "57b9edaf-90a7-4b7c-b072-bc8c50c2e045",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:17:32.935301Z",
     "iopub.status.busy": "2023-03-16T01:17:32.935059Z",
     "iopub.status.idle": "2023-03-16T01:17:32.966474Z",
     "shell.execute_reply": "2023-03-16T01:17:32.965666Z",
     "shell.execute_reply.started": "2023-03-16T01:17:32.935277Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "Current session configs: <tt>{'conf': {'spark.jars': 'hdfs:///tmp/hudi-spark-bundle.jar', 'spark.serializer': 'org.apache.spark.serializer.KryoSerializer', 'spark.sql.extensions': 'org.apache.spark.sql.hudi.HoodieSparkSessionExtension', 'spark.sql.catalog.spark_catalog': 'org.apache.spark.sql.hudi.catalog.HoodieCatalog'}, 'proxyUser': 'user_bluishglc', 'kind': 'spark'}</tt><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n<tbody><tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>User</th><th>Current session?</th></tr><tr><td>80</td><td>application_1678096020253_0135</td><td>spark</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-10-0-129-209.ec2.internal:20888/proxy/application_1678096020253_0135/\" class=\"emr-proxy-link j-6V65AY8X5VHR application_1678096020253_0135\" emr-resource=\"j-6V65AY8X5VHR\n\" application-id=\"application_1678096020253_0135\">Link</a></td><td><a target=\"_blank\" href=\"http://ip-10-0-132-108.ec2.internal:8042/node/containerlogs/container_1678096020253_0135_01_000001/livy\">Link</a></td><td>None</td><td></td></tr></tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%configure -f\n",
    "{\n",
    "    \"conf\" : {\n",
    "        \"spark.jars\":\"hdfs:///tmp/hudi-spark-bundle.jar\",            \n",
    "        \"spark.serializer\":\"org.apache.spark.serializer.KryoSerializer\",\n",
    "        \"spark.sql.extensions\":\"org.apache.spark.sql.hudi.HoodieSparkSessionExtension\",\n",
    "        \"spark.sql.catalog.spark_catalog\":\"org.apache.spark.sql.hudi.catalog.HoodieCatalog\"\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8facfe6d-f3b3-4000-acf2-b567881ee59e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:17:32.968917Z",
     "iopub.status.busy": "2023-03-16T01:17:32.968563Z",
     "iopub.status.idle": "2023-03-16T01:17:32.973916Z",
     "shell.execute_reply": "2023-03-16T01:17:32.973248Z",
     "shell.execute_reply.started": "2023-03-16T01:17:32.968878Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: S3_BUCKET=apache-hudi-core-conceptions\n"
     ]
    }
   ],
   "source": [
    "%env S3_BUCKET=apache-hudi-core-conceptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "51017d2a-0b23-4c2d-bad8-652a73de9e60",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:17:32.975319Z",
     "iopub.status.busy": "2023-03-16T01:17:32.975099Z",
     "iopub.status.idle": "2023-03-16T01:18:02.539522Z",
     "shell.execute_reply": "2023-03-16T01:18:02.538773Z",
     "shell.execute_reply.started": "2023-03-16T01:17:32.975298Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Spark application\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n<tbody><tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>User</th><th>Current session?</th></tr><tr><td>81</td><td>application_1678096020253_0140</td><td>spark</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-10-0-129-209.ec2.internal:20888/proxy/application_1678096020253_0140/\" class=\"emr-proxy-link j-6V65AY8X5VHR application_1678096020253_0140\" emr-resource=\"j-6V65AY8X5VHR\n\" application-id=\"application_1678096020253_0140\">Link</a></td><td><a target=\"_blank\" href=\"http://ip-10-0-132-108.ec2.internal:8042/node/containerlogs/container_1678096020253_0140_01_000001/livy\">Link</a></td><td>None</td><td>✔</td></tr></tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparkSession available as 'spark'.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0af698a5aa740a987b23abb2cf044ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(HTML(value='Type:'), Button(description='Table', layout=Layout(width='70px'), st…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9d9b7997f814bbfad4700ba4a6199f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "set S3_BUCKET=apache-hudi-core-conceptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a21ec2e4-486d-4952-af27-fcf56be0db14",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:18:02.540823Z",
     "iopub.status.busy": "2023-03-16T01:18:02.540509Z",
     "iopub.status.idle": "2023-03-16T01:18:02.545161Z",
     "shell.execute_reply": "2023-03-16T01:18:02.544503Z",
     "shell.execute_reply.started": "2023-03-16T01:18:02.540799Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: WORKSPACE=/home/emr-notebook/apache-hudi-core-conceptions\n"
     ]
    }
   ],
   "source": [
    "%env WORKSPACE=/home/emr-notebook/apache-hudi-core-conceptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3055ecc0-e6ef-4c8f-99f9-0c5b5a6c1c1d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:18:02.546254Z",
     "iopub.status.busy": "2023-03-16T01:18:02.546040Z",
     "iopub.status.idle": "2023-03-16T01:18:02.677200Z",
     "shell.execute_reply": "2023-03-16T01:18:02.676363Z",
     "shell.execute_reply.started": "2023-03-16T01:18:02.546233Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/emr-notebook/apache-hudi-core-conceptions/hudi-stat.sh\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "# make workspace\n",
    "mkdir -p $WORKSPACE\n",
    "# deploy hudi-stat.sh, a utility shell script to output hudi table status\n",
    "wget https://raw.githubusercontent.com/bluishglc/apache-hudi-core-conceptions/master/hudi-stat.sh -O $WORKSPACE/hudi-stat.sh &>/dev/null\n",
    "chmod a+x $WORKSPACE/hudi-stat.sh\n",
    "ls $WORKSPACE/hudi-stat.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6bb44084-8afe-47fe-9454-b5a496275808",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:18:02.678703Z",
     "iopub.status.busy": "2023-03-16T01:18:02.678448Z",
     "iopub.status.idle": "2023-03-16T01:18:02.684523Z",
     "shell.execute_reply": "2023-03-16T01:18:02.683716Z",
     "shell.execute_reply.started": "2023-03-16T01:18:02.678676Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       "table {float:left}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<style>\n",
    "table {float:left}\n",
    "</style>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00bdc485-3043-484c-8603-0e630184d445",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 2. Test Case 1 - COW File Layouts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2199ea90-bd54-4c74-8b16-98f60f1968b3",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 2.1. Test Plan\n",
    "\n",
    "Step No.|Action|Volume Per Partition |Storage\n",
    ":--------:|:------|:------|:----------\n",
    "1|Insert|96MB|+5 Small Files\n",
    "2|Insert|14.6MB|+1 Big File\n",
    "3|Insert|3.7MB|+1 Small File\n",
    "4|Insert|188.5MB|+1 Big File, +1 Small File"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6efd4afb-6bfe-4f77-9928-ade6054fa20b",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.2. Key Settings\n",
    "\n",
    "KEY|DEFAULT VALUE|SET VALUE\n",
    ":---|:---|:---\n",
    "hoodie.parquet.max.file.size|125829120 ( 120MB )|Default Value\n",
    "hoodie.parquet.small.file.limit|104857600 ( 100MB )|Default Value\n",
    "hoodie.copyonwrite.record.size.estimate|1024|175"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "992230f3-5f68-4a33-954c-3e1e6e120f06",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-25T02:57:45.037101Z",
     "iopub.status.busy": "2023-02-25T02:57:45.036681Z",
     "iopub.status.idle": "2023-02-25T02:58:07.353493Z",
     "shell.execute_reply": "2023-02-25T02:58:07.352596Z",
     "shell.execute_reply.started": "2023-02-25T02:57:45.037060Z"
    },
    "tags": []
   },
   "source": [
    "### 2.3. Set Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "63bf2e47-d172-4f81-949c-2b46c408e99e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:18:02.689254Z",
     "iopub.status.busy": "2023-03-16T01:18:02.688809Z",
     "iopub.status.idle": "2023-03-16T01:18:03.646705Z",
     "shell.execute_reply": "2023-03-16T01:18:03.645493Z",
     "shell.execute_reply.started": "2023-03-16T01:18:02.689222Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "831852403f644cffb395a072ae8b9f3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(HTML(value='Type:'), Button(description='Table', layout=Layout(width='70px'), st…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ffd6524c5cc4472bf438a3ece7ba284",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "set TABLE_NAME=reviews_cow_layouts_sizing_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3cb3dd19-a55e-4ebd-ab6d-cb1c2cdef2b3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:18:03.649418Z",
     "iopub.status.busy": "2023-03-16T01:18:03.648891Z",
     "iopub.status.idle": "2023-03-16T01:18:03.656264Z",
     "shell.execute_reply": "2023-03-16T01:18:03.654645Z",
     "shell.execute_reply.started": "2023-03-16T01:18:03.649176Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: TABLE_NAME=reviews_cow_layouts_sizing_1\n"
     ]
    }
   ],
   "source": [
    "%env TABLE_NAME=reviews_cow_layouts_sizing_1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ba3031f-f92c-486d-a0de-3f141410f2bf",
   "metadata": {},
   "source": [
    "### 2.4. Create Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bdcd022c-8a54-463d-9f3e-fbfb92969328",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:18:03.657911Z",
     "iopub.status.busy": "2023-03-16T01:18:03.657529Z",
     "iopub.status.idle": "2023-03-16T01:18:09.259522Z",
     "shell.execute_reply": "2023-03-16T01:18:09.258578Z",
     "shell.execute_reply.started": "2023-03-16T01:18:03.657875Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%sh\n",
    "aws s3 rm s3://${S3_BUCKET}/${TABLE_NAME} --recursive &>/dev/null\n",
    "rm -rf ${WORKSPACE}/${TABLE_NAME}\n",
    "sleep 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2dd45b7f-cce2-4aec-99ec-7359d6d060e7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:18:09.261292Z",
     "iopub.status.busy": "2023-03-16T01:18:09.260982Z",
     "iopub.status.idle": "2023-03-16T01:18:10.591380Z",
     "shell.execute_reply": "2023-03-16T01:18:10.590651Z",
     "shell.execute_reply.started": "2023-03-16T01:18:09.261261Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2f0d6aa766145298659c7f7c06eafcd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad188cd29f99467c93095faf0700a451",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f740f107-0ab8-46b7-bdd3-206571a40c09",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:18:10.592625Z",
     "iopub.status.busy": "2023-03-16T01:18:10.592391Z",
     "iopub.status.idle": "2023-03-16T01:18:16.004784Z",
     "shell.execute_reply": "2023-03-16T01:18:16.003845Z",
     "shell.execute_reply.started": "2023-03-16T01:18:10.592603Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02851f59a12545e38164ebf0a5a906bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a368a23727974938a0edc62030f198a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "create table if not exists ${TABLE_NAME}(\n",
    "    review_id string, \n",
    "    star_rating int, \n",
    "    review_body string, \n",
    "    review_date date, \n",
    "    year long,\n",
    "    timestamp long,\n",
    "    parity int\n",
    ")\n",
    "using hudi\n",
    "location 's3://${S3_BUCKET}/${TABLE_NAME}'\n",
    "partitioned by (parity)\n",
    "options ( \n",
    "    type = 'cow',  \n",
    "    primaryKey = 'review_id', \n",
    "    preCombineField = 'timestamp',\n",
    "    hoodie.copyonwrite.record.size.estimate = '175'\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26d91fec-b0b0-4bc0-b295-71798ac42766",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.5. Insert 96MB / +1 Small File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "de1fac92-fdc7-403a-a491-e11510324f64",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:18:16.006744Z",
     "iopub.status.busy": "2023-03-16T01:18:16.006087Z",
     "iopub.status.idle": "2023-03-16T01:19:17.721765Z",
     "shell.execute_reply": "2023-03-16T01:19:17.720917Z",
     "shell.execute_reply.started": "2023-03-16T01:18:16.006707Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b19c14bd6ba41b488bb5ee402d29056",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e2ce01d1b8642668708773f9615624f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "insert into \n",
    "    ${TABLE_NAME}\n",
    "select \n",
    "    review_id, \n",
    "    star_rating, \n",
    "    review_body, \n",
    "    review_date, \n",
    "    year,\n",
    "    unix_timestamp(current_timestamp()) as timestamp,\n",
    "    mod(crc32(review_id), 2) as parity\n",
    "from\n",
    "    reviews\n",
    "where\n",
    "    year = 2003;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9adfa455-d9ea-4527-8d0e-942dc97a114e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:19:17.723394Z",
     "iopub.status.busy": "2023-03-16T01:19:17.723043Z",
     "iopub.status.idle": "2023-03-16T01:19:33.099294Z",
     "shell.execute_reply": "2023-03-16T01:19:33.098286Z",
     "shell.execute_reply.started": "2023-03-16T01:19:17.723358Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │        │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230316011817174 │ commit │ COMPLETED │ 03-16 01:18 │ 03-16 01:18 │ 03-16 01:19 ║\n",
      "╚═════╧═══════════════════╧════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230316011817174 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_cow_layouts_sizing_1\n",
      "├── [ 96M 01:19:33]  parity=0\n",
      "│   └── [ 96M 01:19:03]  2dc414ed-de4b-4645-87f7-93479f1f91cc-0_0-35-630_20230316011817174.parquet\n",
      "└── [ 96M 01:19:33]  parity=1\n",
      "    └── [ 96M 01:19:03]  a133a77d-c8b4-41cf-a371-eae3dcd43725-0_1-35-631_20230316011817174.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36dd0b25-0283-405f-b991-9e78dae4b353",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.6. Insert 14.6MB / +1 Big File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "06b721c3-1364-4622-ae23-d590e55a8da4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:19:33.100850Z",
     "iopub.status.busy": "2023-03-16T01:19:33.100594Z",
     "iopub.status.idle": "2023-03-16T01:20:22.563632Z",
     "shell.execute_reply": "2023-03-16T01:20:22.562833Z",
     "shell.execute_reply.started": "2023-03-16T01:19:33.100822Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "631d918a1aa1458eaca1b7ec95ebde8c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a7a35c0afa447bcbd21bad9eba26496",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "insert into \n",
    "    ${TABLE_NAME}\n",
    "select \n",
    "    review_id, \n",
    "    star_rating, \n",
    "    review_body, \n",
    "    review_date, \n",
    "    year,\n",
    "    unix_timestamp(current_timestamp()) as timestamp,\n",
    "    mod(crc32(review_id), 2) as parity\n",
    "from\n",
    "    reviews\n",
    "where\n",
    "    year = 1998;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "314c257b-00e9-48ea-a5a6-50e89d871d8f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:20:22.565351Z",
     "iopub.status.busy": "2023-03-16T01:20:22.565014Z",
     "iopub.status.idle": "2023-03-16T01:20:38.478727Z",
     "shell.execute_reply": "2023-03-16T01:20:38.477901Z",
     "shell.execute_reply.started": "2023-03-16T01:20:22.565313Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │        │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230316011817174 │ commit │ COMPLETED │ 03-16 01:18 │ 03-16 01:18 │ 03-16 01:19 ║\n",
      "╟─────┼───────────────────┼────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230316011934548 │ commit │ COMPLETED │ 03-16 01:19 │ 03-16 01:19 │ 03-16 01:20 ║\n",
      "╚═════╧═══════════════════╧════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230316011934548 │ 220.9 MB            │ 0                 │ 2                   │ 2                        │ 1333111               │ 0                            │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230316011817174 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_cow_layouts_sizing_1\n",
      "├── [206M 01:20:38]  parity=0\n",
      "│   ├── [ 96M 01:19:03]  2dc414ed-de4b-4645-87f7-93479f1f91cc-0_0-35-630_20230316011817174.parquet\n",
      "│   └── [110M 01:20:12]  2dc414ed-de4b-4645-87f7-93479f1f91cc-0_0-83-1421_20230316011934548.parquet\n",
      "└── [206M 01:20:38]  parity=1\n",
      "    ├── [ 96M 01:19:03]  a133a77d-c8b4-41cf-a371-eae3dcd43725-0_1-35-631_20230316011817174.parquet\n",
      "    └── [111M 01:20:12]  a133a77d-c8b4-41cf-a371-eae3dcd43725-0_1-83-1422_20230316011934548.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e56f987-da37-4cc5-9aa4-bfbabc80c2dd",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.7. Insert 3.7MB / +1 Small File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9acff543-7a63-4f5b-9d0b-ce1752a095c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:20:38.480460Z",
     "iopub.status.busy": "2023-03-16T01:20:38.480079Z",
     "iopub.status.idle": "2023-03-16T01:20:59.878890Z",
     "shell.execute_reply": "2023-03-16T01:20:59.878034Z",
     "shell.execute_reply.started": "2023-03-16T01:20:38.480402Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e748273459bc4b709e3180db635d829f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7afd3c98059649a7b2b91df2c135e5f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "insert into \n",
    "    ${TABLE_NAME}\n",
    "select \n",
    "    review_id, \n",
    "    star_rating, \n",
    "    review_body, \n",
    "    review_date, \n",
    "    year,\n",
    "    unix_timestamp(current_timestamp()) as timestamp,\n",
    "    mod(crc32(review_id), 2) as parity\n",
    "from\n",
    "    reviews\n",
    "where\n",
    "    year = 1997;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9f8e8282-9766-4c23-a301-0a0143023368",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:20:59.880262Z",
     "iopub.status.busy": "2023-03-16T01:20:59.880042Z",
     "iopub.status.idle": "2023-03-16T01:21:14.130651Z",
     "shell.execute_reply": "2023-03-16T01:21:14.129619Z",
     "shell.execute_reply.started": "2023-03-16T01:20:59.880240Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │        │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230316011817174 │ commit │ COMPLETED │ 03-16 01:18 │ 03-16 01:18 │ 03-16 01:19 ║\n",
      "╟─────┼───────────────────┼────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230316011934548 │ commit │ COMPLETED │ 03-16 01:19 │ 03-16 01:19 │ 03-16 01:20 ║\n",
      "╟─────┼───────────────────┼────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 2   │ 20230316012039888 │ commit │ COMPLETED │ 03-16 01:20 │ 03-16 01:20 │ 03-16 01:20 ║\n",
      "╚═════╧═══════════════════╧════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230316012039888 │ 7.4 MB              │ 2                 │ 0                   │ 2                        │ 39868                 │ 0                            │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230316011934548 │ 220.9 MB            │ 0                 │ 2                   │ 2                        │ 1333111               │ 0                            │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230316011817174 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_cow_layouts_sizing_1\n",
      "├── [210M 01:21:14]  parity=0\n",
      "│   ├── [ 96M 01:19:03]  2dc414ed-de4b-4645-87f7-93479f1f91cc-0_0-35-630_20230316011817174.parquet\n",
      "│   ├── [110M 01:20:12]  2dc414ed-de4b-4645-87f7-93479f1f91cc-0_0-83-1421_20230316011934548.parquet\n",
      "│   └── [3.7M 01:20:51]  3afbf3df-7be9-4fc0-86e3-e9ba4c59b84a-0_0-134-1908_20230316012039888.parquet\n",
      "└── [210M 01:21:14]  parity=1\n",
      "    ├── [ 96M 01:19:03]  a133a77d-c8b4-41cf-a371-eae3dcd43725-0_1-35-631_20230316011817174.parquet\n",
      "    ├── [111M 01:20:12]  a133a77d-c8b4-41cf-a371-eae3dcd43725-0_1-83-1422_20230316011934548.parquet\n",
      "    └── [3.7M 01:20:51]  a8494134-cc61-415f-8c2e-1a1c3fd2788b-0_1-134-1909_20230316012039888.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bad4bcc-7fd4-408b-b10d-78def052bee9",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.8. Insert 188.5MB / +1 Max File +1 Small File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7e51bc42-66e2-422a-bee8-5e6dbf7c7ae1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:21:14.132253Z",
     "iopub.status.busy": "2023-03-16T01:21:14.131986Z",
     "iopub.status.idle": "2023-03-16T01:22:21.652718Z",
     "shell.execute_reply": "2023-03-16T01:22:21.651845Z",
     "shell.execute_reply.started": "2023-03-16T01:21:14.132226Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db153a4f661540df8aae07d059deeeb0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e055236d41e244819243d8b3ca5f203a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "insert into \n",
    "    ${TABLE_NAME}\n",
    "select \n",
    "    review_id, \n",
    "    star_rating, \n",
    "    review_body, \n",
    "    review_date, \n",
    "    year,\n",
    "    unix_timestamp(current_timestamp()) as timestamp,\n",
    "    mod(crc32(review_id), 2) as parity\n",
    "from\n",
    "    reviews\n",
    "where\n",
    "    year = 2007;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "56b5c7a0-78e5-4df8-9e01-571418bc4dcf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:22:21.653990Z",
     "iopub.status.busy": "2023-03-16T01:22:21.653762Z",
     "iopub.status.idle": "2023-03-16T01:22:38.499058Z",
     "shell.execute_reply": "2023-03-16T01:22:38.497984Z",
     "shell.execute_reply.started": "2023-03-16T01:22:21.653968Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │        │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230316011817174 │ commit │ COMPLETED │ 03-16 01:18 │ 03-16 01:18 │ 03-16 01:19 ║\n",
      "╟─────┼───────────────────┼────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230316011934548 │ commit │ COMPLETED │ 03-16 01:19 │ 03-16 01:19 │ 03-16 01:20 ║\n",
      "╟─────┼───────────────────┼────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 2   │ 20230316012039888 │ commit │ COMPLETED │ 03-16 01:20 │ 03-16 01:20 │ 03-16 01:20 ║\n",
      "╟─────┼───────────────────┼────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 3   │ 20230316012115748 │ commit │ COMPLETED │ 03-16 01:21 │ 03-16 01:21 │ 03-16 01:22 ║\n",
      "╚═════╧═══════════════════╧════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230316012115748 │ 370.9 MB            │ 2                 │ 2                   │ 2                        │ 2234582               │ 0                            │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230316012039888 │ 7.4 MB              │ 2                 │ 0                   │ 2                        │ 39868                 │ 0                            │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230316011934548 │ 220.9 MB            │ 0                 │ 2                   │ 2                        │ 1333111               │ 0                            │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230316011817174 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_cow_layouts_sizing_1\n",
      "├── [395M 01:22:38]  parity=0\n",
      "│   ├── [ 96M 01:19:03]  2dc414ed-de4b-4645-87f7-93479f1f91cc-0_0-35-630_20230316011817174.parquet\n",
      "│   ├── [110M 01:20:12]  2dc414ed-de4b-4645-87f7-93479f1f91cc-0_0-83-1421_20230316011934548.parquet\n",
      "│   ├── [3.7M 01:20:51]  3afbf3df-7be9-4fc0-86e3-e9ba4c59b84a-0_0-134-1908_20230316012039888.parquet\n",
      "│   ├── [120M 01:22:09]  3afbf3df-7be9-4fc0-86e3-e9ba4c59b84a-0_0-188-2643_20230316012115748.parquet\n",
      "│   └── [ 66M 01:21:58]  4f2a9069-401e-4a39-90bc-438996351f48-0_1-188-2644_20230316012115748.parquet\n",
      "└── [396M 01:22:38]  parity=1\n",
      "    ├── [ 66M 01:21:58]  97438c40-c97c-48dd-8dd7-171bbcadf8f3-0_3-188-2646_20230316012115748.parquet\n",
      "    ├── [ 96M 01:19:03]  a133a77d-c8b4-41cf-a371-eae3dcd43725-0_1-35-631_20230316011817174.parquet\n",
      "    ├── [111M 01:20:12]  a133a77d-c8b4-41cf-a371-eae3dcd43725-0_1-83-1422_20230316011934548.parquet\n",
      "    ├── [3.7M 01:20:51]  a8494134-cc61-415f-8c2e-1a1c3fd2788b-0_1-134-1909_20230316012039888.parquet\n",
      "    └── [119M 01:22:09]  a8494134-cc61-415f-8c2e-1a1c3fd2788b-0_2-188-2645_20230316012115748.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c383c0c9-9598-4bf3-af10-1953d9589fe9",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 3. Test Case 2 - COW Default Settings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93c96ff2-778d-4a49-b9ac-d0c79083c3b5",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 3.1. Test Plan\n",
    "\n",
    "Step No.|Action|Volume Per Partition |Storage\n",
    ":--------:|:------|:------|:----------\n",
    "1|Insert|96MB|+1 Small File\n",
    "2|Insert|417MB|+3 Max Files + 1 Small File"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6b363ac-1fff-405c-89ab-ad91805d9e86",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3.2. Key Settings\n",
    "\n",
    "KEY|DEFAULT VALUE|SET VALUE\n",
    ":---|:---|:---\n",
    "hoodie.parquet.max.file.size|125829120 ( 120MB )|Default Value\n",
    "hoodie.parquet.small.file.limit|104857600 ( 100MB )|Default Value\n",
    "hoodie.copyonwrite.record.size.estimate|1024|Default Value"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3a15786-7b28-4382-921e-ead944c10530",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-25T02:57:45.037101Z",
     "iopub.status.busy": "2023-02-25T02:57:45.036681Z",
     "iopub.status.idle": "2023-02-25T02:58:07.353493Z",
     "shell.execute_reply": "2023-02-25T02:58:07.352596Z",
     "shell.execute_reply.started": "2023-02-25T02:57:45.037060Z"
    },
    "tags": []
   },
   "source": [
    "### 3.3. Set Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6ff73c99-829a-4834-b175-8920bab4e894",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:22:38.501025Z",
     "iopub.status.busy": "2023-03-16T01:22:38.500662Z",
     "iopub.status.idle": "2023-03-16T01:22:39.430773Z",
     "shell.execute_reply": "2023-03-16T01:22:39.429948Z",
     "shell.execute_reply.started": "2023-03-16T01:22:38.500983Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ff4c00eae654e98b93587785a562600",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(HTML(value='Type:'), Button(description='Table', layout=Layout(width='70px'), st…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2cecabfe66044a8688f8db10fb69fc03",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "set TABLE_NAME=reviews_cow_layouts_sizing_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cde89744-801a-4cb0-b7ec-dbcde90add0c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:22:39.432517Z",
     "iopub.status.busy": "2023-03-16T01:22:39.432166Z",
     "iopub.status.idle": "2023-03-16T01:22:39.437760Z",
     "shell.execute_reply": "2023-03-16T01:22:39.436832Z",
     "shell.execute_reply.started": "2023-03-16T01:22:39.432480Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: TABLE_NAME=reviews_cow_layouts_sizing_2\n"
     ]
    }
   ],
   "source": [
    "%env TABLE_NAME=reviews_cow_layouts_sizing_2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79458fca-7788-49a0-ae8c-8162ce3579da",
   "metadata": {},
   "source": [
    "### 3.4. Create Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f77d0fce-c72f-4eff-81b2-b472a26107af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:22:39.442120Z",
     "iopub.status.busy": "2023-03-16T01:22:39.441899Z",
     "iopub.status.idle": "2023-03-16T01:22:44.941984Z",
     "shell.execute_reply": "2023-03-16T01:22:44.940700Z",
     "shell.execute_reply.started": "2023-03-16T01:22:39.442098Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%sh\n",
    "aws s3 rm s3://${S3_BUCKET}/${TABLE_NAME} --recursive &>/dev/null\n",
    "rm -rf ${WORKSPACE}/${TABLE_NAME}\n",
    "sleep 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "23970243-d522-4d06-a6e2-8ed6e4bd1234",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:22:44.944516Z",
     "iopub.status.busy": "2023-03-16T01:22:44.944214Z",
     "iopub.status.idle": "2023-03-16T01:22:45.811474Z",
     "shell.execute_reply": "2023-03-16T01:22:45.810533Z",
     "shell.execute_reply.started": "2023-03-16T01:22:44.944486Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9425e35aef14617b7f188f1cb9c4bdd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12461ea2939f4e88be6b276eb18056a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a8c6e3bf-e01e-428d-8d26-bffbc4313f68",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:22:45.812967Z",
     "iopub.status.busy": "2023-03-16T01:22:45.812708Z",
     "iopub.status.idle": "2023-03-16T01:22:49.183842Z",
     "shell.execute_reply": "2023-03-16T01:22:49.182940Z",
     "shell.execute_reply.started": "2023-03-16T01:22:45.812943Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25230db236f4405db5c442e3d222a379",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca2fe9b7e33945cb9134f2df6e3c9c6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "create table if not exists ${TABLE_NAME}(\n",
    "    review_id string, \n",
    "    star_rating int, \n",
    "    review_body string, \n",
    "    review_date date, \n",
    "    year long,\n",
    "    timestamp long,\n",
    "    parity int\n",
    ")\n",
    "using hudi\n",
    "location 's3://${S3_BUCKET}/${TABLE_NAME}'\n",
    "partitioned by (parity)\n",
    "options ( \n",
    "    type = 'cow',  \n",
    "    primaryKey = 'review_id', \n",
    "    preCombineField = 'timestamp'\n",
    "    -- hoodie.copyonwrite.record.size.estimate = '175'\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a3bd4c0-8cbb-44ef-aacf-880e54c7638c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3.5. Insert 96MB / +5 Small Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8f8d4370-23da-4c05-b708-ffdb96147712",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:22:49.185036Z",
     "iopub.status.busy": "2023-03-16T01:22:49.184798Z",
     "iopub.status.idle": "2023-03-16T01:23:20.596278Z",
     "shell.execute_reply": "2023-03-16T01:23:20.595528Z",
     "shell.execute_reply.started": "2023-03-16T01:22:49.185013Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee95a446a60646d5b015ae94461a850e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "108311e57d8a42b5b5dc79f88e94b00c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "insert into \n",
    "    ${TABLE_NAME}\n",
    "select \n",
    "    review_id, \n",
    "    star_rating, \n",
    "    review_body, \n",
    "    review_date, \n",
    "    year,\n",
    "    unix_timestamp(current_timestamp()) as timestamp,\n",
    "    mod(crc32(review_id), 2) as parity\n",
    "from\n",
    "    reviews\n",
    "where\n",
    "    year = 2003;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "45164a8c-726a-4949-b09e-e8f7c384c014",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:23:20.597504Z",
     "iopub.status.busy": "2023-03-16T01:23:20.597248Z",
     "iopub.status.idle": "2023-03-16T01:23:36.135641Z",
     "shell.execute_reply": "2023-03-16T01:23:36.134793Z",
     "shell.execute_reply.started": "2023-03-16T01:23:20.597479Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │        │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230316012249944 │ commit │ COMPLETED │ 03-16 01:22 │ 03-16 01:23 │ 03-16 01:23 ║\n",
      "╚═════╧═══════════════════╧════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230316012249944 │ 198.3 MB            │ 10                │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_cow_layouts_sizing_2\n",
      "├── [ 99M 01:23:36]  parity=0\n",
      "│   ├── [ 21M 01:23:10]  46b75569-ba90-438e-a41b-78083fa4c975-0_2-253-3317_20230316012249944.parquet\n",
      "│   ├── [ 21M 01:23:10]  49bb2f8f-9b81-496d-9383-103c561ab56b-0_0-253-3315_20230316012249944.parquet\n",
      "│   ├── [ 15M 01:23:09]  d4cee4f2-223d-439e-8e34-53a8b9df6d7a-0_4-253-3319_20230316012249944.parquet\n",
      "│   ├── [ 21M 01:23:10]  d95b7b46-c5cf-4e37-9cfa-0204395379db-0_3-253-3318_20230316012249944.parquet\n",
      "│   └── [ 21M 01:23:10]  eb41029f-29d1-42c5-a0a4-c3a392251950-0_1-253-3316_20230316012249944.parquet\n",
      "└── [ 99M 01:23:36]  parity=1\n",
      "    ├── [ 15M 01:23:09]  8cc73e38-5adc-4336-b947-d8693127fbdf-0_9-253-3324_20230316012249944.parquet\n",
      "    ├── [ 21M 01:23:10]  c658aa55-6557-40e5-8143-06cfe23ef8bd-0_8-253-3323_20230316012249944.parquet\n",
      "    ├── [ 21M 01:23:10]  c99abfb5-a3a2-4bd7-ab59-a6fd0d60d263-0_6-253-3321_20230316012249944.parquet\n",
      "    ├── [ 21M 01:23:10]  ecc352d2-ebc4-4e11-92f1-6707f374b081-0_5-253-3320_20230316012249944.parquet\n",
      "    └── [ 21M 01:23:10]  f688f94a-76bb-4e76-8df2-40a34d288432-0_7-253-3322_20230316012249944.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37546d50-d336-408c-82b4-c7d33c5b3d89",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3.6. Insert 417MB / +3 Max Files + 1 Small File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f7b92fe2-0f7f-46d9-aead-6345b2a6b571",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:23:36.137174Z",
     "iopub.status.busy": "2023-03-16T01:23:36.136901Z",
     "iopub.status.idle": "2023-03-16T01:25:13.743262Z",
     "shell.execute_reply": "2023-03-16T01:25:13.742378Z",
     "shell.execute_reply.started": "2023-03-16T01:23:36.137147Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eaa56fe6ef3b4e1d994ad280bfb5fe5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc0c5eaec38d43b09b921bb04c0b0b44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "insert into \n",
    "    ${TABLE_NAME}\n",
    "select \n",
    "    review_id, \n",
    "    star_rating, \n",
    "    review_body, \n",
    "    review_date, \n",
    "    year,\n",
    "    unix_timestamp(current_timestamp()) as timestamp,\n",
    "    mod(crc32(review_id), 2) as parity\n",
    "from\n",
    "    reviews\n",
    "where\n",
    "    year = 2010;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d9cc764a-2a72-4364-910b-2c6af64e6d1c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-16T01:25:13.744581Z",
     "iopub.status.busy": "2023-03-16T01:25:13.744291Z",
     "iopub.status.idle": "2023-03-16T01:25:34.045003Z",
     "shell.execute_reply": "2023-03-16T01:25:34.043981Z",
     "shell.execute_reply.started": "2023-03-16T01:25:13.744556Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │        │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230316012249944 │ commit │ COMPLETED │ 03-16 01:22 │ 03-16 01:23 │ 03-16 01:23 ║\n",
      "╟─────┼───────────────────┼────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230316012337424 │ commit │ COMPLETED │ 03-16 01:23 │ 03-16 01:24 │ 03-16 01:25 ║\n",
      "╚═════╧═══════════════════╧════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230316012337424 │ 834.2 MB            │ 0                 │ 8                   │ 2                        │ 5029580               │ 0                            │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230316012249944 │ 198.3 MB            │ 10                │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_cow_layouts_sizing_2\n",
      "├── [516M 01:25:33]  parity=0\n",
      "│   ├── [ 21M 01:23:10]  46b75569-ba90-438e-a41b-78083fa4c975-0_2-253-3317_20230316012249944.parquet\n",
      "│   ├── [ 21M 01:23:10]  49bb2f8f-9b81-496d-9383-103c561ab56b-0_0-253-3315_20230316012249944.parquet\n",
      "│   ├── [116M 01:25:01]  49bb2f8f-9b81-496d-9383-103c561ab56b-0_2-301-4062_20230316012337424.parquet\n",
      "│   ├── [ 69M 01:24:52]  d4cee4f2-223d-439e-8e34-53a8b9df6d7a-0_3-301-4063_20230316012337424.parquet\n",
      "│   ├── [ 15M 01:23:09]  d4cee4f2-223d-439e-8e34-53a8b9df6d7a-0_4-253-3319_20230316012249944.parquet\n",
      "│   ├── [116M 01:25:01]  d95b7b46-c5cf-4e37-9cfa-0204395379db-0_1-301-4061_20230316012337424.parquet\n",
      "│   ├── [ 21M 01:23:10]  d95b7b46-c5cf-4e37-9cfa-0204395379db-0_3-253-3318_20230316012249944.parquet\n",
      "│   ├── [116M 01:25:01]  eb41029f-29d1-42c5-a0a4-c3a392251950-0_0-301-4060_20230316012337424.parquet\n",
      "│   └── [ 21M 01:23:10]  eb41029f-29d1-42c5-a0a4-c3a392251950-0_1-253-3316_20230316012249944.parquet\n",
      "└── [516M 01:25:33]  parity=1\n",
      "    ├── [116M 01:25:01]  8cc73e38-5adc-4336-b947-d8693127fbdf-0_4-301-4064_20230316012337424.parquet\n",
      "    ├── [ 15M 01:23:09]  8cc73e38-5adc-4336-b947-d8693127fbdf-0_9-253-3324_20230316012249944.parquet\n",
      "    ├── [116M 01:25:01]  c658aa55-6557-40e5-8143-06cfe23ef8bd-0_5-301-4065_20230316012337424.parquet\n",
      "    ├── [ 21M 01:23:10]  c658aa55-6557-40e5-8143-06cfe23ef8bd-0_8-253-3323_20230316012249944.parquet\n",
      "    ├── [ 21M 01:23:10]  c99abfb5-a3a2-4bd7-ab59-a6fd0d60d263-0_6-253-3321_20230316012249944.parquet\n",
      "    ├── [ 21M 01:23:10]  ecc352d2-ebc4-4e11-92f1-6707f374b081-0_5-253-3320_20230316012249944.parquet\n",
      "    ├── [116M 01:25:01]  ecc352d2-ebc4-4e11-92f1-6707f374b081-0_6-301-4066_20230316012337424.parquet\n",
      "    ├── [ 21M 01:23:10]  f688f94a-76bb-4e76-8df2-40a34d288432-0_7-253-3322_20230316012249944.parquet\n",
      "    └── [ 70M 01:24:53]  f688f94a-76bb-4e76-8df2-40a34d288432-0_7-301-4067_20230316012337424.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits storage"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Spark",
   "language": "scala",
   "name": "sparkkernel"
  },
  "language_info": {
   "codemirror_mode": "text/x-scala",
   "file_extension": ".sc",
   "mimetype": "text/x-scala",
   "name": "scala",
   "pygments_lexer": "scala"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}