4-mor-compaction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1d574cc5-0034-439e-b4c9-45f0afea203d",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Apache Hudi Core Conceptions (4) - MOR: Compaction"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8f99894-93f3-4cef-a853-d2d73a3515b6",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 1. Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "638504d6-1136-4807-af98-1d4248d1a8f3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:43:39.846078Z",
     "iopub.status.busy": "2023-03-22T01:43:39.845812Z",
     "iopub.status.idle": "2023-03-22T01:43:44.300348Z",
     "shell.execute_reply": "2023-03-22T01:43:44.299528Z",
     "shell.execute_reply.started": "2023-03-22T01:43:39.846053Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--   1 emr-notebook hdfsadmingroup   61421977 2023-03-22 01:43 /tmp/hudi-spark-bundle.jar\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "# deploy hudi bundle jar\n",
    "hdfs dfs -copyFromLocal -f /usr/lib/hudi/hudi-spark-bundle.jar /tmp/hudi-spark-bundle.jar\n",
    "hdfs dfs -ls /tmp/hudi-spark-bundle.jar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c7b533e6-60dc-44b6-be57-d3b86c11c583",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:43:44.302023Z",
     "iopub.status.busy": "2023-03-22T01:43:44.301760Z",
     "iopub.status.idle": "2023-03-22T01:43:44.319010Z",
     "shell.execute_reply": "2023-03-22T01:43:44.318267Z",
     "shell.execute_reply.started": "2023-03-22T01:43:44.301997Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "Current session configs: <tt>{'conf': {'spark.jars': 'hdfs:///tmp/hudi-spark-bundle.jar', 'spark.serializer': 'org.apache.spark.serializer.KryoSerializer', 'spark.sql.extensions': 'org.apache.spark.sql.hudi.HoodieSparkSessionExtension', 'spark.sql.catalog.spark_catalog': 'org.apache.spark.sql.hudi.catalog.HoodieCatalog'}, 'proxyUser': 'user_bluishglc', 'kind': 'spark'}</tt><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "No active sessions."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%configure -f\n",
    "{\n",
    "    \"conf\" : {\n",
    "        \"spark.jars\":\"hdfs:///tmp/hudi-spark-bundle.jar\",            \n",
    "        \"spark.serializer\":\"org.apache.spark.serializer.KryoSerializer\",\n",
    "        \"spark.sql.extensions\":\"org.apache.spark.sql.hudi.HoodieSparkSessionExtension\",\n",
    "        \"spark.sql.catalog.spark_catalog\":\"org.apache.spark.sql.hudi.catalog.HoodieCatalog\"\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8af4b944-fe51-41b9-90fb-ed249199f406",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:43:44.320661Z",
     "iopub.status.busy": "2023-03-22T01:43:44.320437Z",
     "iopub.status.idle": "2023-03-22T01:43:44.325217Z",
     "shell.execute_reply": "2023-03-22T01:43:44.324376Z",
     "shell.execute_reply.started": "2023-03-22T01:43:44.320640Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: S3_BUCKET=apache-hudi-core-conceptions\n"
     ]
    }
   ],
   "source": [
    "%env S3_BUCKET=apache-hudi-core-conceptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c5d08e30-76d6-4d19-af7c-15a49469c013",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:43:44.326844Z",
     "iopub.status.busy": "2023-03-22T01:43:44.326566Z",
     "iopub.status.idle": "2023-03-22T01:44:15.980489Z",
     "shell.execute_reply": "2023-03-22T01:44:15.979540Z",
     "shell.execute_reply.started": "2023-03-22T01:43:44.326822Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Spark application\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tbody><tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>User</th><th>Current session?</th></tr><tr><td>106</td><td>application_1678096020253_0216</td><td>spark</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-10-0-129-209.ec2.internal:20888/proxy/application_1678096020253_0216/\" class=\"emr-proxy-link j-6V65AY8X5VHR application_1678096020253_0216\" emr-resource=\"j-6V65AY8X5VHR\n",
       "\" application-id=\"application_1678096020253_0216\">Link</a></td><td><a target=\"_blank\" href=\"http://ip-10-0-132-108.ec2.internal:8042/node/containerlogs/container_1678096020253_0216_01_000001/livy\">Link</a></td><td>None</td><td>✔</td></tr></tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparkSession available as 'spark'.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "382496935cd4486d8b80a2af5ab7773c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(HTML(value='Type:'), Button(description='Table', layout=Layout(width='70px'), st…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54fb42a8e2894bd7ba95154a66905350",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "set S3_BUCKET=apache-hudi-core-conceptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2b88311f-c410-4935-a41a-ce3d81c84920",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:15.982168Z",
     "iopub.status.busy": "2023-03-22T01:44:15.981761Z",
     "iopub.status.idle": "2023-03-22T01:44:15.990672Z",
     "shell.execute_reply": "2023-03-22T01:44:15.989866Z",
     "shell.execute_reply.started": "2023-03-22T01:44:15.982134Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: WORKSPACE=/home/emr-notebook/apache-hudi-core-conceptions\n"
     ]
    }
   ],
   "source": [
    "%env WORKSPACE=/home/emr-notebook/apache-hudi-core-conceptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "78881ec6-4329-416b-ab27-38339020e322",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:15.992554Z",
     "iopub.status.busy": "2023-03-22T01:44:15.992070Z",
     "iopub.status.idle": "2023-03-22T01:44:16.162938Z",
     "shell.execute_reply": "2023-03-22T01:44:16.161540Z",
     "shell.execute_reply.started": "2023-03-22T01:44:15.992520Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/emr-notebook/apache-hudi-core-conceptions/hudi-stat.sh\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "# make workspace\n",
    "mkdir -p $WORKSPACE\n",
    "# deploy hudi-stat.sh, a utility shell script to output hudi table status\n",
    "wget https://raw.githubusercontent.com/bluishglc/apache-hudi-core-conceptions/master/hudi-stat.sh -O $WORKSPACE/hudi-stat.sh &>/dev/null\n",
    "chmod a+x $WORKSPACE/hudi-stat.sh\n",
    "ls $WORKSPACE/hudi-stat.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "db234f04-c0c0-4be7-829b-0bbfc6e1395f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:16.165058Z",
     "iopub.status.busy": "2023-03-22T01:44:16.164694Z",
     "iopub.status.idle": "2023-03-22T01:44:16.175811Z",
     "shell.execute_reply": "2023-03-22T01:44:16.174440Z",
     "shell.execute_reply.started": "2023-03-22T01:44:16.165002Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       "table {float:left}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<style>\n",
    "table {float:left}\n",
    "</style>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7915fc55-ebe6-4766-b32d-73f052f50c4f",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 2. Test Case 1 - Sync Compaction ( Inline Schedule + Inline Execute )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "792c1cd6-5b05-4ca7-a97b-a30a599fc585",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 2.1. Test Plan\n",
    "\n",
    "Step No.|Action|Volume Per Partition |Storage\n",
    ":--------:|:------|:------|:----------\n",
    "1|Insert|96MB|+1 Base File\n",
    "2|Update|788KB|+1 Log File\n",
    "3|Update|1.2MB|+1 Log File +1 Compacted Base File"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41006950-e17e-46d5-9d8c-5ece022ecebd",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.2. Key Settings\n",
    "\n",
    "KEY|DEFAULT VALUE|SET VALUE\n",
    ":---|:---|:---\n",
    "hoodie.compact.inline|false|true\n",
    "hoodie.compact.schedule.inline|false|false\n",
    "hoodie.compact.inline.max.delta.commits|5|3\n",
    "hoodie.copyonwrite.record.size.estimate|1024|175"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a69bbf4-b366-4974-996e-55533732ff10",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-25T02:57:45.037101Z",
     "iopub.status.busy": "2023-02-25T02:57:45.036681Z",
     "iopub.status.idle": "2023-02-25T02:58:07.353493Z",
     "shell.execute_reply": "2023-02-25T02:58:07.352596Z",
     "shell.execute_reply.started": "2023-02-25T02:57:45.037060Z"
    },
    "tags": []
   },
   "source": [
    "### 2.3. Set Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1c2945f5-27a3-45ab-90e5-3f76e941f758",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:16.179577Z",
     "iopub.status.busy": "2023-03-22T01:44:16.179062Z",
     "iopub.status.idle": "2023-03-22T01:44:17.187836Z",
     "shell.execute_reply": "2023-03-22T01:44:17.186830Z",
     "shell.execute_reply.started": "2023-03-22T01:44:16.179542Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e5320fed4d54e26ac23cc93db96377d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(HTML(value='Type:'), Button(description='Table', layout=Layout(width='70px'), st…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96b9855d10c5422ebfd610acc197bd87",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "set TABLE_NAME=reviews_mor_compaction_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3438353c-696d-4b31-bd71-cc59834b6178",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:17.190484Z",
     "iopub.status.busy": "2023-03-22T01:44:17.190033Z",
     "iopub.status.idle": "2023-03-22T01:44:17.196579Z",
     "shell.execute_reply": "2023-03-22T01:44:17.195598Z",
     "shell.execute_reply.started": "2023-03-22T01:44:17.190451Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: TABLE_NAME=reviews_mor_compaction_1\n"
     ]
    }
   ],
   "source": [
    "%env TABLE_NAME=reviews_mor_compaction_1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d28de246-213d-417f-9846-7f972772dc6f",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.4. Create Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b5dc879f-be2f-4321-a11d-348432c93fc3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:17.198414Z",
     "iopub.status.busy": "2023-03-22T01:44:17.198119Z",
     "iopub.status.idle": "2023-03-22T01:44:23.165907Z",
     "shell.execute_reply": "2023-03-22T01:44:23.165046Z",
     "shell.execute_reply.started": "2023-03-22T01:44:17.198384Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%sh\n",
    "aws s3 rm s3://${S3_BUCKET}/${TABLE_NAME} --recursive &>/dev/null\n",
    "rm -rf ${WORKSPACE}/${TABLE_NAME}\n",
    "sleep 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "18a7e743-3174-48f0-a03a-e0facb60ef31",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:23.167448Z",
     "iopub.status.busy": "2023-03-22T01:44:23.167169Z",
     "iopub.status.idle": "2023-03-22T01:44:26.516284Z",
     "shell.execute_reply": "2023-03-22T01:44:26.515439Z",
     "shell.execute_reply.started": "2023-03-22T01:44:23.167422Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d95ab03890454f6ba54cadbc3b0ed4b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "048372318dc948fb94a5bf600babbdf4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2925561b-4951-4ef5-b911-9652d50c26f7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:26.517575Z",
     "iopub.status.busy": "2023-03-22T01:44:26.517339Z",
     "iopub.status.idle": "2023-03-22T01:44:27.335569Z",
     "shell.execute_reply": "2023-03-22T01:44:27.334795Z",
     "shell.execute_reply.started": "2023-03-22T01:44:26.517552Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85a36aab44c7428daf7ee6c7b23190b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b534a471da254e8d82308332611ba63d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}_ro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "209f965e-f46f-4087-8ab1-455047885371",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:27.337185Z",
     "iopub.status.busy": "2023-03-22T01:44:27.336860Z",
     "iopub.status.idle": "2023-03-22T01:44:28.198584Z",
     "shell.execute_reply": "2023-03-22T01:44:28.197813Z",
     "shell.execute_reply.started": "2023-03-22T01:44:27.337150Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d51ab1e565bb4dea82114fb9e35f5d84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "921e8da82e9c4eb799089135288b5a93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}_rt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "696144de-8567-4b4f-8608-b2e33bfc0b0b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:28.200168Z",
     "iopub.status.busy": "2023-03-22T01:44:28.199906Z",
     "iopub.status.idle": "2023-03-22T01:44:31.542831Z",
     "shell.execute_reply": "2023-03-22T01:44:31.541955Z",
     "shell.execute_reply.started": "2023-03-22T01:44:28.200137Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f65dd4836d57484aafc7a275d1015372",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c3a4bfd68e345969ac1dd35e4c76fee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "create table if not exists ${TABLE_NAME} (\n",
    "    review_id string, \n",
    "    star_rating long, \n",
    "    review_body string, \n",
    "    review_date date, \n",
    "    year long,\n",
    "    timestamp long,\n",
    "    parity int\n",
    ")\n",
    "using hudi\n",
    "location 's3://${S3_BUCKET}/${TABLE_NAME}'\n",
    "partitioned by (parity)\n",
    "options ( \n",
    "    type = 'mor',  \n",
    "    primaryKey = 'review_id', \n",
    "    preCombineField = 'timestamp',\n",
    "    hoodie.compact.inline = 'true',\n",
    "    hoodie.compact.schedule.inline = 'false',\n",
    "    hoodie.compact.inline.max.delta.commits = '3',\n",
    "    hoodie.copyonwrite.record.size.estimate = '175'\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56de236b-7561-4cd0-a6e2-3fedc1074a4a",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.5. Insert 96MB / +1 Base File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "75655d62-397d-4399-87e0-511af8a5e252",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:44:31.544197Z",
     "iopub.status.busy": "2023-03-22T01:44:31.543956Z",
     "iopub.status.idle": "2023-03-22T01:45:31.120061Z",
     "shell.execute_reply": "2023-03-22T01:45:31.118666Z",
     "shell.execute_reply.started": "2023-03-22T01:44:31.544174Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02b922828ca94ba594f27b5d779ab710",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1751d083a3845d895189711fc9f85c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "insert into \n",
    "    ${TABLE_NAME}\n",
    "select \n",
    "    review_id, \n",
    "    star_rating, \n",
    "    review_body, \n",
    "    review_date, \n",
    "    year,\n",
    "    unix_timestamp(current_timestamp()) as timestamp,\n",
    "    mod(crc32(review_id), 2) as parity\n",
    "from\n",
    "    reviews\n",
    "where\n",
    "    year = 2003"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "026ac372-3a9a-4cb2-b05c-d662efc2a2ce",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:45:31.122679Z",
     "iopub.status.busy": "2023-03-22T01:45:31.121688Z",
     "iopub.status.idle": "2023-03-22T01:45:57.232164Z",
     "shell.execute_reply": "2023-03-22T01:45:57.231237Z",
     "shell.execute_reply.started": "2023-03-22T01:45:31.122639Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322014432551 │ deltacommit │ COMPLETED │ 03-22 01:44 │ 03-22 01:44 │ 03-22 01:45 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322014432551 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╧═══════╧═══════════════════════════════╣\n",
      "║ (empty)                                                         ║\n",
      "╚═════════════════════════════════════════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_1\n",
      "├── [ 96M 01:45:57]  parity=0\n",
      "│   └── [ 96M 01:45:17]  3c5ced6f-1613-4420-b8f5-125755ba3012-0_0-35-630_20230322014432551.parquet\n",
      "└── [ 96M 01:45:57]  parity=1\n",
      "    └── [ 96M 01:45:17]  7ac80336-217e-4c40-b278-be6436f275d2-0_1-35-631_20230322014432551.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b09ae78-5fac-4a89-956d-6482046a0cae",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.6. Update 787KB / +1 Log File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d7b2a994-1877-49b3-adb7-3e0bcbc603d6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:45:57.233976Z",
     "iopub.status.busy": "2023-03-22T01:45:57.233597Z",
     "iopub.status.idle": "2023-03-22T01:46:16.631936Z",
     "shell.execute_reply": "2023-03-22T01:46:16.631214Z",
     "shell.execute_reply.started": "2023-03-22T01:45:57.233937Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "776a663146d1410384caf1f72b40a229",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f89b622f5ce5451cbbffe6a056a7dbc9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "update\n",
    "    ${TABLE_NAME}\n",
    "set             \n",
    "    review_body = concat(uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid()),\n",
    "    timestamp = unix_timestamp(current_timestamp())\n",
    "where\n",
    "    review_date = '2003-01-01'\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4a3f08c1-7783-46af-a275-fdbb00f6ba0e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:46:16.633186Z",
     "iopub.status.busy": "2023-03-22T01:46:16.632960Z",
     "iopub.status.idle": "2023-03-22T01:46:37.478153Z",
     "shell.execute_reply": "2023-03-22T01:46:37.477271Z",
     "shell.execute_reply.started": "2023-03-22T01:46:16.633164Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322014432551 │ deltacommit │ COMPLETED │ 03-22 01:44 │ 03-22 01:44 │ 03-22 01:45 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322014557951 │ deltacommit │ COMPLETED │ 03-22 01:46 │ 03-22 01:46 │ 03-22 01:46 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322014557951 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014432551 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╧═══════╧═══════════════════════════════╣\n",
      "║ (empty)                                                         ║\n",
      "╚═════════════════════════════════════════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_1\n",
      "├── [ 97M 01:46:37]  parity=0\n",
      "│   ├── [787K 01:46:07]  .3c5ced6f-1613-4420-b8f5-125755ba3012-0_20230322014432551.log.1_0-83-691\n",
      "│   └── [ 96M 01:45:17]  3c5ced6f-1613-4420-b8f5-125755ba3012-0_0-35-630_20230322014432551.parquet\n",
      "└── [ 97M 01:46:37]  parity=1\n",
      "    ├── [813K 01:46:07]  .7ac80336-217e-4c40-b278-be6436f275d2-0_20230322014432551.log.1_1-83-692\n",
      "    └── [ 96M 01:45:17]  7ac80336-217e-4c40-b278-be6436f275d2-0_1-35-631_20230322014432551.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92188a32-f51e-402b-a851-85b21021d0bb",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.7. Update 1.2MB / +1 Log File +1 Base File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4c2900a0-e7a5-4a0e-9811-4c8511c2d699",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:46:37.479706Z",
     "iopub.status.busy": "2023-03-22T01:46:37.479445Z",
     "iopub.status.idle": "2023-03-22T01:47:25.209560Z",
     "shell.execute_reply": "2023-03-22T01:47:25.208667Z",
     "shell.execute_reply.started": "2023-03-22T01:46:37.479679Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e69e54cbd7046e8a101a365e32f51bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1b8692f74314277a934354d9862b935",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "update\n",
    "    ${TABLE_NAME}\n",
    "set             \n",
    "    review_body = concat(uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid()),\n",
    "    timestamp = unix_timestamp(current_timestamp())\n",
    "where\n",
    "    review_date = '2003-01-02'\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d656a2a9-0d72-4afd-b1b0-ce71be37c43d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:47:25.210872Z",
     "iopub.status.busy": "2023-03-22T01:47:25.210646Z",
     "iopub.status.idle": "2023-03-22T01:47:50.714062Z",
     "shell.execute_reply": "2023-03-22T01:47:50.713263Z",
     "shell.execute_reply.started": "2023-03-22T01:47:25.210851Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322014432551 │ deltacommit │ COMPLETED │ 03-22 01:44 │ 03-22 01:44 │ 03-22 01:45 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322014557951 │ deltacommit │ COMPLETED │ 03-22 01:46 │ 03-22 01:46 │ 03-22 01:46 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 2   │ 20230322014638169 │ deltacommit │ COMPLETED │ 03-22 01:46 │ 03-22 01:46 │ 03-22 01:46 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 3   │ 20230322014653216 │ commit      │ COMPLETED │ 03-22 01:46 │ 03-22 01:46 │ 03-22 01:47 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322014653216 │ 192.3 MB            │ 0                 │ 2                   │ 2                        │ 1155127               │ 7292                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014638169 │ 2.3 MB              │ 0                 │ 2                   │ 2                        │ 4341                  │ 4341                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014557951 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014432551 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State     │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╪═══════════╪═══════════════════════════════╣\n",
      "║ 20230322014653216       │ COMPLETED │ 2                             ║\n",
      "╚═════════════════════════╧═══════════╧═══════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_1\n",
      "├── [194M 01:47:50]  parity=0\n",
      "│   ├── [787K 01:46:07]  .3c5ced6f-1613-4420-b8f5-125755ba3012-0_20230322014432551.log.1_0-83-691\n",
      "│   ├── [1.2M 01:46:47]  .3c5ced6f-1613-4420-b8f5-125755ba3012-0_20230322014432551.log.2_0-131-752\n",
      "│   ├── [ 96M 01:47:17]  3c5ced6f-1613-4420-b8f5-125755ba3012-0_0-154-773_20230322014653216.parquet\n",
      "│   └── [ 96M 01:45:17]  3c5ced6f-1613-4420-b8f5-125755ba3012-0_0-35-630_20230322014432551.parquet\n",
      "└── [194M 01:47:50]  parity=1\n",
      "    ├── [813K 01:46:07]  .7ac80336-217e-4c40-b278-be6436f275d2-0_20230322014432551.log.1_1-83-692\n",
      "    ├── [1.1M 01:46:47]  .7ac80336-217e-4c40-b278-be6436f275d2-0_20230322014432551.log.2_1-131-753\n",
      "    ├── [ 96M 01:47:17]  7ac80336-217e-4c40-b278-be6436f275d2-0_1-154-774_20230322014653216.parquet\n",
      "    └── [ 96M 01:45:17]  7ac80336-217e-4c40-b278-be6436f275d2-0_1-35-631_20230322014432551.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6b4d1a0-eb11-44c8-84d6-e5511b010ee1",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 3. Test Case 2 - Async Compaction ( Offline Schedule, Offline Execute )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "733e27a1-3654-46c9-a983-e5b50cce7a2f",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3.1. Test Plan\n",
    "\n",
    "Step No.|Action|Volume Per Partition |Storage\n",
    ":--------:|:------|:------|:----------\n",
    "1|Insert|96MB|+1 Base File\n",
    "2|Update|788KB|+1 Log File\n",
    "3|Update|1.2MB|+1 Log File\n",
    "4|Offline Schedule|N/A|N/A\n",
    "5|Offline Execute|96.15MB|+1 Compacted Base File"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97e4f5b9-0a56-449d-8ebe-fcc0407f13de",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3.2. Key Settings\n",
    "\n",
    "KEY|DEFAULT VALUE|SET VALUE\n",
    ":---|:---|:---\n",
    "hoodie.compact.inline|false|true\n",
    "hoodie.compact.schedule.inline|false|false\n",
    "hoodie.compact.inline.max.delta.commits|5|3\n",
    "hoodie.copyonwrite.record.size.estimate|1024|175"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fdade50-00ad-4d47-b203-81d8c8066938",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-25T02:57:45.037101Z",
     "iopub.status.busy": "2023-02-25T02:57:45.036681Z",
     "iopub.status.idle": "2023-02-25T02:58:07.353493Z",
     "shell.execute_reply": "2023-02-25T02:58:07.352596Z",
     "shell.execute_reply.started": "2023-02-25T02:57:45.037060Z"
    },
    "tags": []
   },
   "source": [
    "### 3.3. Set Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3e358585-c75c-4840-98da-f3385d2d915d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:47:50.715497Z",
     "iopub.status.busy": "2023-03-22T01:47:50.715244Z",
     "iopub.status.idle": "2023-03-22T01:47:51.723929Z",
     "shell.execute_reply": "2023-03-22T01:47:51.723181Z",
     "shell.execute_reply.started": "2023-03-22T01:47:50.715471Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53027715505940dfbc8598ad7cd59eb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(HTML(value='Type:'), Button(description='Table', layout=Layout(width='70px'), st…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "213d52baff8b44fe88447c240bad42d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "set TABLE_NAME=reviews_mor_compaction_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a961a1cd-7a61-4fe8-aef5-36d54331aaef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:47:51.725447Z",
     "iopub.status.busy": "2023-03-22T01:47:51.725145Z",
     "iopub.status.idle": "2023-03-22T01:47:51.730075Z",
     "shell.execute_reply": "2023-03-22T01:47:51.729358Z",
     "shell.execute_reply.started": "2023-03-22T01:47:51.725417Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: TABLE_NAME=reviews_mor_compaction_2\n"
     ]
    }
   ],
   "source": [
    "%env TABLE_NAME=reviews_mor_compaction_2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50c3d8e7-c792-45c7-b1b7-00122d252000",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3.4. Create Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1d6e4141-cea5-4d95-871c-5d3b1be082ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:47:51.738531Z",
     "iopub.status.busy": "2023-03-22T01:47:51.738191Z",
     "iopub.status.idle": "2023-03-22T01:47:57.672144Z",
     "shell.execute_reply": "2023-03-22T01:47:57.671200Z",
     "shell.execute_reply.started": "2023-03-22T01:47:51.738499Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%sh\n",
    "aws s3 rm s3://${S3_BUCKET}/${TABLE_NAME} --recursive &>/dev/null\n",
    "rm -rf ${WORKSPACE}/${TABLE_NAME}\n",
    "sleep 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "caae37d5-1168-4e68-a8f9-7c3313529971",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:47:57.676097Z",
     "iopub.status.busy": "2023-03-22T01:47:57.675828Z",
     "iopub.status.idle": "2023-03-22T01:47:58.537856Z",
     "shell.execute_reply": "2023-03-22T01:47:58.536845Z",
     "shell.execute_reply.started": "2023-03-22T01:47:57.676071Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dde5640d22014c3692345174b0ff67e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30ab4a6802684531a7ac1426601223cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f0c8e2d7-ff10-4082-b71e-7268359d3574",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:47:58.539595Z",
     "iopub.status.busy": "2023-03-22T01:47:58.539265Z",
     "iopub.status.idle": "2023-03-22T01:47:59.361519Z",
     "shell.execute_reply": "2023-03-22T01:47:59.360668Z",
     "shell.execute_reply.started": "2023-03-22T01:47:58.539560Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88399d5db4fb4a52ada042d1e937683d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7a5f5ab3ed7484a9c73a7d7af8049cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}_ro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3beb5c35-6be9-478d-aad6-ef1caa5f7430",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:47:59.363144Z",
     "iopub.status.busy": "2023-03-22T01:47:59.362821Z",
     "iopub.status.idle": "2023-03-22T01:48:00.190535Z",
     "shell.execute_reply": "2023-03-22T01:48:00.189739Z",
     "shell.execute_reply.started": "2023-03-22T01:47:59.363110Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "223d7680a0da40ad8a317da08902085c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3e6ace63cf141879dbfa0d77fc10adb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}_rt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "39157868-3a23-4309-993f-343d8849ee4b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:48:00.192226Z",
     "iopub.status.busy": "2023-03-22T01:48:00.191898Z",
     "iopub.status.idle": "2023-03-22T01:48:02.520306Z",
     "shell.execute_reply": "2023-03-22T01:48:02.519447Z",
     "shell.execute_reply.started": "2023-03-22T01:48:00.192192Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33cea6c13ffc4ed8ab5ee507cbb31840",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc828fc8dd874cab99989163cacd8b79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "create table if not exists ${TABLE_NAME} (\n",
    "    review_id string, \n",
    "    star_rating long, \n",
    "    review_body string, \n",
    "    review_date date, \n",
    "    year long,\n",
    "    timestamp long,\n",
    "    parity int\n",
    ")\n",
    "using hudi\n",
    "location 's3://${S3_BUCKET}/${TABLE_NAME}'\n",
    "partitioned by (parity)\n",
    "options ( \n",
    "    type = 'mor',  \n",
    "    primaryKey = 'review_id', \n",
    "    preCombineField = 'timestamp',\n",
    "    hoodie.compact.inline = 'false',\n",
    "    hoodie.compact.schedule.inline = 'false',\n",
    "    hoodie.compact.inline.max.delta.commits = '3',\n",
    "    hoodie.copyonwrite.record.size.estimate = '175'\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9245a1f4-bb6b-4866-8e92-1215b1078019",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3.5. Insert 96MB / +1 Base File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8ef4ae94-304a-4b11-911c-39a7bf3eb009",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:48:02.521497Z",
     "iopub.status.busy": "2023-03-22T01:48:02.521252Z",
     "iopub.status.idle": "2023-03-22T01:48:47.972886Z",
     "shell.execute_reply": "2023-03-22T01:48:47.972058Z",
     "shell.execute_reply.started": "2023-03-22T01:48:02.521473Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9392697d8d254f359e7105e3ac2f3b92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e1993e7744849fa849fa4e5184e4365",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "insert into \n",
    "    ${TABLE_NAME}\n",
    "select \n",
    "    review_id, \n",
    "    star_rating, \n",
    "    review_body, \n",
    "    review_date, \n",
    "    year,\n",
    "    unix_timestamp(current_timestamp()) as timestamp,\n",
    "    mod(crc32(review_id), 2) as parity\n",
    "from\n",
    "    reviews\n",
    "where\n",
    "    year = 2003"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "9bd9e397-d36b-4c13-a100-412071fbbe58",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:48:47.974308Z",
     "iopub.status.busy": "2023-03-22T01:48:47.973893Z",
     "iopub.status.idle": "2023-03-22T01:49:10.064978Z",
     "shell.execute_reply": "2023-03-22T01:49:10.064127Z",
     "shell.execute_reply.started": "2023-03-22T01:48:47.974281Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322014803020 │ deltacommit │ COMPLETED │ 03-22 01:48 │ 03-22 01:48 │ 03-22 01:48 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322014803020 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╧═══════╧═══════════════════════════════╣\n",
      "║ (empty)                                                         ║\n",
      "╚═════════════════════════════════════════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_2\n",
      "├── [ 96M 01:49:09]  parity=0\n",
      "│   └── [ 96M 01:48:36]  dd6f3233-2492-4a14-8503-b42595939064-0_0-207-1427_20230322014803020.parquet\n",
      "└── [ 96M 01:49:09]  parity=1\n",
      "    └── [ 96M 01:48:36]  982fce48-e710-43a6-82cd-746332e6c7f4-0_1-207-1428_20230322014803020.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32af67bd-efd3-4b8e-9aed-c204fedc77c8",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3.6. Update 787KB / +1 Log File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a19c0c56-defc-4054-9104-943f2d10b1a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:49:10.066517Z",
     "iopub.status.busy": "2023-03-22T01:49:10.066258Z",
     "iopub.status.idle": "2023-03-22T01:49:27.458223Z",
     "shell.execute_reply": "2023-03-22T01:49:27.457404Z",
     "shell.execute_reply.started": "2023-03-22T01:49:10.066492Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "748285cebde64809a76569e23b3bcd5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a7754bab3b8498fb84613fb6e6de63d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "update\n",
    "    ${TABLE_NAME}\n",
    "set             \n",
    "    review_body = concat(uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid()),\n",
    "    timestamp = unix_timestamp(current_timestamp())\n",
    "where\n",
    "    review_date = '2003-01-01'\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "6f13aace-5dfa-456a-9d7b-6b3d0d08f09c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:49:27.459874Z",
     "iopub.status.busy": "2023-03-22T01:49:27.459518Z",
     "iopub.status.idle": "2023-03-22T01:49:49.494796Z",
     "shell.execute_reply": "2023-03-22T01:49:49.493939Z",
     "shell.execute_reply.started": "2023-03-22T01:49:27.459837Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322014803020 │ deltacommit │ COMPLETED │ 03-22 01:48 │ 03-22 01:48 │ 03-22 01:48 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322014910700 │ deltacommit │ COMPLETED │ 03-22 01:49 │ 03-22 01:49 │ 03-22 01:49 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322014910700 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014803020 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╧═══════╧═══════════════════════════════╣\n",
      "║ (empty)                                                         ║\n",
      "╚═════════════════════════════════════════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_2\n",
      "├── [ 97M 01:49:49]  parity=0\n",
      "│   ├── [788K 01:49:19]  .dd6f3233-2492-4a14-8503-b42595939064-0_20230322014803020.log.1_0-255-1488\n",
      "│   └── [ 96M 01:48:36]  dd6f3233-2492-4a14-8503-b42595939064-0_0-207-1427_20230322014803020.parquet\n",
      "└── [ 97M 01:49:49]  parity=1\n",
      "    ├── [814K 01:49:19]  .982fce48-e710-43a6-82cd-746332e6c7f4-0_20230322014803020.log.1_1-255-1489\n",
      "    └── [ 96M 01:48:36]  982fce48-e710-43a6-82cd-746332e6c7f4-0_1-207-1428_20230322014803020.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e08fab1-1da7-4d56-9105-1f1f198b94e3",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3.7. Update 1.2MB / +1 Log File +1 Base File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "fb41bb13-86c9-476a-abdc-4e680da9137a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:49:49.496710Z",
     "iopub.status.busy": "2023-03-22T01:49:49.496314Z",
     "iopub.status.idle": "2023-03-22T01:50:08.879713Z",
     "shell.execute_reply": "2023-03-22T01:50:08.878867Z",
     "shell.execute_reply.started": "2023-03-22T01:49:49.496671Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12c1dac013a34b968b8695cf43233547",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c30ac8f91c694732af7414e50cb6331e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "update\n",
    "    ${TABLE_NAME}\n",
    "set             \n",
    "    review_body = concat(uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid()),\n",
    "    timestamp = unix_timestamp(current_timestamp())\n",
    "where\n",
    "    review_date = '2003-01-02'\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "cba4b634-6c5a-4399-a7fa-b2d3a1ea7779",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:50:08.880972Z",
     "iopub.status.busy": "2023-03-22T01:50:08.880754Z",
     "iopub.status.idle": "2023-03-22T01:50:30.480061Z",
     "shell.execute_reply": "2023-03-22T01:50:30.479039Z",
     "shell.execute_reply.started": "2023-03-22T01:50:08.880951Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322014803020 │ deltacommit │ COMPLETED │ 03-22 01:48 │ 03-22 01:48 │ 03-22 01:48 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322014910700 │ deltacommit │ COMPLETED │ 03-22 01:49 │ 03-22 01:49 │ 03-22 01:49 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 2   │ 20230322014950132 │ deltacommit │ COMPLETED │ 03-22 01:49 │ 03-22 01:49 │ 03-22 01:50 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322014950132 │ 2.3 MB              │ 0                 │ 2                   │ 2                        │ 4341                  │ 4341                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014910700 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014803020 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╧═══════╧═══════════════════════════════╣\n",
      "║ (empty)                                                         ║\n",
      "╚═════════════════════════════════════════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_2\n",
      "├── [ 98M 01:50:30]  parity=0\n",
      "│   ├── [788K 01:49:19]  .dd6f3233-2492-4a14-8503-b42595939064-0_20230322014803020.log.1_0-255-1488\n",
      "│   ├── [1.2M 01:49:59]  .dd6f3233-2492-4a14-8503-b42595939064-0_20230322014803020.log.2_0-303-1549\n",
      "│   └── [ 96M 01:48:36]  dd6f3233-2492-4a14-8503-b42595939064-0_0-207-1427_20230322014803020.parquet\n",
      "└── [ 98M 01:50:30]  parity=1\n",
      "    ├── [814K 01:49:19]  .982fce48-e710-43a6-82cd-746332e6c7f4-0_20230322014803020.log.1_1-255-1489\n",
      "    ├── [1.1M 01:49:59]  .982fce48-e710-43a6-82cd-746332e6c7f4-0_20230322014803020.log.2_1-303-1550\n",
      "    └── [ 96M 01:48:36]  982fce48-e710-43a6-82cd-746332e6c7f4-0_1-207-1428_20230322014803020.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67fedbb9-282c-4a31-8c45-92c606c0d1e8",
   "metadata": {},
   "source": [
    "### 3.8. Offline Schedule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "d1a86eab-96d4-41ae-8d5c-fde85bceec28",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:50:30.481593Z",
     "iopub.status.busy": "2023-03-22T01:50:30.481294Z",
     "iopub.status.idle": "2023-03-22T01:50:55.706832Z",
     "shell.execute_reply": "2023-03-22T01:50:55.705562Z",
     "shell.execute_reply.started": "2023-03-22T01:50:30.481568Z"
    }
   },
   "outputs": [],
   "source": [
    "%%sh\n",
    "# it's required for current user (emr-notebook) to get sudo permission\n",
    "sudo -u hadoop spark-submit \\\n",
    "  --jars '/usr/lib/hudi/hudi-spark-bundle.jar' \\\n",
    "  --class 'org.apache.hudi.utilities.HoodieCompactor' \\\n",
    "  /usr/lib/hudi/hudi-utilities-bundle.jar \\\n",
    "  --spark-memory '4g' \\\n",
    "  --mode 'schedule' \\\n",
    "  --base-path \"s3://${S3_BUCKET}/${TABLE_NAME}\" \\\n",
    "  --table-name \"$TABLE_NAME\" \\\n",
    "  --hoodie-conf \"hoodie.compact.inline.max.delta.commits=3\" > ${WORKSPACE}/${TABLE_NAME}.schedule.out &>/dev/null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "5a37632a-a3ad-4a04-8ac2-6358dc87ca20",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:50:55.708545Z",
     "iopub.status.busy": "2023-03-22T01:50:55.708291Z",
     "iopub.status.idle": "2023-03-22T01:51:17.484625Z",
     "shell.execute_reply": "2023-03-22T01:51:17.483808Z",
     "shell.execute_reply.started": "2023-03-22T01:50:55.708520Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322014803020 │ deltacommit │ COMPLETED │ 03-22 01:48 │ 03-22 01:48 │ 03-22 01:48 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322014910700 │ deltacommit │ COMPLETED │ 03-22 01:49 │ 03-22 01:49 │ 03-22 01:49 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 2   │ 20230322014950132 │ deltacommit │ COMPLETED │ 03-22 01:49 │ 03-22 01:49 │ 03-22 01:50 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 3   │ 20230322015047702 │ compaction  │ REQUESTED │ 03-22 01:50 │ -           │ -           ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322014950132 │ 2.3 MB              │ 0                 │ 2                   │ 2                        │ 4341                  │ 4341                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014910700 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014803020 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State     │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╪═══════════╪═══════════════════════════════╣\n",
      "║ 20230322015047702       │ REQUESTED │ 2                             ║\n",
      "╚═════════════════════════╧═══════════╧═══════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_2\n",
      "├── [ 98M 01:50:30]  parity=0\n",
      "│   ├── [788K 01:49:19]  .dd6f3233-2492-4a14-8503-b42595939064-0_20230322014803020.log.1_0-255-1488\n",
      "│   ├── [1.2M 01:49:59]  .dd6f3233-2492-4a14-8503-b42595939064-0_20230322014803020.log.2_0-303-1549\n",
      "│   └── [ 96M 01:48:36]  dd6f3233-2492-4a14-8503-b42595939064-0_0-207-1427_20230322014803020.parquet\n",
      "└── [ 98M 01:50:30]  parity=1\n",
      "    ├── [814K 01:49:19]  .982fce48-e710-43a6-82cd-746332e6c7f4-0_20230322014803020.log.1_1-255-1489\n",
      "    ├── [1.1M 01:49:59]  .982fce48-e710-43a6-82cd-746332e6c7f4-0_20230322014803020.log.2_1-303-1550\n",
      "    └── [ 96M 01:48:36]  982fce48-e710-43a6-82cd-746332e6c7f4-0_1-207-1428_20230322014803020.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "767651e7-0f4c-496f-9f26-83be7f2b2d1e",
   "metadata": {},
   "source": [
    "### 3.9. Offline Execute 96.15MB / +1 Compacted File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "339cad2f-0e58-4439-bc68-b6cdda8ccd5f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:51:17.486163Z",
     "iopub.status.busy": "2023-03-22T01:51:17.485914Z",
     "iopub.status.idle": "2023-03-22T01:52:07.527185Z",
     "shell.execute_reply": "2023-03-22T01:52:07.526342Z",
     "shell.execute_reply.started": "2023-03-22T01:51:17.486139Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%sh\n",
    "# it's required for current user (emr-notebook) to get sudo permission\n",
    "sudo -u hadoop spark-submit \\\n",
    "  --jars '/usr/lib/hudi/hudi-spark-bundle.jar' \\\n",
    "  --class \"org.apache.hudi.utilities.HoodieCompactor\" \\\n",
    "  /usr/lib/hudi/hudi-utilities-bundle.jar \\\n",
    "  --spark-memory '4g' \\\n",
    "  --mode 'execute' \\\n",
    "  --base-path \"s3://${S3_BUCKET}/${TABLE_NAME}\" \\\n",
    "  --table-name \"$TABLE_NAME\" > ${WORKSPACE}/${TABLE_NAME}.execute.out &>/dev/null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f7f3db57-9035-4c93-a5a5-0df1c810cafe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:52:07.528673Z",
     "iopub.status.busy": "2023-03-22T01:52:07.528434Z",
     "iopub.status.idle": "2023-03-22T01:52:31.359383Z",
     "shell.execute_reply": "2023-03-22T01:52:31.358562Z",
     "shell.execute_reply.started": "2023-03-22T01:52:07.528649Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322014803020 │ deltacommit │ COMPLETED │ 03-22 01:48 │ 03-22 01:48 │ 03-22 01:48 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322014910700 │ deltacommit │ COMPLETED │ 03-22 01:49 │ 03-22 01:49 │ 03-22 01:49 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 2   │ 20230322014950132 │ deltacommit │ COMPLETED │ 03-22 01:49 │ 03-22 01:49 │ 03-22 01:50 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 3   │ 20230322015047702 │ commit      │ COMPLETED │ 03-22 01:50 │ 03-22 01:51 │ 03-22 01:52 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322015047702 │ 192.3 MB            │ 0                 │ 2                   │ 2                        │ 1155127               │ 7292                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014950132 │ 2.3 MB              │ 0                 │ 2                   │ 2                        │ 4341                  │ 4341                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014910700 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322014803020 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State     │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╪═══════════╪═══════════════════════════════╣\n",
      "║ 20230322015047702       │ COMPLETED │ 2                             ║\n",
      "╚═════════════════════════╧═══════════╧═══════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_2\n",
      "├── [194M 01:52:31]  parity=0\n",
      "│   ├── [788K 01:49:19]  .dd6f3233-2492-4a14-8503-b42595939064-0_20230322014803020.log.1_0-255-1488\n",
      "│   ├── [1.2M 01:49:59]  .dd6f3233-2492-4a14-8503-b42595939064-0_20230322014803020.log.2_0-303-1549\n",
      "│   ├── [ 96M 01:52:01]  dd6f3233-2492-4a14-8503-b42595939064-0_0-0-0_20230322015047702.parquet\n",
      "│   └── [ 96M 01:48:36]  dd6f3233-2492-4a14-8503-b42595939064-0_0-207-1427_20230322014803020.parquet\n",
      "└── [194M 01:52:31]  parity=1\n",
      "    ├── [814K 01:49:19]  .982fce48-e710-43a6-82cd-746332e6c7f4-0_20230322014803020.log.1_1-255-1489\n",
      "    ├── [1.1M 01:49:59]  .982fce48-e710-43a6-82cd-746332e6c7f4-0_20230322014803020.log.2_1-303-1550\n",
      "    ├── [ 96M 01:52:01]  982fce48-e710-43a6-82cd-746332e6c7f4-0_1-0-1_20230322015047702.parquet\n",
      "    └── [ 96M 01:48:36]  982fce48-e710-43a6-82cd-746332e6c7f4-0_1-207-1428_20230322014803020.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "353ea002-8fc0-4bef-a1a5-0360ba9a0c80",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 4. Test Case 3 - Async Compaction ( Offline Schedule + Offline Execute )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bb042ba-800a-4951-8897-8782bbc31635",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 4.1. Test Plan\n",
    "\n",
    "Step No.|Action|Volume Per Partition |Storage\n",
    ":--------:|:------|:------|:----------\n",
    "1|Insert|96MB|+1 Base File\n",
    "2|Update|804KB|+1 Log File\n",
    "3|Update|1.2MB|+1 Log File\n",
    "4|Offline Schedule + Execute|96.15MB|+1 Compacted Base File"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28c6ce94-5a9f-4498-93f5-e353c6205aaa",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 4.2. Key Settings\n",
    "\n",
    "KEY|DEFAULT VALUE|SET VALUE\n",
    ":---|:---|:---\n",
    "hoodie.compact.inline|false|true\n",
    "hoodie.compact.schedule.inline|false|false\n",
    "hoodie.compact.inline.max.delta.commits|5|3\n",
    "hoodie.copyonwrite.record.size.estimate|1024|175"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cb1ef91-2a5f-4ddc-a041-a86238b58b72",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-25T02:57:45.037101Z",
     "iopub.status.busy": "2023-02-25T02:57:45.036681Z",
     "iopub.status.idle": "2023-02-25T02:58:07.353493Z",
     "shell.execute_reply": "2023-02-25T02:58:07.352596Z",
     "shell.execute_reply.started": "2023-02-25T02:57:45.037060Z"
    },
    "tags": []
   },
   "source": [
    "### 4.3. Set Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "aad6edd0-bb39-43d1-805c-fc4aa71e97af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:52:31.360866Z",
     "iopub.status.busy": "2023-03-22T01:52:31.360615Z",
     "iopub.status.idle": "2023-03-22T01:52:38.883627Z",
     "shell.execute_reply": "2023-03-22T01:52:38.882774Z",
     "shell.execute_reply.started": "2023-03-22T01:52:31.360841Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c0c664fe18b4717950f5f22c4af0526",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(HTML(value='Type:'), Button(description='Table', layout=Layout(width='70px'), st…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9eaef62278014fda9c0337d54dafa5dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "set TABLE_NAME=reviews_mor_compaction_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "6520d214-4743-4fad-91cc-eafaa04875eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:52:38.885894Z",
     "iopub.status.busy": "2023-03-22T01:52:38.885524Z",
     "iopub.status.idle": "2023-03-22T01:52:38.892630Z",
     "shell.execute_reply": "2023-03-22T01:52:38.891738Z",
     "shell.execute_reply.started": "2023-03-22T01:52:38.885857Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: TABLE_NAME=reviews_mor_compaction_3\n"
     ]
    }
   ],
   "source": [
    "%env TABLE_NAME=reviews_mor_compaction_3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6303f234-9f98-4f11-967f-206bef13ea59",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 4.4. Create Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a3a837bf-8ad3-4aab-aa89-8542d3c1a1b8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:52:38.894041Z",
     "iopub.status.busy": "2023-03-22T01:52:38.893692Z",
     "iopub.status.idle": "2023-03-22T01:52:45.013800Z",
     "shell.execute_reply": "2023-03-22T01:52:45.012880Z",
     "shell.execute_reply.started": "2023-03-22T01:52:38.894007Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%sh\n",
    "aws s3 rm s3://${S3_BUCKET}/${TABLE_NAME} --recursive &>/dev/null\n",
    "rm -rf ${WORKSPACE}/${TABLE_NAME}\n",
    "sleep 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "c8570ddc-7e9e-4847-a027-939619998fbc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:52:45.015540Z",
     "iopub.status.busy": "2023-03-22T01:52:45.015184Z",
     "iopub.status.idle": "2023-03-22T01:52:45.892837Z",
     "shell.execute_reply": "2023-03-22T01:52:45.891980Z",
     "shell.execute_reply.started": "2023-03-22T01:52:45.015501Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c459d43f141443688da6fe871b261fd9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c327b818503e4a3c99fd79fbe189282c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "b9efb90f-c9fa-4c2a-b553-03ff864a79b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:52:45.894856Z",
     "iopub.status.busy": "2023-03-22T01:52:45.894253Z",
     "iopub.status.idle": "2023-03-22T01:52:46.759964Z",
     "shell.execute_reply": "2023-03-22T01:52:46.758891Z",
     "shell.execute_reply.started": "2023-03-22T01:52:45.894828Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4e174b434014bb28562837125a3308c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a71da18da1546d8a41c6dea90270762",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}_ro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "80e60f4e-57fe-4a53-a748-55961dbb0c23",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:52:46.761740Z",
     "iopub.status.busy": "2023-03-22T01:52:46.761408Z",
     "iopub.status.idle": "2023-03-22T01:52:47.589342Z",
     "shell.execute_reply": "2023-03-22T01:52:47.588509Z",
     "shell.execute_reply.started": "2023-03-22T01:52:46.761705Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b81e35a171f48afbebbe541455562cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02bf3f62eb884a718a3b087fbfadacbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}_rt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "7b5b247a-7796-4633-879c-06801e269992",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:52:47.590633Z",
     "iopub.status.busy": "2023-03-22T01:52:47.590383Z",
     "iopub.status.idle": "2023-03-22T01:52:49.936914Z",
     "shell.execute_reply": "2023-03-22T01:52:49.936074Z",
     "shell.execute_reply.started": "2023-03-22T01:52:47.590610Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d60e3452b13f4dcbbf6208a21c79914f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e4e94e4e2e648969cd2eda99196d5d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "create table if not exists ${TABLE_NAME} (\n",
    "    review_id string, \n",
    "    star_rating long, \n",
    "    review_body string, \n",
    "    review_date date, \n",
    "    year long,\n",
    "    timestamp long,\n",
    "    parity int\n",
    ")\n",
    "using hudi\n",
    "location 's3://${S3_BUCKET}/${TABLE_NAME}'\n",
    "partitioned by (parity)\n",
    "options ( \n",
    "    type = 'mor',  \n",
    "    primaryKey = 'review_id', \n",
    "    preCombineField = 'timestamp',\n",
    "    hoodie.compact.inline = 'false',\n",
    "    hoodie.compact.schedule.inline = 'false',\n",
    "    hoodie.compact.inline.max.delta.commits = '3',\n",
    "    hoodie.copyonwrite.record.size.estimate = '175'\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "102b9479-86ee-464f-a053-760031dbca07",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 4.5. Insert 96MB / +1 Base File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "abf83a33-35fc-4d38-af3d-9a63750cf704",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:52:49.938583Z",
     "iopub.status.busy": "2023-03-22T01:52:49.938249Z",
     "iopub.status.idle": "2023-03-22T01:53:51.450347Z",
     "shell.execute_reply": "2023-03-22T01:53:51.449436Z",
     "shell.execute_reply.started": "2023-03-22T01:52:49.938547Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6a6d46717024b1c9dc0bf599f6be99b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77632c208f0d42b7b37963e8776324f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "insert into \n",
    "    ${TABLE_NAME}\n",
    "select \n",
    "    review_id, \n",
    "    star_rating, \n",
    "    review_body, \n",
    "    review_date, \n",
    "    year,\n",
    "    unix_timestamp(current_timestamp()) as timestamp,\n",
    "    mod(crc32(review_id), 2) as parity\n",
    "from\n",
    "    reviews\n",
    "where\n",
    "    year = 2003"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "12087bf4-d1d2-451d-919f-b42fa59bfc81",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:53:51.452148Z",
     "iopub.status.busy": "2023-03-22T01:53:51.451716Z",
     "iopub.status.idle": "2023-03-22T01:54:13.521025Z",
     "shell.execute_reply": "2023-03-22T01:54:13.520265Z",
     "shell.execute_reply.started": "2023-03-22T01:53:51.452112Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322015250427 │ deltacommit │ COMPLETED │ 03-22 01:52 │ 03-22 01:53 │ 03-22 01:53 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322015250427 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╧═══════╧═══════════════════════════════╣\n",
      "║ (empty)                                                         ║\n",
      "╚═════════════════════════════════════════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_3\n",
      "├── [ 96M 01:54:13]  parity=0\n",
      "│   └── [ 96M 01:53:37]  96d24bb8-dedb-45ae-9878-b09a84597ba6-0_0-365-1696_20230322015250427.parquet\n",
      "└── [ 96M 01:54:13]  parity=1\n",
      "    └── [ 96M 01:53:37]  71c5c98d-c52a-4be6-8f00-c1db26911d37-0_1-365-1697_20230322015250427.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c70b5692-da44-461b-a765-67a9c9660d02",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 4.6. Update 787KB / +1 Log File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "909db704-a684-4c60-968b-3d9c2ed8cb50",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:54:13.522549Z",
     "iopub.status.busy": "2023-03-22T01:54:13.522294Z",
     "iopub.status.idle": "2023-03-22T01:54:32.904607Z",
     "shell.execute_reply": "2023-03-22T01:54:32.903952Z",
     "shell.execute_reply.started": "2023-03-22T01:54:13.522516Z"
    },
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fd384acdf4149a3af1c32c8f71fafde",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2fa757c25fc43e7a913598435c386e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "update\n",
    "    ${TABLE_NAME}\n",
    "set             \n",
    "    review_body = concat(uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid()),\n",
    "    timestamp = unix_timestamp(current_timestamp())\n",
    "where\n",
    "    review_date = '2003-01-01'\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "06c66e94-bd3a-40a7-a632-daa8cbaccd3c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:54:32.905768Z",
     "iopub.status.busy": "2023-03-22T01:54:32.905526Z",
     "iopub.status.idle": "2023-03-22T01:54:54.766087Z",
     "shell.execute_reply": "2023-03-22T01:54:54.765066Z",
     "shell.execute_reply.started": "2023-03-22T01:54:32.905728Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322015250427 │ deltacommit │ COMPLETED │ 03-22 01:52 │ 03-22 01:53 │ 03-22 01:53 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322015414076 │ deltacommit │ COMPLETED │ 03-22 01:54 │ 03-22 01:54 │ 03-22 01:54 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322015414076 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015250427 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╧═══════╧═══════════════════════════════╣\n",
      "║ (empty)                                                         ║\n",
      "╚═════════════════════════════════════════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_3\n",
      "├── [ 97M 01:54:54]  parity=0\n",
      "│   ├── [787K 01:54:25]  .96d24bb8-dedb-45ae-9878-b09a84597ba6-0_20230322015250427.log.1_0-413-1758\n",
      "│   └── [ 96M 01:53:37]  96d24bb8-dedb-45ae-9878-b09a84597ba6-0_0-365-1696_20230322015250427.parquet\n",
      "└── [ 97M 01:54:54]  parity=1\n",
      "    ├── [812K 01:54:25]  .71c5c98d-c52a-4be6-8f00-c1db26911d37-0_20230322015250427.log.1_1-413-1759\n",
      "    └── [ 96M 01:53:37]  71c5c98d-c52a-4be6-8f00-c1db26911d37-0_1-365-1697_20230322015250427.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cddaeaad-e2a7-4c8d-a801-a19ee8a12407",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 4.7. Update 1.2MB / +1 Log File +1 Base File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ca24f073-125a-42d2-bdbd-89a50c0e2cf3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:54:54.767705Z",
     "iopub.status.busy": "2023-03-22T01:54:54.767439Z",
     "iopub.status.idle": "2023-03-22T01:55:14.154753Z",
     "shell.execute_reply": "2023-03-22T01:55:14.154074Z",
     "shell.execute_reply.started": "2023-03-22T01:54:54.767680Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7c2738f7f4c44a2ad6f6a688dabbb2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6f19384ed0441858c9ab575f93003d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "update\n",
    "    ${TABLE_NAME}\n",
    "set             \n",
    "    review_body = concat(uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid()),\n",
    "    timestamp = unix_timestamp(current_timestamp())\n",
    "where\n",
    "    review_date = '2003-01-02'\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "42abe1d9-b86c-46dc-b24f-7656003239c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:55:14.155950Z",
     "iopub.status.busy": "2023-03-22T01:55:14.155693Z",
     "iopub.status.idle": "2023-03-22T01:55:35.993042Z",
     "shell.execute_reply": "2023-03-22T01:55:35.992066Z",
     "shell.execute_reply.started": "2023-03-22T01:55:14.155927Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322015250427 │ deltacommit │ COMPLETED │ 03-22 01:52 │ 03-22 01:53 │ 03-22 01:53 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322015414076 │ deltacommit │ COMPLETED │ 03-22 01:54 │ 03-22 01:54 │ 03-22 01:54 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 2   │ 20230322015455449 │ deltacommit │ COMPLETED │ 03-22 01:54 │ 03-22 01:55 │ 03-22 01:55 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322015455449 │ 2.3 MB              │ 0                 │ 2                   │ 2                        │ 4341                  │ 4341                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015414076 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015250427 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╧═══════╧═══════════════════════════════╣\n",
      "║ (empty)                                                         ║\n",
      "╚═════════════════════════════════════════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_3\n",
      "├── [ 98M 01:55:35]  parity=0\n",
      "│   ├── [787K 01:54:25]  .96d24bb8-dedb-45ae-9878-b09a84597ba6-0_20230322015250427.log.1_0-413-1758\n",
      "│   ├── [1.2M 01:55:05]  .96d24bb8-dedb-45ae-9878-b09a84597ba6-0_20230322015250427.log.2_0-461-1819\n",
      "│   └── [ 96M 01:53:37]  96d24bb8-dedb-45ae-9878-b09a84597ba6-0_0-365-1696_20230322015250427.parquet\n",
      "└── [ 98M 01:55:35]  parity=1\n",
      "    ├── [812K 01:54:25]  .71c5c98d-c52a-4be6-8f00-c1db26911d37-0_20230322015250427.log.1_1-413-1759\n",
      "    ├── [1.1M 01:55:05]  .71c5c98d-c52a-4be6-8f00-c1db26911d37-0_20230322015250427.log.2_1-461-1820\n",
      "    └── [ 96M 01:53:37]  71c5c98d-c52a-4be6-8f00-c1db26911d37-0_1-365-1697_20230322015250427.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a7811a8-7495-4e4c-87e2-6d5e43d02f1f",
   "metadata": {},
   "source": [
    "### 4.8. Offline Schedule + Execute 96.15MB / +1 Compacted File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "4c3c64ce-866f-4740-986c-27f9d4004f42",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:55:35.994576Z",
     "iopub.status.busy": "2023-03-22T01:55:35.994322Z",
     "iopub.status.idle": "2023-03-22T01:56:29.691160Z",
     "shell.execute_reply": "2023-03-22T01:56:29.690194Z",
     "shell.execute_reply.started": "2023-03-22T01:55:35.994552Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%sh\n",
    "# it's required for current user (emr-notebook) to get sudo permission\n",
    "sudo -u hadoop spark-submit \\\n",
    "  --jars '/usr/lib/hudi/hudi-spark-bundle.jar' \\\n",
    "  --class \"org.apache.hudi.utilities.HoodieCompactor\" \\\n",
    "  /usr/lib/hudi/hudi-utilities-bundle.jar \\\n",
    "  --spark-memory '4g' \\\n",
    "  --mode 'scheduleAndExecute' \\\n",
    "  --base-path \"s3://${S3_BUCKET}/${TABLE_NAME}\" \\\n",
    "  --table-name \"$TABLE_NAME\" \\\n",
    "  --hoodie-conf \"hoodie.compact.inline.max.delta.commits=3\" > ${WORKSPACE}/${TABLE_NAME}.execute.out &>/dev/null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "9560755c-6a0a-48ee-8cc0-ff4b3d346ea3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:56:29.693002Z",
     "iopub.status.busy": "2023-03-22T01:56:29.692649Z",
     "iopub.status.idle": "2023-03-22T01:56:53.010934Z",
     "shell.execute_reply": "2023-03-22T01:56:53.010000Z",
     "shell.execute_reply.started": "2023-03-22T01:56:29.692964Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322015250427 │ deltacommit │ COMPLETED │ 03-22 01:52 │ 03-22 01:53 │ 03-22 01:53 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322015414076 │ deltacommit │ COMPLETED │ 03-22 01:54 │ 03-22 01:54 │ 03-22 01:54 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 2   │ 20230322015455449 │ deltacommit │ COMPLETED │ 03-22 01:54 │ 03-22 01:55 │ 03-22 01:55 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 3   │ 20230322015552582 │ commit      │ COMPLETED │ 03-22 01:56 │ 03-22 01:56 │ 03-22 01:56 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322015552582 │ 192.3 MB            │ 0                 │ 2                   │ 2                        │ 1155127               │ 7292                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015455449 │ 2.3 MB              │ 0                 │ 2                   │ 2                        │ 4341                  │ 4341                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015414076 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015250427 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State     │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╪═══════════╪═══════════════════════════════╣\n",
      "║ 20230322015552582       │ COMPLETED │ 2                             ║\n",
      "╚═════════════════════════╧═══════════╧═══════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_3\n",
      "├── [194M 01:56:52]  parity=0\n",
      "│   ├── [787K 01:54:25]  .96d24bb8-dedb-45ae-9878-b09a84597ba6-0_20230322015250427.log.1_0-413-1758\n",
      "│   ├── [1.2M 01:55:05]  .96d24bb8-dedb-45ae-9878-b09a84597ba6-0_20230322015250427.log.2_0-461-1819\n",
      "│   ├── [ 96M 01:53:37]  96d24bb8-dedb-45ae-9878-b09a84597ba6-0_0-365-1696_20230322015250427.parquet\n",
      "│   └── [ 96M 01:56:23]  96d24bb8-dedb-45ae-9878-b09a84597ba6-0_1-1-3_20230322015552582.parquet\n",
      "└── [194M 01:56:52]  parity=1\n",
      "    ├── [812K 01:54:25]  .71c5c98d-c52a-4be6-8f00-c1db26911d37-0_20230322015250427.log.1_1-413-1759\n",
      "    ├── [1.1M 01:55:05]  .71c5c98d-c52a-4be6-8f00-c1db26911d37-0_20230322015250427.log.2_1-461-1820\n",
      "    ├── [ 96M 01:56:24]  71c5c98d-c52a-4be6-8f00-c1db26911d37-0_0-1-2_20230322015552582.parquet\n",
      "    └── [ 96M 01:53:37]  71c5c98d-c52a-4be6-8f00-c1db26911d37-0_1-365-1697_20230322015250427.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55444eec-8bca-40a6-9e55-93b01a62b14b",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 5. Test Case 4 - Semi Async Compaction ( Inline Schedule, Offline Execute )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f23e94a0-c051-48ef-bbc8-ac490daec733",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 5.1. Test Plan\n",
    "\n",
    "Step No.|Action|Volume Per Partition |Storage\n",
    ":--------:|:------|:------|:----------\n",
    "1|Insert|96MB|+1 Base File\n",
    "2|Update|788KB|+1 Log File\n",
    "3|Update|1.2MB|+1 Log File\n",
    "4|Offline Execute|96.15MB|+1 Compacted Base File"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca054749-97a3-4d76-ab1f-97c14628eb45",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 5.2. Key Settings\n",
    "\n",
    "KEY|DEFAULT VALUE|SET VALUE\n",
    ":---|:---|:---\n",
    "hoodie.compact.inline|false|true\n",
    "hoodie.compact.schedule.inline|false|false\n",
    "hoodie.compact.inline.max.delta.commits|5|3\n",
    "hoodie.copyonwrite.record.size.estimate|1024|175"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "958f85aa-bf3f-4c94-9ab2-a8e6826fa07d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-25T02:57:45.037101Z",
     "iopub.status.busy": "2023-02-25T02:57:45.036681Z",
     "iopub.status.idle": "2023-02-25T02:58:07.353493Z",
     "shell.execute_reply": "2023-02-25T02:58:07.352596Z",
     "shell.execute_reply.started": "2023-02-25T02:57:45.037060Z"
    },
    "tags": []
   },
   "source": [
    "### 5.3. Set Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "c70795a8-5ad1-498a-9c77-1d22804448e3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:56:53.012470Z",
     "iopub.status.busy": "2023-03-22T01:56:53.012213Z",
     "iopub.status.idle": "2023-03-22T01:57:00.522909Z",
     "shell.execute_reply": "2023-03-22T01:57:00.521901Z",
     "shell.execute_reply.started": "2023-03-22T01:56:53.012444Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22ae16c5a13e4a249c66c9827f6538c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(HTML(value='Type:'), Button(description='Table', layout=Layout(width='70px'), st…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec9f813233a449268992d0719f43120a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "set TABLE_NAME=reviews_mor_compaction_4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "8924e11f-9d8b-4e98-b1b5-bacae57433cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:57:00.524544Z",
     "iopub.status.busy": "2023-03-22T01:57:00.524242Z",
     "iopub.status.idle": "2023-03-22T01:57:00.530022Z",
     "shell.execute_reply": "2023-03-22T01:57:00.528967Z",
     "shell.execute_reply.started": "2023-03-22T01:57:00.524515Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: TABLE_NAME=reviews_mor_compaction_4\n"
     ]
    }
   ],
   "source": [
    "%env TABLE_NAME=reviews_mor_compaction_4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1fffe57-a602-40f5-aa20-ea1285eeeb7f",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 5.4. Create Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e2064d50-0ca9-4765-8282-e0d2f5fed32a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:57:00.531613Z",
     "iopub.status.busy": "2023-03-22T01:57:00.531284Z",
     "iopub.status.idle": "2023-03-22T01:57:06.071857Z",
     "shell.execute_reply": "2023-03-22T01:57:06.070975Z",
     "shell.execute_reply.started": "2023-03-22T01:57:00.531545Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%sh\n",
    "aws s3 rm s3://${S3_BUCKET}/${TABLE_NAME} --recursive &>/dev/null\n",
    "rm -rf ${WORKSPACE}/${TABLE_NAME}\n",
    "sleep 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "177683eb-3b33-4f22-a549-f8d86c16cd5d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:57:06.073640Z",
     "iopub.status.busy": "2023-03-22T01:57:06.073274Z",
     "iopub.status.idle": "2023-03-22T01:57:06.397821Z",
     "shell.execute_reply": "2023-03-22T01:57:06.397049Z",
     "shell.execute_reply.started": "2023-03-22T01:57:06.073602Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6749bee510443e39327e5f33c1cf8b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fe119d70fd84843afb6379e26b261a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "48de5702-579b-49aa-b29d-69d75ae3b4a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:57:06.399230Z",
     "iopub.status.busy": "2023-03-22T01:57:06.399004Z",
     "iopub.status.idle": "2023-03-22T01:57:06.714163Z",
     "shell.execute_reply": "2023-03-22T01:57:06.713467Z",
     "shell.execute_reply.started": "2023-03-22T01:57:06.399208Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d3acdc82f204f5aa5119d813259c69a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6662abaef844d4cb7ab39aae789f7af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}_ro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "de2a228d-9310-4e93-a5dd-fad37ff10b92",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:57:06.715350Z",
     "iopub.status.busy": "2023-03-22T01:57:06.715127Z",
     "iopub.status.idle": "2023-03-22T01:57:07.036898Z",
     "shell.execute_reply": "2023-03-22T01:57:07.036216Z",
     "shell.execute_reply.started": "2023-03-22T01:57:06.715328Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90462a3b704a4ccf9e818276e0d39634",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "def9f5477eed4808bb93debef68db10d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "drop table if exists ${TABLE_NAME}_rt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "bcbbdfb8-0757-4815-a551-2eed34a41eb8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:57:07.038008Z",
     "iopub.status.busy": "2023-03-22T01:57:07.037801Z",
     "iopub.status.idle": "2023-03-22T01:57:09.364316Z",
     "shell.execute_reply": "2023-03-22T01:57:09.363474Z",
     "shell.execute_reply.started": "2023-03-22T01:57:07.037987Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86ee14b29c0c43f7be44a9e45eecee65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f831df68f264da9bec83004116bb189",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "create table if not exists ${TABLE_NAME} (\n",
    "    review_id string, \n",
    "    star_rating long, \n",
    "    review_body string, \n",
    "    review_date date, \n",
    "    year long,\n",
    "    timestamp long,\n",
    "    parity int\n",
    ")\n",
    "using hudi\n",
    "location 's3://${S3_BUCKET}/${TABLE_NAME}'\n",
    "partitioned by (parity)\n",
    "options ( \n",
    "    type = 'mor',  \n",
    "    primaryKey = 'review_id', \n",
    "    preCombineField = 'timestamp',\n",
    "    hoodie.compact.inline = 'false',\n",
    "    hoodie.compact.schedule.inline = 'true',\n",
    "    hoodie.compact.inline.max.delta.commits = '3',\n",
    "    hoodie.copyonwrite.record.size.estimate = '175'\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e63a3f6-b9ee-4eb9-a14c-4f970aed7325",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 5.5. Insert 96MB / +1 Base File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "d15b6a2d-7e1f-4c6b-95df-4bf6c2fcd3ea",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:57:09.365959Z",
     "iopub.status.busy": "2023-03-22T01:57:09.365636Z",
     "iopub.status.idle": "2023-03-22T01:58:10.974829Z",
     "shell.execute_reply": "2023-03-22T01:58:10.974114Z",
     "shell.execute_reply.started": "2023-03-22T01:57:09.365924Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c24f261a6d5048139f03b2cb1940edc3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10dcc1ba736c4ebcb43d71f43a6bfc97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "insert into \n",
    "    ${TABLE_NAME}\n",
    "select \n",
    "    review_id, \n",
    "    star_rating, \n",
    "    review_body, \n",
    "    review_date, \n",
    "    year,\n",
    "    unix_timestamp(current_timestamp()) as timestamp,\n",
    "    mod(crc32(review_id), 2) as parity\n",
    "from\n",
    "    reviews\n",
    "where\n",
    "    year = 2003"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "e7da5cb3-6e06-4dcb-9540-3ae2f14d9d2e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:58:10.976076Z",
     "iopub.status.busy": "2023-03-22T01:58:10.975846Z",
     "iopub.status.idle": "2023-03-22T01:58:33.396263Z",
     "shell.execute_reply": "2023-03-22T01:58:33.395399Z",
     "shell.execute_reply.started": "2023-03-22T01:58:10.976055Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322015709801 │ deltacommit │ COMPLETED │ 03-22 01:57 │ 03-22 01:57 │ 03-22 01:58 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322015709801 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╧═══════╧═══════════════════════════════╣\n",
      "║ (empty)                                                         ║\n",
      "╚═════════════════════════════════════════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_4\n",
      "├── [ 96M 01:58:33]  parity=0\n",
      "│   └── [ 96M 01:57:56]  885c3cb4-9d5e-4836-b190-68386d2ad1b8-0_0-523-1967_20230322015709801.parquet\n",
      "└── [ 96M 01:58:33]  parity=1\n",
      "    └── [ 96M 01:57:58]  a9a37853-b25f-4dd8-81bb-d712380cf57d-0_1-523-1968_20230322015709801.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e536299d-3816-4a25-8903-20fd252316fe",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 5.6. Update 787KB / +1 Log File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "6de7d86c-c5c0-4a25-a59c-e2aa7f8d309f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:58:33.397708Z",
     "iopub.status.busy": "2023-03-22T01:58:33.397464Z",
     "iopub.status.idle": "2023-03-22T01:58:54.797159Z",
     "shell.execute_reply": "2023-03-22T01:58:54.796486Z",
     "shell.execute_reply.started": "2023-03-22T01:58:33.397684Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af0e83b117fd478891a12b38a246f642",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5820c505681f42c5879d8570a9502359",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "update\n",
    "    ${TABLE_NAME}\n",
    "set             \n",
    "    review_body = concat(uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid()),\n",
    "    timestamp = unix_timestamp(current_timestamp())\n",
    "where\n",
    "    review_date = '2003-01-01'\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "2658943d-5b22-4ccc-b781-9238f3b4ae69",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:58:54.798372Z",
     "iopub.status.busy": "2023-03-22T01:58:54.798152Z",
     "iopub.status.idle": "2023-03-22T01:59:17.210998Z",
     "shell.execute_reply": "2023-03-22T01:59:17.210025Z",
     "shell.execute_reply.started": "2023-03-22T01:58:54.798350Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322015709801 │ deltacommit │ COMPLETED │ 03-22 01:57 │ 03-22 01:57 │ 03-22 01:58 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322015834047 │ deltacommit │ COMPLETED │ 03-22 01:58 │ 03-22 01:58 │ 03-22 01:58 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322015834047 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015709801 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╧═══════╧═══════════════════════════════╣\n",
      "║ (empty)                                                         ║\n",
      "╚═════════════════════════════════════════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_4\n",
      "├── [ 97M 01:59:17]  parity=0\n",
      "│   ├── [787K 01:58:45]  .885c3cb4-9d5e-4836-b190-68386d2ad1b8-0_20230322015709801.log.1_0-571-2029\n",
      "│   └── [ 96M 01:57:56]  885c3cb4-9d5e-4836-b190-68386d2ad1b8-0_0-523-1967_20230322015709801.parquet\n",
      "└── [ 97M 01:59:17]  parity=1\n",
      "    ├── [812K 01:58:45]  .a9a37853-b25f-4dd8-81bb-d712380cf57d-0_20230322015709801.log.1_1-571-2030\n",
      "    └── [ 96M 01:57:58]  a9a37853-b25f-4dd8-81bb-d712380cf57d-0_1-523-1968_20230322015709801.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68b92eb9-a751-4605-bda8-8ec009560fbb",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 5.7. Update 1.2MB / +1 Log File +1 Base File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "86584bca-b9a6-4170-9c7b-e45750bd5515",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:59:17.212935Z",
     "iopub.status.busy": "2023-03-22T01:59:17.212559Z",
     "iopub.status.idle": "2023-03-22T01:59:40.711031Z",
     "shell.execute_reply": "2023-03-22T01:59:40.710218Z",
     "shell.execute_reply.started": "2023-03-22T01:59:17.212896Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.jp-OutputArea-child .jupyter-widgets.jp-OutputArea-output { overflow: auto; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71bbb06b5e944331ac76a749f6a311ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(), EncodingWidget(children=(VBox(children=(HTML(value='Encoding:'), Dropdown(description='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3651aaef42a94a43a302cf305640c897",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%sql\n",
    "update\n",
    "    ${TABLE_NAME}\n",
    "set             \n",
    "    review_body = concat(uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid(),uuid()),\n",
    "    timestamp = unix_timestamp(current_timestamp())\n",
    "where\n",
    "    review_date = '2003-01-02'\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "6111147c-bac6-456b-85f2-849750eaa772",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T01:59:40.712300Z",
     "iopub.status.busy": "2023-03-22T01:59:40.712079Z",
     "iopub.status.idle": "2023-03-22T02:00:03.489224Z",
     "shell.execute_reply": "2023-03-22T02:00:03.488400Z",
     "shell.execute_reply.started": "2023-03-22T01:59:40.712279Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322015709801 │ deltacommit │ COMPLETED │ 03-22 01:57 │ 03-22 01:57 │ 03-22 01:58 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322015834047 │ deltacommit │ COMPLETED │ 03-22 01:58 │ 03-22 01:58 │ 03-22 01:58 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 2   │ 20230322015917952 │ deltacommit │ COMPLETED │ 03-22 01:59 │ 03-22 01:59 │ 03-22 01:59 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 3   │ 20230322015934748 │ compaction  │ REQUESTED │ 03-22 01:59 │ -           │ -           ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322015917952 │ 2.3 MB              │ 0                 │ 2                   │ 2                        │ 4341                  │ 4341                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015834047 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015709801 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State     │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╪═══════════╪═══════════════════════════════╣\n",
      "║ 20230322015934748       │ REQUESTED │ 2                             ║\n",
      "╚═════════════════════════╧═══════════╧═══════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_4\n",
      "├── [ 98M 02:00:03]  parity=0\n",
      "│   ├── [787K 01:58:45]  .885c3cb4-9d5e-4836-b190-68386d2ad1b8-0_20230322015709801.log.1_0-571-2029\n",
      "│   ├── [1.2M 01:59:27]  .885c3cb4-9d5e-4836-b190-68386d2ad1b8-0_20230322015709801.log.2_0-619-2090\n",
      "│   └── [ 96M 01:57:56]  885c3cb4-9d5e-4836-b190-68386d2ad1b8-0_0-523-1967_20230322015709801.parquet\n",
      "└── [ 98M 02:00:03]  parity=1\n",
      "    ├── [812K 01:58:45]  .a9a37853-b25f-4dd8-81bb-d712380cf57d-0_20230322015709801.log.1_1-571-2030\n",
      "    ├── [1.1M 01:59:27]  .a9a37853-b25f-4dd8-81bb-d712380cf57d-0_20230322015709801.log.2_1-619-2091\n",
      "    └── [ 96M 01:57:58]  a9a37853-b25f-4dd8-81bb-d712380cf57d-0_1-523-1968_20230322015709801.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a82f370d-cf74-4a46-b590-fb051d32e5f6",
   "metadata": {},
   "source": [
    "### 5.8. Offline Execute 96.15MB / +1 Compacted File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "ce5ffcac-1e3b-454d-bab8-3fbfdd2a2512",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T02:00:03.490726Z",
     "iopub.status.busy": "2023-03-22T02:00:03.490473Z",
     "iopub.status.idle": "2023-03-22T02:00:54.950568Z",
     "shell.execute_reply": "2023-03-22T02:00:54.949361Z",
     "shell.execute_reply.started": "2023-03-22T02:00:03.490701Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%sh\n",
    "# it's required for current user (emr-notebook) to get sudo permission\n",
    "sudo -u hadoop spark-submit \\\n",
    "  --jars '/usr/lib/hudi/hudi-spark-bundle.jar' \\\n",
    "  --class \"org.apache.hudi.utilities.HoodieCompactor\" \\\n",
    "  /usr/lib/hudi/hudi-utilities-bundle.jar \\\n",
    "  --spark-memory '4g' \\\n",
    "  --mode 'execute' \\\n",
    "  --base-path \"s3://${S3_BUCKET}/${TABLE_NAME}\" \\\n",
    "  --table-name \"$TABLE_NAME\" > ${WORKSPACE}/${TABLE_NAME}.execute.out &>/dev/null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "55e677b2-6151-4b87-b22a-55eba3550a1c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-22T02:00:54.952129Z",
     "iopub.status.busy": "2023-03-22T02:00:54.951858Z",
     "iopub.status.idle": "2023-03-22T02:01:18.796722Z",
     "shell.execute_reply": "2023-03-22T02:01:18.795899Z",
     "shell.execute_reply.started": "2023-03-22T02:00:54.952105Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ TIMELINE ]\n",
      "\n",
      "╔═════╤═══════════════════╤═════════════╤═══════════╤═════════════╤═════════════╤═════════════╗\n",
      "║ No. │ Instant           │ Action      │ State     │ Requested   │ Inflight    │ Completed   ║\n",
      "║     │                   │             │           │ Time        │ Time        │ Time        ║\n",
      "╠═════╪═══════════════════╪═════════════╪═══════════╪═════════════╪═════════════╪═════════════╣\n",
      "║ 0   │ 20230322015709801 │ deltacommit │ COMPLETED │ 03-22 01:57 │ 03-22 01:57 │ 03-22 01:58 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 1   │ 20230322015834047 │ deltacommit │ COMPLETED │ 03-22 01:58 │ 03-22 01:58 │ 03-22 01:58 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 2   │ 20230322015917952 │ deltacommit │ COMPLETED │ 03-22 01:59 │ 03-22 01:59 │ 03-22 01:59 ║\n",
      "╟─────┼───────────────────┼─────────────┼───────────┼─────────────┼─────────────┼─────────────╢\n",
      "║ 3   │ 20230322015934748 │ commit      │ COMPLETED │ 03-22 01:59 │ 03-22 02:00 │ 03-22 02:00 ║\n",
      "╚═════╧═══════════════════╧═════════════╧═══════════╧═════════════╧═════════════╧═════════════╝\n",
      "\n",
      "[ COMMITS ]\n",
      "\n",
      "╔═══════════════════╤═════════════════════╤═══════════════════╤═════════════════════╤══════════════════════════╤═══════════════════════╤══════════════════════════════╤══════════════╗\n",
      "║ CommitTime        │ Total Bytes Written │ Total Files Added │ Total Files Updated │ Total Partitions Written │ Total Records Written │ Total Update Records Written │ Total Errors ║\n",
      "╠═══════════════════╪═════════════════════╪═══════════════════╪═════════════════════╪══════════════════════════╪═══════════════════════╪══════════════════════════════╪══════════════╣\n",
      "║ 20230322015934748 │ 192.3 MB            │ 0                 │ 2                   │ 2                        │ 1155127               │ 7292                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015917952 │ 2.3 MB              │ 0                 │ 2                   │ 2                        │ 4341                  │ 4341                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015834047 │ 1.6 MB              │ 0                 │ 2                   │ 2                        │ 2951                  │ 2951                         │ 0            ║\n",
      "╟───────────────────┼─────────────────────┼───────────────────┼─────────────────────┼──────────────────────────┼───────────────────────┼──────────────────────────────┼──────────────╢\n",
      "║ 20230322015709801 │ 191.6 MB            │ 2                 │ 0                   │ 2                        │ 1155127               │ 0                            │ 0            ║\n",
      "╚═══════════════════╧═════════════════════╧═══════════════════╧═════════════════════╧══════════════════════════╧═══════════════════════╧══════════════════════════════╧══════════════╝\n",
      "\n",
      "[ COMPACTIONS ]\n",
      "\n",
      "╔═════════════════════════╤═══════════╤═══════════════════════════════╗\n",
      "║ Compaction Instant Time │ State     │ Total FileIds to be Compacted ║\n",
      "╠═════════════════════════╪═══════════╪═══════════════════════════════╣\n",
      "║ 20230322015934748       │ COMPLETED │ 2                             ║\n",
      "╚═════════════════════════╧═══════════╧═══════════════════════════════╝\n",
      "\n",
      "[ STORAGE ]\n",
      "\n",
      "/home/emr-notebook/apache-hudi-core-conceptions/reviews_mor_compaction_4\n",
      "├── [194M 02:01:18]  parity=0\n",
      "│   ├── [787K 01:58:45]  .885c3cb4-9d5e-4836-b190-68386d2ad1b8-0_20230322015709801.log.1_0-571-2029\n",
      "│   ├── [1.2M 01:59:27]  .885c3cb4-9d5e-4836-b190-68386d2ad1b8-0_20230322015709801.log.2_0-619-2090\n",
      "│   ├── [ 96M 02:00:48]  885c3cb4-9d5e-4836-b190-68386d2ad1b8-0_0-0-0_20230322015934748.parquet\n",
      "│   └── [ 96M 01:57:56]  885c3cb4-9d5e-4836-b190-68386d2ad1b8-0_0-523-1967_20230322015709801.parquet\n",
      "└── [194M 02:01:18]  parity=1\n",
      "    ├── [812K 01:58:45]  .a9a37853-b25f-4dd8-81bb-d712380cf57d-0_20230322015709801.log.1_1-571-2030\n",
      "    ├── [1.1M 01:59:27]  .a9a37853-b25f-4dd8-81bb-d712380cf57d-0_20230322015709801.log.2_1-619-2091\n",
      "    ├── [ 96M 02:00:47]  a9a37853-b25f-4dd8-81bb-d712380cf57d-0_1-0-1_20230322015934748.parquet\n",
      "    └── [ 96M 01:57:58]  a9a37853-b25f-4dd8-81bb-d712380cf57d-0_1-523-1968_20230322015709801.parquet\n"
     ]
    }
   ],
   "source": [
    "%%sh\n",
    "${WORKSPACE}/hudi-stat.sh s3://${S3_BUCKET}/${TABLE_NAME} timeline commits compactions storage"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Spark",
   "language": "scala",
   "name": "sparkkernel"
  },
  "language_info": {
   "codemirror_mode": "text/x-scala",
   "file_extension": ".sc",
   "mimetype": "text/x-scala",
   "name": "scala",
   "pygments_lexer": "scala"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}