span-batch-analysis/results_ipynb/span-batch-data-preparation-mainnet-base.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Span Batch Analysis - Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before start, Fill in [secret.yml](span-batch-analysis/secret.yaml). Refer to [secret_example.yml](span-batch-analysis/secret_example.yaml). This is for specifying RPC endpoints.\n",
    "\n",
    "Also check config located at [config](span-batch-analysis/config/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build binaries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cd cmd/batch_decoder && go build . && cd ../../\n",
    "cd cmd/span_batch_tester && go build . && cd ../../"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "%env BATCH_DECODER=cmd/batch_decoder/batch_decoder\n",
    "%env SPAN_BATCH_TESTER=cmd/span_batch_tester/span_batch_tester"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clone superchain config."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'superchain-registry' already exists and is not an empty directory.\n",
      "HEAD is now at 5d3ca19 superchain/extra: include minified genesis state data (#2)\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "git clone https://github.com/ethereum-optimism/superchain-registry\n",
    "cd superchain-registry && git checkout 5d3ca1966435827ac8c4216bddcb541ec64c1d6c && cd --"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select your chain."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "%env L1_CHAIN_NAME=mainnet\n",
    "%env L2_CHAIN_NAME=base\n",
    "\n",
    "# available networks\n",
    "#      L1 - L2 list\n",
    "# mainnet - base, op, pgn, zora\n",
    "# goerli  - base, op, zora\n",
    "# sepolia - op, pgn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1_chain_name = os.environ[\"L1_CHAIN_NAME\"]\n",
    "l2_chain_name = os.environ[\"L2_CHAIN_NAME\"]\n",
    "\n",
    "start_number = !yq eval \".start_l1_number\" \"span-batch-analysis/config/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "start_number_value = int(start_number[0])\n",
    "end_number = !yq eval \".end_l1_number\" \"span-batch-analysis/config/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "end_number_value = int(end_number[0])\n",
    "batch_tx_sender = !yq eval \".batch_tx_sender\" \"span-batch-analysis/config/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "batch_tx_sender_value = batch_tx_sender[0]\n",
    "l2_genesis_timestamp = !yq eval \".genesis.l2_time\" \"superchain-registry/superchain/configs/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "l2_genesis_timestamp_value = int(l2_genesis_timestamp[0])\n",
    "l2_chain_id = !yq eval \".chain_id\" \"superchain-registry/superchain/configs/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "l2_chain_id_value = int(l2_chain_id[0])\n",
    "inbox_addr = !yq eval \".batch_inbox_addr\" \"superchain-registry/superchain/configs/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "inbox_addr_value = inbox_addr[0]\n",
    "\n",
    "acc_start_number = !yq eval \".acc_l2_start_number\" \"span-batch-analysis/config/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "acc_start_number_value = int(acc_start_number[0])\n",
    "acc_end_number = !yq eval \".acc_l2_end_number\" \"span-batch-analysis/config/{l1_chain_name}/{l2_chain_name}.yaml\"\n",
    "acc_end_number_value = int(acc_end_number[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read analyis config."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: L1_START=18005000\n",
      "env: L1_END=18025000\n",
      "env: BATCH_TX_SENDER=0x5050F69a9786F081509234F1a7F4684b5E5b76C9\n",
      "env: L2_GENESIS_TIMESTAMP=1686789347\n",
      "env: L2_CHAIN_ID=8453\n",
      "env: INBOX=0xff00000000000000000000000000000000008453\n",
      "env: ACC_L2_START=3301000\n",
      "env: ACC_L2_END=3302000\n"
     ]
    }
   ],
   "source": [
    "%env L1_START=$start_number_value\n",
    "%env L1_END=$end_number_value\n",
    "%env BATCH_TX_SENDER=$batch_tx_sender_value\n",
    "%env L2_GENESIS_TIMESTAMP=$l2_genesis_timestamp_value\n",
    "%env L2_CHAIN_ID=$l2_chain_id_value\n",
    "%env INBOX=$inbox_addr_value\n",
    "\n",
    "%env ACC_L2_START=$acc_start_number_value\n",
    "%env ACC_L2_END=$acc_end_number_value\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get L1 RPC from secret."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "key_name = f\"L1_{l1_chain_name}_RPC\".upper()\n",
    "L1_RPC = !yq eval \".{key_name}\" span-batch-analysis/secret.yaml\n",
    "L1_RPC_ENDPOINT = L1_RPC[0]\n",
    "\n",
    "key_name = f\"L2_{l2_chain_name}_{l1_chain_name}_RPC\".upper()\n",
    "L2_RPC = !yq eval \".{key_name}\" span-batch-analysis/secret.yaml\n",
    "L2_RPC_ENDPOINT = L2_RPC[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "%env L1_RPC=$L1_RPC_ENDPOINT\n",
    "%env L2_RPC=$L2_RPC_ENDPOINT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Collection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All data will be stored at `./artifacts/{L1_CHAIN_NAME}/{L2_CHAIN_NAME}`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: DB_PREFIX=./artifacts/mainnet/base\n"
     ]
    }
   ],
   "source": [
    "l1_chain_name = os.environ[\"L1_CHAIN_NAME\"]\n",
    "l2_chain_name = os.environ[\"L2_CHAIN_NAME\"]\n",
    "\n",
    "%env DB_PREFIX=./artifacts/$l1_chain_name/$l2_chain_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "batch decoder `fetch`: Read channel frames from L1 by reading batch transactions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetched block:  18024991\n",
      "Fetched block:  18024958\n",
      "Fetched block:  18024993\n",
      "Fetched block:  18024984\n",
      "Fetched block:  18024988\n",
      "Fetched block:  18024994\n",
      "Fetched block:  18024998\n",
      "Fetched batches in range [18005000,18025000). Found 3999 valid & 0 invalid batches\n",
      "Fetch Config: Chain ID: 1. Inbox Address: 0xFf00000000000000000000000000000000008453. Valid Senders: map[0x5050F69a9786F081509234F1a7F4684b5E5b76C9:{}].\n",
      "Wrote transactions with batches to ./artifacts/mainnet/base/transactions_cache\n"
     ]
    }
   ],
   "source": [
    "!$BATCH_DECODER fetch --l1=$L1_RPC --start=$L1_START --end=$L1_END --inbox=$INBOX --out=$DB_PREFIX/transactions_cache --sender=$BATCH_TX_SENDER --concurrent-requests=25 | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "batch decoder `reassemble`: Reassemble channel frames to channels. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "!$BATCH_DECODER reassemble --inbox=$INBOX --in=$DB_PREFIX/transactions_cache --out=$DB_PREFIX/channel_cache | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "span batch tester `convert`: Convert channels with singular batches to span batches. Try multiple tx encoding types."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3990/3999] Channel ID: ff17565c00c4a65998c94425383adb2f, L2StartNum: 3265034, L2EndNum: 3265063, L2BlockCnt, 30\n",
      "[3991/3999] Channel ID: ff37bd71dd602ea5fe5157c1f1d19025, L2StartNum: 3176811, L2EndNum: 3176840, L2BlockCnt, 30\n",
      "[3992/3999] Channel ID: ff491053c9822ad16edefdf15643c5f6, L2StartNum: 3177140, L2EndNum: 3177169, L2BlockCnt, 30\n",
      "[3993/3999] Channel ID: ff5095660abcbcb0224f5515e7d9b499, L2StartNum: 3218937, L2EndNum: 3218965, L2BlockCnt, 29\n",
      "[3994/3999] Channel ID: ff896c648a4fcaab17a39575d14ce01c, L2StartNum: 3258326, L2EndNum: 3258355, L2BlockCnt, 30\n",
      "[3995/3999] Channel ID: ff9493a3218f720b56703b97edfce912, L2StartNum: 3247178, L2EndNum: 3247207, L2BlockCnt, 30\n",
      "[3996/3999] Channel ID: ffbc24e973a96562c3a8f3b785624694, L2StartNum: 3227306, L2EndNum: 3227335, L2BlockCnt, 30\n",
      "[3997/3999] Channel ID: ffe79769e81d0fb85eb5a92f1aa14671, L2StartNum: 3239738, L2EndNum: 3239767, L2BlockCnt, 30\n",
      "[3998/3999] Channel ID: fff1bf256e1bf054846d62167bf1bd23, L2StartNum: 3169334, L2EndNum: 3169363, L2BlockCnt, 30\n",
      "[3999/3999] Channel ID: fff44887c7ce0dae0705b1fc986d56e8, L2StartNum: 3273482, L2EndNum: 3273512, L2BlockCnt, 31\n"
     ]
    }
   ],
   "source": [
    "!TX_ENCODE_TYPE=0  && $SPAN_BATCH_TESTER convert --in=$DB_PREFIX/channel_cache --out=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --l2=$L2_RPC --genesis-timestamp=$L2_GENESIS_TIMESTAMP --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3990/3999] Channel ID: ff17565c00c4a65998c94425383adb2f, L2StartNum: 3265034, L2EndNum: 3265063, L2BlockCnt, 30\n",
      "[3991/3999] Channel ID: ff37bd71dd602ea5fe5157c1f1d19025, L2StartNum: 3176811, L2EndNum: 3176840, L2BlockCnt, 30\n",
      "[3992/3999] Channel ID: ff491053c9822ad16edefdf15643c5f6, L2StartNum: 3177140, L2EndNum: 3177169, L2BlockCnt, 30\n",
      "[3993/3999] Channel ID: ff5095660abcbcb0224f5515e7d9b499, L2StartNum: 3218937, L2EndNum: 3218965, L2BlockCnt, 29\n",
      "[3994/3999] Channel ID: ff896c648a4fcaab17a39575d14ce01c, L2StartNum: 3258326, L2EndNum: 3258355, L2BlockCnt, 30\n",
      "[3995/3999] Channel ID: ff9493a3218f720b56703b97edfce912, L2StartNum: 3247178, L2EndNum: 3247207, L2BlockCnt, 30\n",
      "[3996/3999] Channel ID: ffbc24e973a96562c3a8f3b785624694, L2StartNum: 3227306, L2EndNum: 3227335, L2BlockCnt, 30\n",
      "[3997/3999] Channel ID: ffe79769e81d0fb85eb5a92f1aa14671, L2StartNum: 3239738, L2EndNum: 3239767, L2BlockCnt, 30\n",
      "[3998/3999] Channel ID: fff1bf256e1bf054846d62167bf1bd23, L2StartNum: 3169334, L2EndNum: 3169363, L2BlockCnt, 30\n",
      "[3999/3999] Channel ID: fff44887c7ce0dae0705b1fc986d56e8, L2StartNum: 3273482, L2EndNum: 3273512, L2BlockCnt, 31\n"
     ]
    }
   ],
   "source": [
    "!TX_ENCODE_TYPE=1  && $SPAN_BATCH_TESTER convert --in=$DB_PREFIX/channel_cache --out=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --l2=$L2_RPC --genesis-timestamp=$L2_GENESIS_TIMESTAMP --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3990/3999] Channel ID: ff17565c00c4a65998c94425383adb2f, L2StartNum: 3265034, L2EndNum: 3265063, L2BlockCnt, 30\n",
      "[3991/3999] Channel ID: ff37bd71dd602ea5fe5157c1f1d19025, L2StartNum: 3176811, L2EndNum: 3176840, L2BlockCnt, 30\n",
      "[3992/3999] Channel ID: ff491053c9822ad16edefdf15643c5f6, L2StartNum: 3177140, L2EndNum: 3177169, L2BlockCnt, 30\n",
      "[3993/3999] Channel ID: ff5095660abcbcb0224f5515e7d9b499, L2StartNum: 3218937, L2EndNum: 3218965, L2BlockCnt, 29\n",
      "[3994/3999] Channel ID: ff896c648a4fcaab17a39575d14ce01c, L2StartNum: 3258326, L2EndNum: 3258355, L2BlockCnt, 30\n",
      "[3995/3999] Channel ID: ff9493a3218f720b56703b97edfce912, L2StartNum: 3247178, L2EndNum: 3247207, L2BlockCnt, 30\n",
      "[3996/3999] Channel ID: ffbc24e973a96562c3a8f3b785624694, L2StartNum: 3227306, L2EndNum: 3227335, L2BlockCnt, 30\n",
      "[3997/3999] Channel ID: ffe79769e81d0fb85eb5a92f1aa14671, L2StartNum: 3239738, L2EndNum: 3239767, L2BlockCnt, 30\n",
      "[3998/3999] Channel ID: fff1bf256e1bf054846d62167bf1bd23, L2StartNum: 3169334, L2EndNum: 3169363, L2BlockCnt, 30\n",
      "[3999/3999] Channel ID: fff44887c7ce0dae0705b1fc986d56e8, L2StartNum: 3273482, L2EndNum: 3273512, L2BlockCnt, 31\n"
     ]
    }
   ],
   "source": [
    "!TX_ENCODE_TYPE=2  && $SPAN_BATCH_TESTER convert --in=$DB_PREFIX/channel_cache --out=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --l2=$L2_RPC --genesis-timestamp=$L2_GENESIS_TIMESTAMP --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "span batch tester `analyze`: Analyze channels with singluar batches by comparing with corresponding span batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3990/3999] Channel ID: ff17565c00c4a65998c94425383adb2f, CompressedReductionPercent: 6.490385 %\n",
      "[3991/3999] Channel ID: ff37bd71dd602ea5fe5157c1f1d19025, CompressedReductionPercent: 6.579933 %\n",
      "[3992/3999] Channel ID: ff491053c9822ad16edefdf15643c5f6, CompressedReductionPercent: 6.141326 %\n",
      "[3993/3999] Channel ID: ff5095660abcbcb0224f5515e7d9b499, CompressedReductionPercent: 7.640014 %\n",
      "[3994/3999] Channel ID: ff896c648a4fcaab17a39575d14ce01c, CompressedReductionPercent: 6.230066 %\n",
      "[3995/3999] Channel ID: ff9493a3218f720b56703b97edfce912, CompressedReductionPercent: 6.391471 %\n",
      "[3996/3999] Channel ID: ffbc24e973a96562c3a8f3b785624694, CompressedReductionPercent: 7.494848 %\n",
      "[3997/3999] Channel ID: ffe79769e81d0fb85eb5a92f1aa14671, CompressedReductionPercent: 7.637616 %\n",
      "[3998/3999] Channel ID: fff1bf256e1bf054846d62167bf1bd23, CompressedReductionPercent: 7.337095 %\n",
      "[3999/3999] Channel ID: fff44887c7ce0dae0705b1fc986d56e8, CompressedReductionPercent: 6.365954 %\n"
     ]
    }
   ],
   "source": [
    "!TX_ENCODE_TYPE=0 && $SPAN_BATCH_TESTER analyze --in-channel=$DB_PREFIX/channel_cache --in-span-batch=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --out=$DB_PREFIX/result_v$TX_ENCODE_TYPE --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3990/3999] Channel ID: ff17565c00c4a65998c94425383adb2f, CompressedReductionPercent: 3.596612 %\n",
      "[3991/3999] Channel ID: ff37bd71dd602ea5fe5157c1f1d19025, CompressedReductionPercent: 3.353455 %\n",
      "[3992/3999] Channel ID: ff491053c9822ad16edefdf15643c5f6, CompressedReductionPercent: 2.927491 %\n",
      "[3993/3999] Channel ID: ff5095660abcbcb0224f5515e7d9b499, CompressedReductionPercent: 4.177178 %\n",
      "[3994/3999] Channel ID: ff896c648a4fcaab17a39575d14ce01c, CompressedReductionPercent: 3.181560 %\n",
      "[3995/3999] Channel ID: ff9493a3218f720b56703b97edfce912, CompressedReductionPercent: 3.914714 %\n",
      "[3996/3999] Channel ID: ffbc24e973a96562c3a8f3b785624694, CompressedReductionPercent: 4.646127 %\n",
      "[3997/3999] Channel ID: ffe79769e81d0fb85eb5a92f1aa14671, CompressedReductionPercent: 4.980092 %\n",
      "[3998/3999] Channel ID: fff1bf256e1bf054846d62167bf1bd23, CompressedReductionPercent: 3.682335 %\n",
      "[3999/3999] Channel ID: fff44887c7ce0dae0705b1fc986d56e8, CompressedReductionPercent: 3.690569 %\n"
     ]
    }
   ],
   "source": [
    "!TX_ENCODE_TYPE=1 && $SPAN_BATCH_TESTER analyze --in-channel=$DB_PREFIX/channel_cache --in-span-batch=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --out=$DB_PREFIX/result_v$TX_ENCODE_TYPE --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3990/3999] Channel ID: ff17565c00c4a65998c94425383adb2f, CompressedReductionPercent: 8.461538 %\n",
      "[3991/3999] Channel ID: ff37bd71dd602ea5fe5157c1f1d19025, CompressedReductionPercent: 8.224396 %\n",
      "[3992/3999] Channel ID: ff491053c9822ad16edefdf15643c5f6, CompressedReductionPercent: 7.870693 %\n",
      "[3993/3999] Channel ID: ff5095660abcbcb0224f5515e7d9b499, CompressedReductionPercent: 9.042144 %\n",
      "[3994/3999] Channel ID: ff896c648a4fcaab17a39575d14ce01c, CompressedReductionPercent: 8.550688 %\n",
      "[3995/3999] Channel ID: ff9493a3218f720b56703b97edfce912, CompressedReductionPercent: 8.079831 %\n",
      "[3996/3999] Channel ID: ffbc24e973a96562c3a8f3b785624694, CompressedReductionPercent: 9.211384 %\n",
      "[3997/3999] Channel ID: ffe79769e81d0fb85eb5a92f1aa14671, CompressedReductionPercent: 10.162422 %\n",
      "[3998/3999] Channel ID: fff1bf256e1bf054846d62167bf1bd23, CompressedReductionPercent: 9.290684 %\n",
      "[3999/3999] Channel ID: fff44887c7ce0dae0705b1fc986d56e8, CompressedReductionPercent: 8.236728 %\n"
     ]
    }
   ],
   "source": [
    "!TX_ENCODE_TYPE=2 && $SPAN_BATCH_TESTER analyze --in-channel=$DB_PREFIX/channel_cache --in-span-batch=$DB_PREFIX/span_batch_cache_v$TX_ENCODE_TYPE --out=$DB_PREFIX/result_v$TX_ENCODE_TYPE --chain-id=$L2_CHAIN_ID --tx-type=$TX_ENCODE_TYPE | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "span batch tester `fetch`: Fetch consecutive singular batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetched L2 block:  3301992\n",
      "Fetched L2 block:  3301993\n",
      "Fetched L2 block:  3301994\n",
      "Fetched L2 block:  3301995\n",
      "Fetched L2 block:  3301996\n",
      "Fetched L2 block:  3301997\n",
      "Fetched L2 block:  3301998\n",
      "Fetched L2 block:  3301999\n",
      "Fetched v0 batches in range [3301000,3302000).\n",
      "Fetch Config: Chain ID: 8453.\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER fetch --l2 $L2_RPC --start=$ACC_L2_START --end=$ACC_L2_END --concurrent-requests=5 --out=$DB_PREFIX/singular_batches_cache | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "span batch tester `merge`: Merge singular batches:\n",
    "\n",
    "This command will analyze `start - end` number of batches, Comparing uncompressed size of below two batches:\n",
    "- For `i` in range  `0` to `start - end - 1`\n",
    "  - singular batch from number `start` to `start + i`, inclusive.\n",
    "  - span batch from number `start` to `start + i`, inclusive.\n",
    "\n",
    "Upper command will make a result json file which has name of the form `{start_number}_{end_number}.json`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Merged v0 batches in range [3301000,3302000).\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER merge --start=$ACC_L2_START --end=$ACC_L2_END --l2 $L2_RPC --genesis-timestamp=$L2_GENESIS_TIMESTAMP --in=$DB_PREFIX/singular_batches_cache --out=$DB_PREFIX/merge_result | tail -10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "span batch tester `format`: Compare different tx encodings for span batch.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduction Percentage: -0.118630 %\n",
      "[3996/3999] cnt: 1537, delta: 19, deltasum: -84531, originalCompressedSizeSum: 152687385\n",
      "Reduction Percentage: 0.054593 %\n",
      "[3997/3999] cnt: 1537, delta: -19, deltasum: -84550, originalCompressedSizeSum: 152715816\n",
      "Reduction Percentage: -0.066828 %\n",
      "[3998/3999] cnt: 1538, delta: 29, deltasum: -84521, originalCompressedSizeSum: 152758610\n",
      "Reduction Percentage: 0.067767 %\n",
      "[3999/3999] cnt: 1539, delta: 58, deltasum: -84463, originalCompressedSizeSum: 152800029\n",
      "Reduction Percentage: 0.140032 %\n",
      "Final Reduction Percentage: -0.055277 %\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0123465 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduction Percentage: -0.350499 %\n",
      "[3996/3999] cnt: 625, delta: -68, deltasum: -352538, originalCompressedSizeSum: 152687385\n",
      "Reduction Percentage: -0.195385 %\n",
      "[3997/3999] cnt: 625, delta: -149, deltasum: -352687, originalCompressedSizeSum: 152715816\n",
      "Reduction Percentage: -0.524076 %\n",
      "[3998/3999] cnt: 625, delta: -14, deltasum: -352701, originalCompressedSizeSum: 152758610\n",
      "Reduction Percentage: -0.032715 %\n",
      "[3999/3999] cnt: 626, delta: 89, deltasum: -352612, originalCompressedSizeSum: 152800029\n",
      "Reduction Percentage: 0.214877 %\n",
      "Final Reduction Percentage: -0.230767 %\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0134625 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduction Percentage: -0.118630 %\n",
      "[3996/3999] cnt: 981, delta: -2, deltasum: -175193, originalCompressedSizeSum: 152687385\n",
      "Reduction Percentage: -0.005747 %\n",
      "[3997/3999] cnt: 981, delta: -19, deltasum: -175212, originalCompressedSizeSum: 152715816\n",
      "Reduction Percentage: -0.066828 %\n",
      "[3998/3999] cnt: 981, delta: -18, deltasum: -175230, originalCompressedSizeSum: 152758610\n",
      "Reduction Percentage: -0.042062 %\n",
      "[3999/3999] cnt: 982, delta: 12, deltasum: -175218, originalCompressedSizeSum: 152800029\n",
      "Reduction Percentage: 0.028972 %\n",
      "Final Reduction Percentage: -0.114671 %\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 3041265 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduction Percentage: 0.037746 %\n",
      "[3996/3999] cnt: 2198, delta: 1, deltasum: 11311, originalCompressedSizeSum: 152687385\n",
      "Reduction Percentage: 0.002873 %\n",
      "[3997/3999] cnt: 2199, delta: 1, deltasum: 11312, originalCompressedSizeSum: 152715816\n",
      "Reduction Percentage: 0.003517 %\n",
      "[3998/3999] cnt: 2200, delta: 30, deltasum: 11342, originalCompressedSizeSum: 152758610\n",
      "Reduction Percentage: 0.070103 %\n",
      "[3999/3999] cnt: 2201, delta: 20, deltasum: 11362, originalCompressedSizeSum: 152800029\n",
      "Reduction Percentage: 0.048287 %\n",
      "Final Reduction Percentage: 0.007436 %\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0125634 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduction Percentage: -0.118630 %\n",
      "[3996/3999] cnt: 1321, delta: 14, deltasum: -118409, originalCompressedSizeSum: 152687385\n",
      "Reduction Percentage: 0.040226 %\n",
      "[3997/3999] cnt: 1321, delta: -19, deltasum: -118428, originalCompressedSizeSum: 152715816\n",
      "Reduction Percentage: -0.066828 %\n",
      "[3998/3999] cnt: 1322, delta: 15, deltasum: -118413, originalCompressedSizeSum: 152758610\n",
      "Reduction Percentage: 0.035052 %\n",
      "[3999/3999] cnt: 1323, delta: 30, deltasum: -118383, originalCompressedSizeSum: 152800029\n",
      "Reduction Percentage: 0.072431 %\n",
      "Final Reduction Percentage: -0.077476 %\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0324165 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduction Percentage: 0.000000 %\n",
      "[3996/3999] cnt: 1545, delta: -21, deltasum: -83555, originalCompressedSizeSum: 152687385\n",
      "Reduction Percentage: -0.060340 %\n",
      "[3997/3999] cnt: 1546, delta: 0, deltasum: -83555, originalCompressedSizeSum: 152715816\n",
      "Reduction Percentage: 0.000000 %\n",
      "[3998/3999] cnt: 1546, delta: -44, deltasum: -83599, originalCompressedSizeSum: 152758610\n",
      "Reduction Percentage: -0.102818 %\n",
      "[3999/3999] cnt: 1546, delta: -43, deltasum: -83642, originalCompressedSizeSum: 152800029\n",
      "Reduction Percentage: -0.103817 %\n",
      "Final Reduction Percentage: -0.054740 %\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 3041256 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduction Percentage: 0.000000 %\n",
      "[3996/3999] cnt: 1724, delta: -21, deltasum: -83128, originalCompressedSizeSum: 152687385\n",
      "Reduction Percentage: -0.060340 %\n",
      "[3997/3999] cnt: 1725, delta: 0, deltasum: -83128, originalCompressedSizeSum: 152715816\n",
      "Reduction Percentage: 0.000000 %\n",
      "[3998/3999] cnt: 1725, delta: -42, deltasum: -83170, originalCompressedSizeSum: 152758610\n",
      "Reduction Percentage: -0.098145 %\n",
      "[3999/3999] cnt: 1725, delta: -44, deltasum: -83214, originalCompressedSizeSum: 152800029\n",
      "Reduction Percentage: -0.106231 %\n",
      "Final Reduction Percentage: -0.054459 %\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 3401256 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduction Percentage: -0.062011 %\n",
      "[3996/3999] cnt: 1693, delta: -16, deltasum: -63348, originalCompressedSizeSum: 152687385\n",
      "Reduction Percentage: -0.045973 %\n",
      "[3997/3999] cnt: 1693, delta: -82, deltasum: -63430, originalCompressedSizeSum: 152715816\n",
      "Reduction Percentage: -0.288418 %\n",
      "[3998/3999] cnt: 1694, delta: 95, deltasum: -63335, originalCompressedSizeSum: 152758610\n",
      "Reduction Percentage: 0.221994 %\n",
      "[3999/3999] cnt: 1695, delta: 111, deltasum: -63224, originalCompressedSizeSum: 152800029\n",
      "Reduction Percentage: 0.267993 %\n",
      "Final Reduction Percentage: -0.041377 %\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 6051234 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduction Percentage: 0.000000 %\n",
      "[3996/3999] cnt: 1912, delta: 0, deltasum: -30009, originalCompressedSizeSum: 152687385\n",
      "Reduction Percentage: 0.000000 %\n",
      "[3997/3999] cnt: 1913, delta: 0, deltasum: -30009, originalCompressedSizeSum: 152715816\n",
      "Reduction Percentage: 0.000000 %\n",
      "[3998/3999] cnt: 1913, delta: -16, deltasum: -30025, originalCompressedSizeSum: 152758610\n",
      "Reduction Percentage: -0.037388 %\n",
      "[3999/3999] cnt: 1913, delta: -25, deltasum: -30050, originalCompressedSizeSum: 152800029\n",
      "Reduction Percentage: -0.060359 %\n",
      "Final Reduction Percentage: -0.019666 %\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0132456 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduction Percentage: -0.013481 %\n",
      "[3996/3999] cnt: 48, delta: -96, deltasum: -374234, originalCompressedSizeSum: 152687385\n",
      "Reduction Percentage: -0.275838 %\n",
      "[3997/3999] cnt: 48, delta: -60, deltasum: -374294, originalCompressedSizeSum: 152715816\n",
      "Reduction Percentage: -0.211037 %\n",
      "[3998/3999] cnt: 48, delta: -92, deltasum: -374386, originalCompressedSizeSum: 152758610\n",
      "Reduction Percentage: -0.214983 %\n",
      "[3999/3999] cnt: 48, delta: -75, deltasum: -374461, originalCompressedSizeSum: 152800029\n",
      "Reduction Percentage: -0.181076 %\n",
      "Final Reduction Percentage: -0.245066 %\n"
     ]
    }
   ],
   "source": [
    "!$SPAN_BATCH_TESTER format --permutation 0352614 --in-span-batch=$DB_PREFIX/span_batch_cache_v2 --out=$DB_PREFIX/format_result --chain-id=$L2_CHAIN_ID | tail -10"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}