feat: improve devnet command

.env.example

@@ -0,0 +1,4 @@
+export L1_FORK_PUBLIC_NETWORK=true
+export L1_RPC='https://eth-mainnet.g.alchemy.com/v2/'
+export IMPL_SALT='tokamak network'
+export L2_NATIVE_TOKEN=0x2be5e8c109e2197D077D13A82dAead6a9b3433C5

Makefile

@@ -1,7 +1,7 @@
 COMPOSEFLAGS=-d
 ITESTS_L2_HOST=http://localhost:9545
 BEDROCK_TAGS_REMOTE?=origin
-OP_STACK_GO_BUILDER?=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
+OP_STACK_GO_BUILDER?=onthertech/titan-op-stack-go:latest
 
 # Requires at least Python v3.9; specify a minor version below if needed
 PYTHON?=python3
@@ -105,7 +105,7 @@ nuke: clean devnet-clean
 .PHONY: nuke
 
 pre-devnet:
-	@if [ "${FORK_PUBLIC_NETWORK}" != "true" ]  && ! [ -x "$(command -v geth)" ]; then \
+	@if ! [ -x "$(command -v geth)" ]; then \
 		make install-geth; \
 	fi
 	@if [ ! -e op-program/bin ]; then \

bedrock-devnet/devnet/__init__.py

@@ -10,11 +10,12 @@
 import http.client
 import gzip
 import glob
+import ssl
+import urllib
 from multiprocessing import Process, Queue
 import concurrent.futures
 from collections import namedtuple
 
-import devnet.log_setup
 
 pjoin = os.path.join
 
@@ -25,9 +26,9 @@
 parser.add_argument('--fork-public-network',
                     help='Fork the public network',
                     type=bool,
-                    default=os.environ.get('FORK_PUBLIC_NETWORK').lower() == 'true' if os.environ.get('FORK_PUBLIC_NETWORK') else False)
+                    default=os.environ.get('L1_FORK_PUBLIC_NETWORK').lower() == 'true' if os.environ.get('L1_FORK_PUBLIC_NETWORK') else False)
 parser.add_argument('--l1-rpc-url', help='Public L1 RPC URL', type=str, default=os.environ.get('L1_RPC'))
-parser.add_argument('--from-block-number', help='From block number', type=int, default=os.environ.get('FROM_BLOCK_NUMBER'))
+parser.add_argument('--block-number', help='From block number', type=int, default=os.environ.get('BLOCK_NUMBER'))
 parser.add_argument('--l2-native-token', help='L2 native token', type=str, default=os.environ.get('L2_NATIVE_TOKEN'))
 parser.add_argument('--admin-key', help='The admin private key for upgrade contracts', type=str, default=os.environ.get('DEVNET_ADMIN_PRIVATE_KEY'))
 parser.add_argument('--l2-image', help='Using local l2', type=str, default=os.environ.get('L2_IMAGE') if os.environ.get('L2_IMAGE') is not None else 'onthertech/titan-op-geth:nightly')
@@ -77,6 +78,15 @@ def main():
     sdk_dir = pjoin(monorepo_dir, 'packages', 'tokamak', 'sdk')
     bedrock_devnet_dir = pjoin(monorepo_dir, 'bedrock-devnet')
 
+    if args.fork_public_network:
+      if args.block_number is not None:
+        block_number = str(args.block_number)
+      else:
+        response = json.loads(eth_new_head(args.l1_rpc_url))
+        block_number = str(int(response['result'], 16))
+    else:
+      block_number = "0"
+
     paths = Bunch(
       mono_repo_dir=monorepo_dir,
       devnet_dir=devnet_dir,
@@ -97,9 +107,9 @@ def main():
       sdk_addresses_json_path=pjoin(devnet_dir, 'sdk-addresses.json'),
       rollup_config_path=pjoin(devnet_dir, 'rollup.json'),
       fork_public_network = args.fork_public_network,
-      l1_rpc_url = args.l1_rpc_url,
-      l2_native_token = args.l2_native_token,
-      from_block_number = args.from_block_number,
+      l1_rpc_url=args.l1_rpc_url,
+      block_number=block_number,
+      l2_native_token=args.l2_native_token,
       bedrock_devnet_path=bedrock_devnet_dir,
       admin_key=args.admin_key
     )
@@ -123,12 +133,16 @@ def main():
     else:
         log.info(f'Building docker images for git commit {git_commit} ({git_date})')
         run_command(['docker', 'compose', 'build', '--progress', 'plain',
-                     '--build-arg', f'GIT_COMMIT={git_commit}', '--build-arg', f'GIT_DATE={git_date}'],
-                    cwd=paths.ops_bedrock_dir, env={
+                    '--build-arg', f'GIT_COMMIT={git_commit}', '--build-arg', f'GIT_DATE={git_date}'],
+                cwd=paths.ops_bedrock_dir, env={
             'PWD': paths.ops_bedrock_dir,
-            'L2_IMAGE': args.l2_image,
             'DOCKER_BUILDKIT': '1', # (should be available by default in later versions, but explicitly enable it anyway)
-            'COMPOSE_DOCKER_CLI_BUILD': '1'  # use the docker cache
+            'COMPOSE_DOCKER_CLI_BUILD': '1',  # use the docker cache
+            'L2_IMAGE': args.l2_image,
+            'L1_DOCKER_FILE': 'Dockerfile.l1.fork' if paths.fork_public_network else 'Dockerfile.l1',
+            'L1_RPC': paths.l1_rpc_url if paths.fork_public_network else '',
+            'BLOCK_NUMBER': paths.block_number,
+            'L1_FORK_PUBLIC_NETWORK': str(paths.fork_public_network)
         })
 
     log.info('Devnet starting')
@@ -143,24 +157,20 @@ def deploy_contracts(paths):
     account = response['result'][0 if paths.admin_key is None else 1]
     log.info(f'Deploying with {account}')
 
-    # Proxy exists on the fork public network(used by anvil)
-    # We don't need to deploy the proxy contract
-    # https://book.getfoundry.sh/tutorials/create2-tutorial
-    if not paths.fork_public_network:
-        # send some ether to the create2 deployer account
-        cmd = [
-            'cast', 'send',
-            '--rpc-url', 'http://127.0.0.1:8545',
-            '--value', '1ether', '0x3fAB184622Dc19b6109349B94811493BF2a45362'
-        ]
-        cmd.extend(['--from', account, '--unlocked']) if paths.admin_key is None else cmd.extend(['--password', '1234'])
-        run_command(cmd, env={}, cwd=paths.contracts_bedrock_dir)
-
-        # deploy the create2 deployer
-        run_command([
-          'cast', 'publish', '--rpc-url', 'http://127.0.0.1:8545',
-          '0xf8a58085174876e800830186a08080b853604580600e600039806000f350fe7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf31ba02222222222222222222222222222222222222222222222222222222222222222a02222222222222222222222222222222222222222222222222222222222222222'
-        ], env={}, cwd=paths.contracts_bedrock_dir)
+    # send some ether to the create2 deployer account
+    cmd = [
+      'cast', 'send',
+      '--rpc-url', 'http://127.0.0.1:8545',
+      '--value', '1ether', '0x3fAB184622Dc19b6109349B94811493BF2a45362'
+    ]
+    cmd.extend(['--from', account, '--unlocked']) if paths.admin_key is None else cmd.extend(['--password', '1234'])
+    run_command(cmd, env={}, cwd=paths.contracts_bedrock_dir)
+
+    # deploy the create2 deployer
+    run_command([
+      'cast', 'publish', '--rpc-url', 'http://127.0.0.1:8545',
+      '0xf8a58085174876e800830186a08080b853604580600e600039806000f350fe7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf31ba02222222222222222222222222222222222222222222222222222222222222222a02222222222222222222222222222222222222222222222222222222222222222'
+    ], env={}, cwd=paths.contracts_bedrock_dir)
 
     fqn = 'scripts/Deploy.s.sol:Deploy'
     cmd = [
@@ -239,22 +249,14 @@ def devnet_l1_genesis(paths):
     log.info('Generating L1 genesis state')
     init_devnet_l1_deploy_config(paths)
 
-    if paths.fork_public_network:
-        log.info('Start to fork the public network. Wait to warm up the fork public network.')
-        geth = subprocess.Popen([
-            'anvil', '--fork-url', paths.l1_rpc_url, '--fork-block-number', str(paths.from_block_number),
-            '--chain-id', '1337'
-        ])
-        time.sleep(30)
-    else:
-        if paths.admin_key is not None and not os.path.exists(pjoin(paths.bedrock_devnet_path, 'data')) :
-            init_admin_geth(paths)
+    if paths.admin_key is not None and not os.path.exists(pjoin(paths.bedrock_devnet_path, 'data')) :
+      init_admin_geth(paths)
 
-        geth = subprocess.Popen([
-            'geth', '--dev', '--dev.period', '2', '--http', '--http.api', 'eth,debug',
-            '--verbosity', '4', '--gcmode', 'archive', '--dev.gaslimit', '30000000',
-            '--rpc.allow-unprotected-txs'
-        ], cwd=pjoin(paths.mono_repo_dir, 'bedrock-devnet'))
+    geth = subprocess.Popen([
+      'geth', '--dev', '--dev.period', '2', '--http', '--http.api', 'eth,debug',
+      '--verbosity', '4', '--gcmode', 'archive', '--dev.gaslimit', '30000000',
+      '--rpc.allow-unprotected-txs'
+    ], cwd=pjoin(paths.mono_repo_dir, 'bedrock-devnet'))
 
     try:
         forge = ChildProcess(deploy_contracts, paths)
@@ -264,13 +266,9 @@ def devnet_l1_genesis(paths):
         if err:
             raise Exception(f"Exception occurred in child process: {err}")
 
-        if paths.fork_public_network:
-          res = anvil_dumpState('127.0.0.1:8545')
-          allocs = convert_anvil_dump(res)
-        else:
-          res = debug_dumpBlock('127.0.0.1:8545')
-          response = json.loads(res)
-          allocs = response['result']
+        res = debug_dumpBlock('127.0.0.1:8545')
+        response = json.loads(res)
+        allocs = response['result']
         write_json(paths.allocs_path, allocs)
     finally:
         geth.terminate()
@@ -303,6 +301,8 @@ def devnet_deploy(paths, args):
     log.info('Starting L1.')
     run_command(['docker', 'compose', 'up', '-d', 'l1'], cwd=paths.ops_bedrock_dir, env={
         'PWD': paths.ops_bedrock_dir,
+        'L1_RPC': paths.l1_rpc_url if paths.fork_public_network else '',
+        'BLOCK_NUMBER': paths.block_number,
     })
     wait_up(8545)
     wait_for_rpc_server('127.0.0.1:8545')
@@ -340,12 +340,18 @@ def devnet_deploy(paths, args):
     run_command(['docker', 'compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher'], cwd=paths.ops_bedrock_dir, env={
         'PWD': paths.ops_bedrock_dir,
         'L2OO_ADDRESS': l2_output_oracle,
-        'SEQUENCER_BATCH_INBOX_ADDRESS': batch_inbox_address
+        'SEQUENCER_BATCH_INBOX_ADDRESS': batch_inbox_address,
+        'L1_RPC': paths.l1_rpc_url if paths.fork_public_network else '',
+        'BLOCK_NUMBER': paths.block_number,
+        'WAITING_L1_PORT': '9999' if paths.fork_public_network else '8545',
+        'L1_FORK_PUBLIC_NETWORK': str(paths.fork_public_network)
     })
 
     log.info('Bringing up `artifact-server`')
     run_command(['docker', 'compose', 'up', '-d', 'artifact-server'], cwd=paths.ops_bedrock_dir, env={
-        'PWD': paths.ops_bedrock_dir
+        'PWD': paths.ops_bedrock_dir,
+        'L1_RPC': paths.l1_rpc_url if paths.fork_public_network else '',
+        'BLOCK_NUMBER': paths.block_number,
     })
 
     shutil.rmtree(pjoin(paths.bedrock_devnet_path, 'data'), ignore_errors=True)
@@ -364,6 +370,29 @@ def eth_accounts(url):
     conn.close()
     return data
 
+def eth_new_head(url):
+    parsed_url = urllib.parse.urlparse(url.strip())
+    hostname = parsed_url.hostname
+    path = parsed_url.path if parsed_url.path else '/'
+    # Create a context that does not verify SSL certificates
+    context = ssl._create_unverified_context()
+    conn = http.client.HTTPSConnection(hostname, context=context)
+    headers = {'Content-type': 'application/json'}
+    body = '{"id":2, "jsonrpc":"2.0", "method": "eth_blockNumber", "params":[]}'
+    try:
+        conn.request('POST', path, body, headers)
+        response = conn.getresponse()
+        data = response.read().decode()
+
+    except Exception as e:
+        log.error(f"Failed to fetch the latest block: {e}")
+        data = None
+    finally:
+        conn.close()
+
+    return data
+
+
 def debug_dumpBlock(url):
     log.info(f'Fetch debug_dumpBlock {url}')
     conn = http.client.HTTPConnection(url)
@@ -503,7 +532,7 @@ def validate_fork_public_network(args):
     fork_public_network = args.fork_public_network
     l1_rpc_url = args.l1_rpc_url
     l2_native_token = args.l2_native_token
-    from_block_number = args.from_block_number
+    block_number = args.block_number
     # If fork the public network, validate the required params related to
     if fork_public_network:
       if not l1_rpc_url:
@@ -512,43 +541,8 @@ def validate_fork_public_network(args):
       if not l2_native_token:
         raise Exception("Please provide the L2_NATIVE_TOKEN for the forked network.")
 
+      log.info(f'Fork from RPC URL: {l1_rpc_url}, block number: {block_number}, l2 native token: {l2_native_token}')
 
-      if not from_block_number:
-        raise Exception("Please provide the FROM_BLOCK_NUMBER for the forked network.")
-
-      if from_block_number <= 0:
-        raise Exception("Please provide the FROM_BLOCK_NUMBER is bigger than zero.")
-      log.info(f'Fork from RPC URL: {l1_rpc_url}, from block number: {from_block_number}, l2 native token: {l2_native_token}')
-
-
-def anvil_dumpState(url):
-    log.info(f'Fetch debug_dumpBlock {url}')
-    conn = http.client.HTTPConnection(url)
-    headers = {'Content-type': 'application/json'}
-    body = '{"id":3, "jsonrpc":"2.0", "method": "anvil_dumpState", "params":[]}'
-    conn.request('POST', '/', body, headers)
-    data = conn.getresponse().read()
-    # Anvil returns a JSON-RPC response with a hex-encoded "result" field
-    result = json.loads(data.decode('utf-8'))['result']
-    result_bytes = bytes.fromhex(result[2:])
-    uncompressed = gzip.decompress(result_bytes).decode()
-    return json.loads(uncompressed)
-
-def convert_anvil_dump(dump):
-    accounts = dump['accounts']
-
-    for account in accounts.values():
-      bal = account['balance']
-      account['balance'] = str(int(bal, 16))
-
-      if 'storage' in account:
-        storage = account['storage']
-        storage_keys = list(storage.keys())
-        for key in storage_keys:
-          value = storage[key]
-          del storage[key]
-          storage[pad_hex(key)] = pad_hex(value)
-    return dump
 
 def pad_hex(input):
     return '0x' + input.replace('0x', '').zfill(64)

op-batcher/Dockerfile

@@ -1,4 +1,4 @@
-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
+ARG OP_STACK_GO_BUILDER=onthertech/titan-op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-challenger/Dockerfile

@@ -1,4 +1,4 @@
-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
+ARG OP_STACK_GO_BUILDER=onthertech/titan-op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-heartbeat/Dockerfile

@@ -1,4 +1,4 @@
-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
+ARG OP_STACK_GO_BUILDER=onthertech/titan-op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-node/Dockerfile

@@ -1,4 +1,4 @@
-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
+ARG OP_STACK_GO_BUILDER=onthertech/titan-op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-node/flags/flags.go

@@ -265,6 +265,13 @@ var (
 		EnvVars: prefixEnvVars("OVERRIDE_CANYON"),
 		Hidden:  false,
 	}
+	IsForkPublicNetworkFlag = &cli.BoolFlag{
+		Name:     "l1.fork-public-network",
+		Usage:    "Determine to use the fork public network or not",
+		EnvVars:  prefixEnvVars("L1_FORK_PUBLIC_NETWORK"),
+		Required: false,
+		Hidden:   true,
+	}
 )
 
 var requiredFlags = []cli.Flag{
@@ -311,6 +318,7 @@ var optionalFlags = []cli.Flag{
 	RollupLoadProtocolVersions,
 	CanyonOverrideFlag,
 	L1RethDBPath,
+	IsForkPublicNetworkFlag,
 }
 
 // Flags contains the list of configuration options available to the binary.

op-node/node/config.go

@@ -63,6 +63,9 @@ type Config struct {
 
 	// [OPTIONAL] The reth DB path to read receipts from
 	RethDBPath string
+
+	// [OPTIONAL] The flag determines whatever the network is the fork public network
+	IsForkPublicNetwork bool
 }
 
 type RPCConfig struct {

op-node/node/node.go

@@ -159,6 +159,9 @@ func (n *OpNode) initL1(ctx context.Context, cfg *Config) error {
 	// Set the RethDB path in the EthClientConfig, if there is one configured.
 	rpcCfg.EthClientConfig.RethDBPath = cfg.RethDBPath
 
+	// Set the flag in the EthClientConfig, if the network is the fork public network
+	rpcCfg.EthClientConfig.IsForkPublicNetwork = cfg.IsForkPublicNetwork
+
 	n.l1Source, err = sources.NewL1Client(
 		client.NewInstrumentedRPC(l1Node, n.metrics), n.log, n.metrics.L1SourceCache, rpcCfg)
 	if err != nil {

op-node/service.go

@@ -101,10 +101,11 @@ func NewConfig(ctx *cli.Context, log log.Logger) (*node.Config, error) {
 			Moniker: ctx.String(flags.HeartbeatMonikerFlag.Name),
 			URL:     ctx.String(flags.HeartbeatURLFlag.Name),
 		},
-		ConfigPersistence: configPersistence,
-		Sync:              *syncConfig,
-		RollupHalt:        haltOption,
-		RethDBPath:        ctx.String(flags.L1RethDBPath.Name),
+		ConfigPersistence:   configPersistence,
+		Sync:                *syncConfig,
+		RollupHalt:          haltOption,
+		RethDBPath:          ctx.String(flags.L1RethDBPath.Name),
+		IsForkPublicNetwork: ctx.Bool(flags.IsForkPublicNetworkFlag.Name),
 	}
 
 	if err := cfg.LoadPersisted(log); err != nil {

op-program/Dockerfile

@@ -1,4 +1,4 @@
-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
+ARG OP_STACK_GO_BUILDER=onthertech/titan-op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-proposer/Dockerfile

@@ -1,4 +1,4 @@
-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
+ARG OP_STACK_GO_BUILDER=onthertech/titan-op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go