Merge branch 'develop' into omerfirmak/sanitize-vm-config

.github/workflows/l2geth_ci.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Build
@@ -33,7 +33,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Install rust
       uses: actions-rs/toolchain@v1
       with:
@@ -55,7 +55,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Lint
@@ -88,7 +88,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Checkout code
       uses: actions/checkout@v2
     - run: go mod tidy
@@ -105,12 +105,11 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Test
       run: |
-        go get ./...
         make test
     - name: Upload coverage report
       run: bash <(curl -s https://codecov.io/bash)

.github/workflows/semgrep.yml

@@ -0,0 +1,24 @@
+on:
+  workflow_dispatch: {}
+  pull_request: {}
+  push:
+    branches:
+    - main
+    - master
+    paths:
+    - .github/workflows/semgrep.yml
+  schedule:
+  # random HH:MM to avoid a load spike on GitHub Actions at 00:00
+  - cron: 20 19 * * *
+name: Semgrep
+jobs:
+  semgrep:
+    name: semgrep/ci
+    runs-on: ubuntu-20.04
+    env:
+      SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}
+    container:
+      image: returntocorp/semgrep
+    steps:
+    - uses: actions/checkout@v3
+    - run: semgrep ci

Dockerfile

@@ -2,6 +2,7 @@
 ARG COMMIT=""
 ARG VERSION=""
 ARG BUILDNUM=""
+ARG SCROLL_LIB_PATH=/scroll/lib
 
 # Build libzkp dependency
 FROM scrolltech/go-rust-builder:go-1.20-rust-nightly-2022-12-10 as chef
@@ -25,9 +26,17 @@ RUN find ./ | grep libzktrie.so | xargs -I{} cp {} /app/target/release/
 FROM scrolltech/go-rust-builder:go-1.20-rust-nightly-2022-12-10 as builder
 
 ADD . /go-ethereum
-COPY --from=zkp-builder /app/target/release/libzkp.so /usr/local/lib/
-COPY --from=zkp-builder /app/target/release/libzktrie.so /usr/local/lib/
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
+
+ARG SCROLL_LIB_PATH
+
+RUN mkdir -p $SCROLL_LIB_PATH
+
+COPY --from=zkp-builder /app/target/release/libzkp.so $SCROLL_LIB_PATH
+COPY --from=zkp-builder /app/target/release/libzktrie.so $SCROLL_LIB_PATH
+
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
+ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
+
 RUN cd /go-ethereum && env GO111MODULE=on go run build/ci.go install -buildtags circuit_capacity_checker ./cmd/geth
 
 # Pull Geth into a second stage deploy alpine container
@@ -37,9 +46,16 @@ RUN apt-get -qq update \
     && apt-get -qq install -y --no-install-recommends ca-certificates
 
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
-COPY --from=zkp-builder /app/target/release/libzkp.so /usr/local/lib/
-COPY --from=zkp-builder /app/target/release/libzktrie.so /usr/local/lib/
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
+
+ARG SCROLL_LIB_PATH
+
+RUN mkdir -p $SCROLL_LIB_PATH
+
+COPY --from=zkp-builder /app/target/release/libzkp.so $SCROLL_LIB_PATH
+COPY --from=zkp-builder /app/target/release/libzktrie.so $SCROLL_LIB_PATH
+
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
+ENV CGO_LDFLAGS="-ldl -L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
 
 EXPOSE 8545 8546 30303 30303/udp
 ENTRYPOINT ["geth"]

Dockerfile.mockccc

@@ -2,16 +2,21 @@
 ARG COMMIT=""
 ARG VERSION=""
 ARG BUILDNUM=""
+ARG SCROLL_LIB_PATH=/scroll/lib
 
 # Build Geth in a stock Go builder container
 FROM scrolltech/go-rust-builder:go-1.20-rust-nightly-2022-12-10 as builder
 
 ADD . /go-ethereum
 RUN cd /go-ethereum && env GO111MODULE=on go run build/ci.go install ./cmd/geth
 
+RUN cd /go-ethereum && env GO111MODULE=on go run build/ci.go install ./cmd/geth
+
 # Pull Geth into a second stage deploy alpine container
 FROM ubuntu:20.04
 
+ENV CGO_LDFLAGS="-ldl"
+
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
 EXPOSE 8545 8546 30303 30303/udp

Makefile

@@ -40,7 +40,7 @@ test: all
 	# genesis test
 	cd ${PWD}/cmd/geth; go test -test.run TestCustomGenesis
 	# module test
-	$(GORUN) build/ci.go test ./consensus ./core ./eth ./miner ./node ./trie
+	$(GORUN) build/ci.go test ./consensus ./core ./eth ./miner ./node ./trie ./rollup/...
 
 lint: ## Run linters.
 	$(GORUN) build/ci.go lint

accounts/abi/bind/backends/simulated.go

@@ -640,7 +640,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	vmEnv := vm.NewEVM(evmContext, txContext, stateDB, b.config, vm.Config{NoBaseFee: true})
 	gasPool := new(core.GasPool).AddGas(math.MaxUint64)
 	signer := types.MakeSigner(b.blockchain.Config(), head.Number)
-	l1DataFee, err := fees.EstimateL1DataFeeForMessage(msg, head.BaseFee, b.blockchain.Config().ChainID, signer, stateDB)
+	l1DataFee, err := fees.EstimateL1DataFeeForMessage(msg, head.BaseFee, b.blockchain.Config(), signer, stateDB, head.Number)
 	if err != nil {
 		return nil, err
 	}

build/ci.go

@@ -232,6 +232,9 @@ func doInstall(cmdline []string) {
 	// Show packages during build.
 	gobuild.Args = append(gobuild.Args, "-v")
 
+	// Add -ldl flags for libscroll_zstd.a.
+	gobuild.Args = append(gobuild.Args, "-ldflags", "-extldflags -ldl")
+
 	// Now we choose what we're even building.
 	// Default: collect all 'main' packages in cmd/ and build those.
 	packages := flag.Args()

cmd/evm/internal/t8ntool/execution.go

@@ -148,6 +148,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		chainConfig.DAOForkBlock.Cmp(new(big.Int).SetUint64(pre.Env.Number)) == 0 {
 		misc.ApplyDAOHardFork(statedb)
 	}
+	// Apply Curie hard fork
+	if chainConfig.CurieBlock != nil && chainConfig.CurieBlock.Cmp(new(big.Int).SetUint64(pre.Env.Number)) == 0 {
+		misc.ApplyCurieHardFork(statedb)
+	}
 
 	for i, tx := range txs {
 		msg, err := tx.AsMessage(signer, pre.Env.BaseFee)
@@ -167,7 +171,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		snapshot := statedb.Snapshot()
 		evm := vm.NewEVM(vmContext, txContext, statedb, chainConfig, vmConfig)
 
-		l1DataFee, err := fees.CalculateL1DataFee(tx, statedb)
+		l1DataFee, err := fees.CalculateL1DataFee(tx, statedb, chainConfig, new(big.Int).SetUint64(pre.Env.Number))
 		if err != nil {
 			log.Info("rejected tx due to fees.CalculateL1DataFee", "index", i, "hash", tx.Hash(), "from", msg.From(), "error", err)
 			rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()})

cmd/geth/main.go

@@ -159,6 +159,8 @@ var (
 		utils.GpoPercentileFlag,
 		utils.GpoMaxGasPriceFlag,
 		utils.GpoIgnoreGasPriceFlag,
+		utils.GpoCongestionThresholdFlag,
+
 		utils.MinerNotifyFullFlag,
 		configFileFlag,
 		utils.CatalystFlag,

cmd/geth/usage.go

@@ -204,6 +204,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.GpoPercentileFlag,
 			utils.GpoMaxGasPriceFlag,
 			utils.GpoIgnoreGasPriceFlag,
+			utils.GpoCongestionThresholdFlag,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -735,6 +735,11 @@ var (
 		Usage: "Gas price below which gpo will ignore transactions",
 		Value: ethconfig.Defaults.GPO.IgnorePrice.Int64(),
 	}
+	GpoCongestionThresholdFlag = cli.IntFlag{
+		Name:  "gpo.congestionthreshold",
+		Usage: "Number of pending transactions to consider the network congested and suggest a minimum tip cap",
+		Value: ethconfig.Defaults.GPO.CongestedThreshold,
+	}
 
 	// Metrics flags
 	MetricsEnabledFlag = cli.BoolFlag{
@@ -1429,6 +1434,9 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config, light bool) {
 	if ctx.GlobalIsSet(GpoIgnoreGasPriceFlag.Name) {
 		cfg.IgnorePrice = big.NewInt(ctx.GlobalInt64(GpoIgnoreGasPriceFlag.Name))
 	}
+	if ctx.GlobalIsSet(GpoCongestionThresholdFlag.Name) {
+		cfg.CongestedThreshold = ctx.GlobalInt(GpoCongestionThresholdFlag.Name)
+	}
 }
 
 func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) {

consensus/clique/clique.go

@@ -648,6 +648,8 @@ func (c *Clique) Seal(chain consensus.ChainHeaderReader, block *types.Block, res
 	// Wait until sealing is terminated or delay timeout.
 	log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay))
 	go func() {
+		defer close(results)
+
 		select {
 		case <-stop:
 			return
@@ -656,7 +658,7 @@ func (c *Clique) Seal(chain consensus.ChainHeaderReader, block *types.Block, res
 
 		select {
 		case results <- block.WithSeal(header):
-		default:
+		case <-time.After(time.Second):
 			log.Warn("Sealing result is not read by miner", "sealhash", SealHash(header))
 		}
 	}()

consensus/misc/curie.go

@@ -0,0 +1,23 @@
+package misc
+
+import (
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/core/state"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/scroll-tech/go-ethereum/rollup/rcfg"
+)
+
+// ApplyCurieHardFork modifies the state database according to the Curie hard-fork rules,
+// updating the bytecode and storage of the L1GasPriceOracle contract.
+func ApplyCurieHardFork(statedb *state.StateDB) {
+	log.Info("Applying Curie hard fork")
+
+	// update contract byte code
+	statedb.SetCode(rcfg.L1GasPriceOracleAddress, rcfg.CurieL1GasPriceOracleBytecode)
+
+	// initialize new storage slots
+	statedb.SetState(rcfg.L1GasPriceOracleAddress, rcfg.IsCurieSlot, common.BytesToHash([]byte{1}))
+	statedb.SetState(rcfg.L1GasPriceOracleAddress, rcfg.L1BlobBaseFeeSlot, common.BytesToHash([]byte{1}))
+	statedb.SetState(rcfg.L1GasPriceOracleAddress, rcfg.CommitScalarSlot, common.BigToHash(rcfg.InitialCommitScalar))
+	statedb.SetState(rcfg.L1GasPriceOracleAddress, rcfg.BlobScalarSlot, common.BigToHash(rcfg.InitialBlobScalar))
+}

core/blockchain_test.go

@@ -17,6 +17,7 @@
 package core
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -39,6 +40,7 @@ import (
 	"github.com/scroll-tech/go-ethereum/crypto"
 	"github.com/scroll-tech/go-ethereum/ethdb"
 	"github.com/scroll-tech/go-ethereum/params"
+	"github.com/scroll-tech/go-ethereum/rollup/rcfg"
 	"github.com/scroll-tech/go-ethereum/trie"
 )
 
@@ -3712,3 +3714,78 @@ func TestTransientStorageReset(t *testing.T) {
 		t.Fatalf("Unexpected dirty storage slot")
 	}
 }
+
+func TestCurieTransition(t *testing.T) {
+	// Set fork blocks in config
+	// (we make a deep copy to avoid interference with other tests)
+	var config *params.ChainConfig
+	b, _ := json.Marshal(params.AllEthashProtocolChanges)
+	json.Unmarshal(b, &config)
+	config.CurieBlock = big.NewInt(2)
+	config.DescartesBlock = nil
+
+	var (
+		db      = rawdb.NewMemoryDatabase()
+		gspec   = &Genesis{Config: config}
+		genesis = gspec.MustCommit(db)
+	)
+
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil)
+	defer blockchain.Stop()
+	blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, nil)
+
+	if _, err := blockchain.InsertChain(blocks); err != nil {
+		t.Fatal(err)
+	}
+
+	latestBlock := uint64(4)
+	assert.Equal(t, latestBlock, blockchain.CurrentHeader().Number.Uint64())
+
+	for ii := uint64(0); ii <= latestBlock; ii++ {
+		block := blockchain.GetBlockByNumber(ii)
+
+		number := block.Number().Uint64()
+		baseFee := block.BaseFee()
+
+		statedb, _ := state.New(block.Root(), state.NewDatabase(db), nil)
+
+		code := statedb.GetCode(rcfg.L1GasPriceOracleAddress)
+		codeSize := statedb.GetCodeSize(rcfg.L1GasPriceOracleAddress)
+		keccakCodeHash := statedb.GetKeccakCodeHash(rcfg.L1GasPriceOracleAddress)
+		poseidonCodeHash := statedb.GetPoseidonCodeHash(rcfg.L1GasPriceOracleAddress)
+
+		l1BlobBaseFee := statedb.GetState(rcfg.L1GasPriceOracleAddress, rcfg.L1BlobBaseFeeSlot)
+		commitScalar := statedb.GetState(rcfg.L1GasPriceOracleAddress, rcfg.CommitScalarSlot)
+		blobScalar := statedb.GetState(rcfg.L1GasPriceOracleAddress, rcfg.BlobScalarSlot)
+		isCurie := statedb.GetState(rcfg.L1GasPriceOracleAddress, rcfg.IsCurieSlot)
+
+		if number < config.CurieBlock.Uint64() {
+			assert.Nil(t, baseFee, "Expected zero base fee before Curie")
+
+			// we don't have predeploys configured in this test,
+			// so there is no gas oracle deployed before Curie
+			assert.Nil(t, code)
+			assert.Equal(t, uint64(0), codeSize)
+			assert.Equal(t, common.Hash{}, keccakCodeHash)
+			assert.Equal(t, common.Hash{}, poseidonCodeHash)
+
+			assert.Equal(t, common.Hash{}, l1BlobBaseFee)
+			assert.Equal(t, common.Hash{}, commitScalar)
+			assert.Equal(t, common.Hash{}, blobScalar)
+			assert.Equal(t, common.Hash{}, isCurie)
+		} else {
+			assert.NotNil(t, baseFee, "Expected nonzero base fee after Curie")
+
+			// all gas oracle entries updated
+			assert.NotNil(t, code)
+			assert.NotEqual(t, uint64(0), codeSize)
+			assert.NotEqual(t, common.Hash{}, keccakCodeHash)
+			assert.NotEqual(t, common.Hash{}, poseidonCodeHash)
+
+			assert.NotEqual(t, common.Hash{}, l1BlobBaseFee)
+			assert.NotEqual(t, common.Hash{}, commitScalar)
+			assert.NotEqual(t, common.Hash{}, blobScalar)
+			assert.NotEqual(t, common.Hash{}, isCurie)
+		}
+	}
+}

core/chain_makers.go

@@ -240,6 +240,9 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		if config.DAOForkSupport && config.DAOForkBlock != nil && config.DAOForkBlock.Cmp(b.header.Number) == 0 {
 			misc.ApplyDAOHardFork(statedb)
 		}
+		if config.CurieBlock != nil && config.CurieBlock.Cmp(b.header.Number) == 0 {
+			misc.ApplyCurieHardFork(statedb)
+		}
 		// Execute any user modifications to the block
 		if gen != nil {
 			gen(i, b)

core/state_prefetcher.go

@@ -71,7 +71,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c
 		}
 		statedb.SetTxContext(tx.Hash(), i)
 
-		l1DataFee, err := fees.CalculateL1DataFee(tx, statedb)
+		l1DataFee, err := fees.CalculateL1DataFee(tx, statedb, p.config, block.Number())
 		if err != nil {
 			return
 		}