run_block_generator2() (#16241)

.github/workflows/benchmarks.yml

@@ -34,7 +34,7 @@ jobs:
         python-version: [ 3.9 ]
     env:
       CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet
-      BLOCKS_AND_PLOTS_VERSION: 0.30.0
+      BLOCKS_AND_PLOTS_VERSION: 0.32.0
 
     steps:
       - name: Clean workspace

.github/workflows/test-single.yml

@@ -113,7 +113,7 @@ jobs:
       CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet
       CHIA_SIMULATOR_ROOT: ${{ github.workspace }}/.chia/simulator
       JOB_FILE_NAME: tests_${{ matrix.os.file_name }}_python-${{ matrix.python.file_name }}_${{ matrix.configuration.name }}
-      BLOCKS_AND_PLOTS_VERSION: 0.30.0
+      BLOCKS_AND_PLOTS_VERSION: 0.32.0
 
     steps:
       - name: Configure git

chia/consensus/constants.py

@@ -74,6 +74,7 @@ class ConsensusConstants:
     # the hard fork planned with the 2.0 release
     # this is the block with the first plot filter adjustment
     HARD_FORK_HEIGHT: uint32
+    HARD_FORK_FIX_HEIGHT: uint32
 
     # the plot filter adjustment heights
     PLOT_FILTER_128_HEIGHT: uint32

chia/consensus/default_constants.py

@@ -61,6 +61,7 @@
     # June 2024
     "SOFT_FORK4_HEIGHT": 5496000,
     "HARD_FORK_HEIGHT": 5496000,
+    "HARD_FORK_FIX_HEIGHT": 5496000,
     # June 2027
     "PLOT_FILTER_128_HEIGHT": 10542000,
     # June 2030

chia/full_node/mempool_check_conditions.py

@@ -18,7 +18,7 @@
     NO_RELATIVE_CONDITIONS_ON_EPHEMERAL,
 )
 from chia_rs import get_puzzle_and_solution_for_coin as get_puzzle_and_solution_for_coin_rust
-from chia_rs import run_block_generator, run_chia_program
+from chia_rs import run_block_generator, run_block_generator2, run_chia_program
 from clvm.casts import int_from_bytes
 
 from chia.consensus.constants import ConsensusConstants
@@ -51,6 +51,8 @@ def get_name_puzzle_conditions(
     height: uint32,
     constants: ConsensusConstants,
 ) -> NPCResult:
+    run_block = run_block_generator
+
     flags = 0
     if mempool_mode:
         flags = flags | MEMPOOL_MODE
@@ -91,9 +93,12 @@ def get_name_puzzle_conditions(
             | ALLOW_BACKREFS
         )
 
+    if height >= constants.HARD_FORK_FIX_HEIGHT:
+        run_block = run_block_generator2
+
     try:
         block_args = [bytes(gen) for gen in generator.generator_refs]
-        err, result = run_block_generator(bytes(generator.program), block_args, max_cost, flags)
+        err, result = run_block(bytes(generator.program), block_args, max_cost, flags)
         assert (err is None) != (result is None)
         if err is not None:
             return NPCResult(uint16(err), None, uint64(0))

chia/server/start_full_node.py

@@ -81,6 +81,8 @@ def update_testnet_overrides(network_id: str, overrides: Dict[str, Any]) -> None
         overrides["SOFT_FORK4_HEIGHT"] = 2997292
     if "HARD_FORK_HEIGHT" not in overrides:
         overrides["HARD_FORK_HEIGHT"] = 2997292
+    if "HARD_FORK_FIX_HEIGHT" not in overrides:
+        overrides["HARD_FORK_FIX_HEIGHT"] = 3426000
     if "PLOT_FILTER_128_HEIGHT" not in overrides:
         overrides["PLOT_FILTER_128_HEIGHT"] = 3061804
     if "PLOT_FILTER_64_HEIGHT" not in overrides:

chia/simulator/block_tools.py

@@ -130,6 +130,10 @@
     "rom_bootstrap_generator.clsp", package_or_requirement="chia.consensus.puzzles"
 )
 
+DESERIALIZE_MOD = load_serialized_clvm_maybe_recompile(
+    "chialisp_deserialisation.clsp", package_or_requirement="chia.consensus.puzzles"
+)
+
 test_constants = DEFAULT_CONSTANTS.replace(
     **{
         "MIN_PLOT_SIZE": 18,
@@ -151,6 +155,10 @@
         "MAX_FUTURE_TIME2": 3600 * 24 * 10,
         "MEMPOOL_BLOCK_BUFFER": 6,
         "UNIQUE_PLOTS_WINDOW": 2,
+        # we deliberately make this different from HARD_FORK_HEIGHT in the
+        # tests, to ensure they operate independently (which they need to do for
+        # testnet10)
+        "HARD_FORK_FIX_HEIGHT": 5496100,
     }
 )
 
@@ -1836,27 +1844,40 @@ def conditions_cost(conds: Program, hard_fork: bool) -> uint64:
     return uint64(condition_cost)
 
 
-def compute_cost_test(generator: BlockGenerator, cost_per_byte: int, hard_fork: bool = False) -> uint64:
+def compute_cost_test(generator: BlockGenerator, constants: ConsensusConstants, height: uint32) -> uint64:
     # this function cannot *validate* the block or any of the transactions. We
     # deliberately create invalid blocks as parts of the tests, and we still
     # need to be able to compute the cost of it
 
     condition_cost = 0
     clvm_cost = 0
 
-    flags = MEMPOOL_MODE
-    if hard_fork:
-        flags |= ALLOW_BACKREFS
-    block_program_args = Program.to([[bytes(g) for g in generator.generator_refs]])
-    clvm_cost, result = GENERATOR_MOD._run(INFINITE_COST, flags, generator.program, block_program_args)
+    if height >= constants.HARD_FORK_FIX_HEIGHT:
+        blocks = [bytes(g) for g in generator.generator_refs]
+        cost, result = generator.program._run(INFINITE_COST, MEMPOOL_MODE | ALLOW_BACKREFS, DESERIALIZE_MOD, blocks)
+        clvm_cost += cost
 
-    for res in result.first().as_iter():
-        res = res.rest()  # skip parent coin id
-        res = res.rest()  # skip puzzle hash
-        res = res.rest()  # skip amount
-        condition_cost += conditions_cost(res.first(), hard_fork)
+        for spend in result.first().as_iter():
+            # each spend is a list of:
+            # (parent-coin-id puzzle amount solution)
+            puzzle = spend.at("rf")
+            solution = spend.at("rrrf")
 
-    size_cost = len(bytes(generator.program)) * cost_per_byte
+            cost, result = puzzle._run(INFINITE_COST, MEMPOOL_MODE, solution)
+            clvm_cost += cost
+            condition_cost += conditions_cost(result, height >= constants.HARD_FORK_HEIGHT)
+
+    else:
+        block_program_args = Program.to([[bytes(g) for g in generator.generator_refs]])
+        clvm_cost, result = GENERATOR_MOD._run(INFINITE_COST, MEMPOOL_MODE, generator.program, block_program_args)
+
+        for res in result.first().as_iter():
+            # each condition item is:
+            # (parent-coin-id puzzle-hash amount conditions)
+            conditions = res.at("rrrf")
+            condition_cost += conditions_cost(conditions, height >= constants.HARD_FORK_HEIGHT)
+
+    size_cost = len(bytes(generator.program)) * constants.COST_PER_BYTE
 
     return uint64(clvm_cost + size_cost + condition_cost)
 
@@ -1953,9 +1974,7 @@ def create_test_foliage(
         # Calculate the cost of transactions
         if block_generator is not None:
             generator_block_heights_list = block_generator.block_height_list
-            cost = compute_cost_test(
-                block_generator, constants.COST_PER_BYTE, hard_fork=height >= constants.HARD_FORK_HEIGHT
-            )
+            cost = compute_cost_test(block_generator, constants, height)
 
             removal_amount = 0
             addition_amount = 0

chia/util/initial-config.yaml

@@ -77,6 +77,8 @@ network_overrides: &network_overrides
       # planned 2.0 release is July 26, height 2965036 on testnet
       # 1 week later
       HARD_FORK_HEIGHT: 2997292
+      # November 2023
+      HARD_FORK_FIX_HEIGHT: 3426000
       # another 2 weeks later
       PLOT_FILTER_128_HEIGHT: 3061804
       # 3 years later

tests/conftest.py

@@ -124,7 +124,11 @@ def blockchain_constants(consensus_mode) -> ConsensusConstants:
         return test_constants.replace(SOFT_FORK3_HEIGHT=3, SOFT_FORK4_HEIGHT=3)
     if consensus_mode == ConsensusMode.HARD_FORK_2_0:
         return test_constants.replace(
-            HARD_FORK_HEIGHT=2, PLOT_FILTER_128_HEIGHT=10, PLOT_FILTER_64_HEIGHT=15, PLOT_FILTER_32_HEIGHT=20
+            HARD_FORK_HEIGHT=2,
+            HARD_FORK_FIX_HEIGHT=2,
+            PLOT_FILTER_128_HEIGHT=10,
+            PLOT_FILTER_64_HEIGHT=15,
+            PLOT_FILTER_32_HEIGHT=20,
         )
     raise AssertionError("Invalid Blockchain mode in simulation")
 
@@ -177,7 +181,7 @@ def db_version(request) -> int:
     return request.param
 
 
-SOFTFORK_HEIGHTS = [1000000, 4510000, 5496000]
+SOFTFORK_HEIGHTS = [1000000, 4510000, 5496000, 5496100]
 
 
 @pytest.fixture(scope="function", params=SOFTFORK_HEIGHTS)
@@ -193,6 +197,8 @@ def default_400_blocks(bt, consensus_mode):
     version = ""
     if consensus_mode == ConsensusMode.SOFT_FORK4:
         version = "_softfork3"
+    if consensus_mode == ConsensusMode.HARD_FORK_2_0:
+        version = "_hardfork"
 
     from tests.util.blockchain import persistent_blocks
 
@@ -204,6 +210,8 @@ def default_1000_blocks(bt, consensus_mode):
     version = ""
     if consensus_mode == ConsensusMode.SOFT_FORK4:
         version = "_softfork3"
+    if consensus_mode == ConsensusMode.HARD_FORK_2_0:
+        version = "_hardfork"
 
     from tests.util.blockchain import persistent_blocks
 
@@ -215,6 +223,8 @@ def pre_genesis_empty_slots_1000_blocks(bt, consensus_mode):
     version = ""
     if consensus_mode == ConsensusMode.SOFT_FORK4:
         version = "_softfork3"
+    if consensus_mode == ConsensusMode.HARD_FORK_2_0:
+        version = "_hardfork"
 
     from tests.util.blockchain import persistent_blocks
 
@@ -232,6 +242,8 @@ def default_1500_blocks(bt, consensus_mode):
     version = ""
     if consensus_mode == ConsensusMode.SOFT_FORK4:
         version = "_softfork3"
+    if consensus_mode == ConsensusMode.HARD_FORK_2_0:
+        version = "_hardfork"
 
     from tests.util.blockchain import persistent_blocks
 
@@ -245,14 +257,20 @@ def default_10000_blocks(bt, consensus_mode):
     if consensus_mode == ConsensusMode.SOFT_FORK4:
         pytest.skip("Test cache not available yet")
 
-    return persistent_blocks(10000, f"test_blocks_10000_{saved_blocks_version}.db", bt, seed=b"10000")
+    version = ""
+    if consensus_mode == ConsensusMode.HARD_FORK_2_0:
+        version = "_hardfork"
+
+    return persistent_blocks(10000, f"test_blocks_10000_{saved_blocks_version}{version}.db", bt, seed=b"10000")
 
 
 @pytest.fixture(scope="session")
 def test_long_reorg_blocks(bt, consensus_mode, default_1500_blocks):
     version = ""
     if consensus_mode == ConsensusMode.SOFT_FORK4:
         version = "_softfork3"
+    if consensus_mode == ConsensusMode.HARD_FORK_2_0:
+        version = "_hardfork"
 
     from tests.util.blockchain import persistent_blocks
 
@@ -271,6 +289,8 @@ def default_2000_blocks_compact(bt, consensus_mode):
     version = ""
     if consensus_mode == ConsensusMode.SOFT_FORK4:
         version = "_softfork3"
+    if consensus_mode == ConsensusMode.HARD_FORK_2_0:
+        version = "_hardfork"
 
     from tests.util.blockchain import persistent_blocks
 
@@ -292,9 +312,14 @@ def default_10000_blocks_compact(bt, consensus_mode):
 
     if consensus_mode == ConsensusMode.SOFT_FORK4:
         pytest.skip("Test cache not available yet")
+
+    version = ""
+    if consensus_mode == ConsensusMode.HARD_FORK_2_0:
+        version = "_hardfork"
+
     return persistent_blocks(
         10000,
-        f"test_blocks_10000_compact_{saved_blocks_version}.db",
+        f"test_blocks_10000_compact_{saved_blocks_version}{version}.db",
         bt,
         normalized_to_identity_cc_eos=True,
         normalized_to_identity_icc_eos=True,

tests/core/full_node/test_conditions.py

@@ -151,7 +151,12 @@ async def test_unknown_conditions_with_cost(
             # before the hard fork, all unknown conditions have 0 cost
             expected_cost = 0
 
-        block_base_cost = 761056
+        # once the hard fork activates, blocks no longer pay the cost of the ROM
+        # generator (which includes hashing all puzzles).
+        if consensus_mode == ConsensusMode.HARD_FORK_2_0:
+            block_base_cost = 756064
+        else:
+            block_base_cost = 761056
         assert new_block.transactions_info is not None
         assert new_block.transactions_info.cost - block_base_cost == expected_cost
 
@@ -170,11 +175,15 @@ async def test_softfork_condition(self, condition: str, expected_cost: int, bt,
         if consensus_mode != ConsensusMode.HARD_FORK_2_0:
             # the SOFTFORK condition is not recognized before the hard fork
             expected_cost = 0
+            block_base_cost = 737056
+        else:
+            # once the hard fork activates, blocks no longer pay the cost of the ROM
+            # generator (which includes hashing all puzzles).
+            block_base_cost = 732064
 
-        # this includes the cost of the bytes for the condition with 2 bytes
-        # argument. This test works as long as the conditions it's parameterized
-        # on has the same size
-        block_base_cost = 737056
+        # the block_base_cost includes the cost of the bytes for the condition
+        # with 2 bytes argument. This test works as long as the conditions it's
+        # parameterized on has the same size
         assert new_block.transactions_info is not None
         assert new_block.transactions_info.cost - block_base_cost == expected_cost
 

tests/core/mempool/test_mempool.py

@@ -12,7 +12,6 @@
 
 from chia.consensus.condition_costs import ConditionCost
 from chia.consensus.cost_calculator import NPCResult
-from chia.consensus.default_constants import DEFAULT_CONSTANTS
 from chia.full_node.bitcoin_fee_estimator import create_bitcoin_fee_estimator
 from chia.full_node.fee_estimation import EmptyMempoolInfo, MempoolInfo
 from chia.full_node.full_node_api import FullNodeAPI
@@ -24,6 +23,7 @@
 from chia.protocols.wallet_protocol import TransactionAck
 from chia.server.outbound_message import Message
 from chia.server.ws_connection import WSChiaConnection
+from chia.simulator.block_tools import test_constants
 from chia.simulator.simulator_protocol import FarmNewBlockProtocol
 from chia.simulator.time_out_assert import time_out_assert
 from chia.simulator.wallet_tools import WalletTool
@@ -1966,7 +1966,7 @@ def generator_condition_tester(
     generator = BlockGenerator(program, [], [])
     print(f"len: {len(bytes(program))}")
     npc_result: NPCResult = get_name_puzzle_conditions(
-        generator, max_cost, mempool_mode=mempool_mode, height=height, constants=DEFAULT_CONSTANTS
+        generator, max_cost, mempool_mode=mempool_mode, height=height, constants=test_constants
     )
     return npc_result
 
@@ -2001,7 +2001,7 @@ def test_div(self, mempool, operand, expected, softfork_height):
         )
 
         # with the 2.0 hard fork, division with negative numbers is allowed
-        if operand < 0 and softfork_height >= DEFAULT_CONSTANTS.HARD_FORK_HEIGHT:
+        if operand < 0 and softfork_height >= test_constants.HARD_FORK_HEIGHT:
             expected = None
 
         assert npc_result.error == expected
@@ -2098,7 +2098,10 @@ def test_create_coin_cost(self, softfork_height):
         # CREATE_COIN
         puzzle_hash = "abababababababababababababababab"
 
-        generator_base_cost = 20470
+        if softfork_height >= test_constants.HARD_FORK_FIX_HEIGHT:
+            generator_base_cost = 40
+        else:
+            generator_base_cost = 20470
 
         # this max cost is exactly enough for the create coin condition
         npc_result = generator_condition_tester(
@@ -2123,7 +2126,10 @@ def test_agg_sig_cost(self, softfork_height):
         # AGG_SIG_ME
         pubkey = "abababababababababababababababababababababababab"
 
-        generator_base_cost = 20512
+        if softfork_height >= test_constants.HARD_FORK_FIX_HEIGHT:
+            generator_base_cost = 40
+        else:
+            generator_base_cost = 20512
 
         # this max cost is exactly enough for the AGG_SIG condition
         npc_result = generator_condition_tester(
@@ -2155,7 +2161,7 @@ def test_create_coin_different_parent(self, softfork_height):
         )
         generator = BlockGenerator(program, [], [])
         npc_result: NPCResult = get_name_puzzle_conditions(
-            generator, MAX_BLOCK_COST_CLVM, mempool_mode=False, height=softfork_height, constants=DEFAULT_CONSTANTS
+            generator, MAX_BLOCK_COST_CLVM, mempool_mode=False, height=softfork_height, constants=test_constants
         )
         assert npc_result.error is None
         assert len(npc_result.conds.spends) == 2
@@ -2239,7 +2245,7 @@ def test_softfork_condition(
             expect_error = Err.INVALID_CONDITION.value
         # the SOFTFORK condition is only activated with the hard fork, so
         # before then there are no errors
-        elif softfork_height < DEFAULT_CONSTANTS.HARD_FORK_HEIGHT:
+        elif softfork_height < test_constants.HARD_FORK_HEIGHT:
             expect_error = None
 
         assert npc_result.error == expect_error
@@ -2484,7 +2490,7 @@ def test_duplicate_reserve_fee_negative(self, request: pytest.FixtureRequest, so
     def test_duplicate_coin_announces(self, request, opcode, softfork_height):
         # with soft-fork3, we only allow 1024 create- or assert announcements
         # per spend
-        if softfork_height >= DEFAULT_CONSTANTS.SOFT_FORK3_HEIGHT:
+        if softfork_height >= test_constants.SOFT_FORK3_HEIGHT:
             condition = CREATE_ANNOUNCE_COND.format(opcode=opcode.value[0], num=1024)
         else:
             condition = CREATE_ANNOUNCE_COND.format(opcode=opcode.value[0], num=5950000)