From 6bcf2c725e631474769807ae4f9edd36b591472f Mon Sep 17 00:00:00 2001 From: lyfsn Date: Mon, 15 Apr 2024 19:16:14 +0800 Subject: [PATCH 1/2] optimize node startup speed and memory allocation Signed-off-by: lyfsn --- .../condition/clique/CliqueConditions.java | 2 +- .../org/hyperledger/besu/cli/BesuCommand.java | 49 +++++-- .../besu/cli/config/EthNetworkConfig.java | 2 +- .../operator/GenerateBlockchainConfig.java | 4 +- .../besu/controller/BesuController.java | 24 +++- .../besu/config/GenesisConfigFile.java | 10 ++ .../org/hyperledger/besu/config/JsonUtil.java | 135 ++++++++++++++++++ .../hyperledger/besu/config/JsonUtilTest.java | 23 +++ .../api/jsonrpc/BlockchainImporter.java | 2 +- 9 files changed, 237 insertions(+), 14 deletions(-) diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/clique/CliqueConditions.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/clique/CliqueConditions.java index 750e587f6e3..bb1b7fb9dff 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/clique/CliqueConditions.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/clique/CliqueConditions.java @@ -89,7 +89,7 @@ public Condition awaitSignerSetChange(final Node node) { private int cliqueBlockPeriod(final BesuNode node) { final String config = node.getGenesisConfigProvider().create(emptyList()).get(); - final GenesisConfigFile genesisConfigFile = GenesisConfigFile.fromConfig(config); + final GenesisConfigFile genesisConfigFile = GenesisConfigFile.fromConfigWithoutAccounts(config); final CliqueConfigOptions cliqueConfigOptions = genesisConfigFile.getConfigOptions().getCliqueConfigOptions(); return cliqueConfigOptions.getBlockPeriodSeconds(); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 9648122de3b..9244911eae1 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -91,6 +91,7 @@ import org.hyperledger.besu.config.CheckpointConfigOptions; import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.config.GenesisConfigOptions; +import org.hyperledger.besu.config.JsonUtil; import org.hyperledger.besu.config.MergeConfigOptions; import org.hyperledger.besu.consensus.qbft.pki.PkiBlockCreationConfiguration; import org.hyperledger.besu.consensus.qbft.pki.PkiBlockCreationConfigurationProvider; @@ -117,6 +118,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.chain.VariablesStorage; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.MiningParametersMetrics; import org.hyperledger.besu.ethereum.core.PrivacyParameters; @@ -1598,7 +1600,8 @@ private void validateChainDataPruningParams() { private GenesisConfigOptions readGenesisConfigOptions() { try { - final GenesisConfigFile genesisConfigFile = GenesisConfigFile.fromConfig(genesisConfig()); + final GenesisConfigFile genesisConfigFile = + GenesisConfigFile.fromConfigWithoutAccounts(genesisConfig()); genesisConfigOptions = genesisConfigFile.getConfigOptions(genesisConfigOverrides); } catch (final Exception e) { throw new ParameterException( @@ -2352,16 +2355,46 @@ private EthNetworkConfig updateNetworkConfig(final NetworkName network) { } private GenesisConfigFile getGenesisConfigFile() { - return GenesisConfigFile.fromConfig(genesisConfig()); + return GenesisConfigFile.fromConfigWithoutAccounts(genesisConfig()); } + private String genesisConfigString = ""; + private String genesisConfig() { - try { - return Resources.toString(genesisFile.toURI().toURL(), UTF_8); - } catch (final IOException e) { - throw new ParameterException( - this.commandLine, String.format("Unable to load genesis URL %s.", genesisFile), e); + if (!genesisConfigString.isEmpty()) { + return genesisConfigString; + } + if (genesisStateHashCacheEnabled) { + // If the genesis state hash is present in the database, we can use the genesis file without + pluginCommonConfiguration.init( + dataDir(), + dataDir().resolve(DATABASE_PATH), + getDataStorageConfiguration(), + getMiningParameters()); + final KeyValueStorageProvider storageProvider = keyValueStorageProvider(keyValueStorageName); + if (storageProvider != null) { + boolean isGenesisStateHashPresent; + try { + // A null pointer exception may be thrown here if the database is not initialized. + VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); + Optional genesisStateHash = variablesStorage.getGenesisStateHash(); + isGenesisStateHashPresent = genesisStateHash.isPresent(); + } catch (Exception ignored) { + isGenesisStateHashPresent = false; + } + if (isGenesisStateHashPresent) { + genesisConfigString = JsonUtil.getJsonFromFileWithout(genesisFile, "alloc"); + } + } + } + if (genesisConfigString.isEmpty()) { + try { + genesisConfigString = Resources.toString(genesisFile.toURI().toURL(), UTF_8); + } catch (Exception e) { + throw new RuntimeException(e); + } } + return genesisConfigString; } private static String genesisConfig(final NetworkName networkName) { @@ -2607,7 +2640,7 @@ protected GenesisConfigOptions getActualGenesisConfigOptions() { return Optional.ofNullable(genesisConfigOptions) .orElseGet( () -> - GenesisConfigFile.fromConfig( + GenesisConfigFile.fromConfigWithoutAccounts( genesisConfig(Optional.ofNullable(network).orElse(MAINNET))) .getConfigOptions(genesisConfigOverrides)); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java b/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java index 3af85016c8c..2844e1b9be2 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java @@ -138,7 +138,7 @@ public String toString() { public static EthNetworkConfig getNetworkConfig(final NetworkName networkName) { final String genesisContent = jsonConfig(networkName.getGenesisFile()); final GenesisConfigOptions genesisConfigOptions = - GenesisConfigFile.fromConfig(genesisContent).getConfigOptions(); + GenesisConfigFile.fromConfigWithoutAccounts(genesisContent).getConfigOptions(); final Optional> rawBootNodes = genesisConfigOptions.getDiscoveryOptions().getBootNodes(); final List bootNodes = diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java index f53f3a80574..eccab65ea75 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java @@ -283,7 +283,9 @@ private void parseConfig() throws IOException { /** Sets the selected signature algorithm instance in SignatureAlgorithmFactory. */ private void processEcCurve() { - GenesisConfigOptions options = GenesisConfigFile.fromConfig(genesisConfig).getConfigOptions(); + GenesisConfigOptions options = + GenesisConfigFile.fromConfigWithoutAccounts(String.valueOf(genesisConfig)) + .getConfigOptions(); Optional ecCurve = options.getEcCurve(); if (ecCurve.isEmpty()) { diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuController.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuController.java index 2849846f213..594cdfb40bb 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuController.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuController.java @@ -332,6 +332,25 @@ public BesuControllerBuilder fromEthNetworkConfig( .networkId(ethNetworkConfig.getNetworkId()); } + /** + * From eth network config without alloc besu controller builder. + * + * @param ethNetworkConfig the eth network config + * @param genesisConfigOverrides the genesis config overrides + * @param syncMode The sync mode + * @return the besu controller builder + */ + public BesuControllerBuilder fromEthNetworkConfigWithoutAccounts( + final EthNetworkConfig ethNetworkConfig, + final Map genesisConfigOverrides, + final SyncMode syncMode) { + return fromGenesisConfig( + GenesisConfigFile.fromConfigWithoutAccounts(ethNetworkConfig.getGenesisConfig()), + genesisConfigOverrides, + syncMode) + .networkId(ethNetworkConfig.getNetworkId()); + } + /** * From genesis config besu controller builder. * @@ -390,8 +409,9 @@ BesuControllerBuilder fromGenesisConfig( return new TransitionBesuControllerBuilder(builder, new MergeBesuControllerBuilder()) .genesisConfigFile(genesisConfig); } - - } else return builder.genesisConfigFile(genesisConfig); + } else { + return builder.genesisConfigFile(genesisConfig); + } } private BesuControllerBuilder createConsensusScheduleBesuControllerBuilder( diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java b/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java index 638d221e0ad..da68f93d08f 100644 --- a/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java +++ b/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java @@ -106,6 +106,16 @@ public static GenesisConfigFile fromConfig(final String jsonString) { return fromConfig(JsonUtil.objectNodeFromString(jsonString, false)); } + /** + * From config without account genesis config file. + * + * @param jsonString the json string + * @return the genesis config file + */ + public static GenesisConfigFile fromConfigWithoutAccounts(final String jsonString) { + return fromConfig(JsonUtil.objectNodeFromStringWithout(jsonString, false, "alloc")); + } + /** * From config genesis config file. * diff --git a/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java b/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java index ef19bc7b835..0332bd13d2f 100644 --- a/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java +++ b/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.util.number.PositiveNumber; +import java.io.File; import java.io.IOException; import java.util.Locale; import java.util.Map; @@ -23,8 +24,11 @@ import java.util.OptionalInt; import java.util.OptionalLong; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonParser.Feature; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -321,6 +325,47 @@ public static ObjectNode objectNodeFromString( } } + /** + * Object node from string without some field. + * + * @param jsonData the json data + * @param allowComments true to allow comments + * @param withoutField the without field + * @return the object node + */ + public static ObjectNode objectNodeFromStringWithout( + final String jsonData, final boolean allowComments, final String withoutField) { + final ObjectMapper objectMapper = new ObjectMapper(); + JsonFactory jsonFactory = + JsonFactory.builder() + .configure(JsonFactory.Feature.INTERN_FIELD_NAMES, false) + .configure(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES, false) + .build(); + jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, allowComments); + + ObjectNode root = objectMapper.createObjectNode(); + + try (JsonParser jp = jsonFactory.createParser(jsonData)) { + if (jp.nextToken() != JsonToken.START_OBJECT) { + throw new RuntimeException("Expected data to start with an Object"); + } + + while (jp.nextToken() != JsonToken.END_OBJECT) { + String fieldName = jp.getCurrentName(); + if (withoutField.equals(fieldName)) { + jp.nextToken(); + jp.skipChildren(); + } else { + jp.nextToken(); + root.set(fieldName, objectMapper.readTree(jp)); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return root; + } + /** * Gets json. * @@ -466,4 +511,94 @@ private static boolean validateInt(final JsonNode node) { } return true; } + + /** + * Get the JSON representation of a genesis file without a specific field. + * + * @param genesisFile The genesis file to read. + * @param excludedFieldName The field to exclude from the JSON representation. + * @return The JSON representation of the genesis file without the excluded field. + */ + public static String getJsonFromFileWithout( + final File genesisFile, final String excludedFieldName) { + StringBuilder jsonBuilder = new StringBuilder(); + JsonFactory jsonFactory = + JsonFactory.builder() + .configure(JsonFactory.Feature.INTERN_FIELD_NAMES, false) + .configure(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES, false) + .build(); + try (JsonParser parser = jsonFactory.createParser(genesisFile)) { + JsonToken token; + while ((token = parser.nextToken()) != null) { + if (token == JsonToken.START_OBJECT) { + jsonBuilder.append(handleObject(parser, excludedFieldName)); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return jsonBuilder.toString(); + } + + private static String handleObject(final JsonParser parser, final String excludedFieldName) + throws IOException { + StringBuilder objectBuilder = new StringBuilder(); + objectBuilder.append("{"); + String fieldName; + boolean isFirstField = true; + while (parser.nextToken() != JsonToken.END_OBJECT) { + fieldName = parser.getCurrentName(); + if (fieldName != null && fieldName.equals(excludedFieldName)) { + parser.skipChildren(); // Skip this field + continue; + } + if (!isFirstField) objectBuilder.append(", "); + parser.nextToken(); // move to value + objectBuilder + .append("\"") + .append(fieldName) + .append("\":") + .append(handleValue(parser, excludedFieldName)); + isFirstField = false; + } + objectBuilder.append("}"); + return objectBuilder.toString(); + } + + private static String handleValue(final JsonParser parser, final String excludedFieldName) + throws IOException { + JsonToken token = parser.getCurrentToken(); + switch (token) { + case START_OBJECT: + return handleObject(parser, excludedFieldName); + case START_ARRAY: + return handleArray(parser, excludedFieldName); + case VALUE_STRING: + return "\"" + parser.getText() + "\""; + case VALUE_NUMBER_INT: + case VALUE_NUMBER_FLOAT: + return parser.getNumberValue().toString(); + case VALUE_TRUE: + case VALUE_FALSE: + return parser.getBooleanValue() ? "true" : "false"; + case VALUE_NULL: + return "null"; + default: + throw new IllegalStateException("Unrecognized token: " + token); + } + } + + private static String handleArray(final JsonParser parser, final String excludedFieldName) + throws IOException { + StringBuilder arrayBuilder = new StringBuilder(); + arrayBuilder.append("["); + boolean isFirstElement = true; + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (!isFirstElement) arrayBuilder.append(", "); + arrayBuilder.append(handleValue(parser, excludedFieldName)); + isFirstElement = false; + } + arrayBuilder.append("]"); + return arrayBuilder.toString(); + } } diff --git a/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java b/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java index 962bb42f055..68b4c98e4ef 100644 --- a/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java +++ b/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java @@ -16,7 +16,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import java.io.File; import java.util.List; import java.util.Locale; import java.util.Map; @@ -781,4 +783,25 @@ public void hasKey_nonEmptyMatchingKey() { assertThat(JsonUtil.hasKey(rootNode, "target")).isTrue(); } + + @Test + void objectNodeFromStringWithoutAllocField() { + String genesisSting = + "{\"coinbase\":\"0x0000000000000000000000000000000000000000\",\"alloc\":{\"000d836201318ec6899a67540690382780743280\":{\"balance\":\"0xad78ebc5ac6200000\"}}}"; + + ObjectNode jsonNodes = JsonUtil.objectNodeFromStringWithout(genesisSting, false, "alloc"); + + assertThat(jsonNodes.get("coinbase").toString()).isNotEmpty(); + assertThat(jsonNodes.get("alloc")).isNull(); + } + + @Test + void getJsonFromFileWithoutAllocField() { + String genesisStingWithoutAllocField = + "{\"config\":{\"chainId\":1337, \"londonBlock\":0, \"phillyBlock\":5, \"parisBlock\":10, \"contractSizeLimit\":2147483647, \"ethash\":{\"fixeddifficulty\":100}}, \"nonce\":\"0x42\", \"timestamp\":\"0x0\", \"extraData\":\"0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa\", \"gasLimit\":\"0x1fffffffffffff\", \"difficulty\":\"0x10000\", \"mixHash\":\"0x0000000000000000000000000000000000000000000000000000000000000000\", \"coinbase\":\"0x0000000000000000000000000000000000000000\"}"; + File genesisFile = new File("src/test/resources/preMerge.json"); + + String genesisStringFromFile = JsonUtil.getJsonFromFileWithout(genesisFile, "alloc"); + assertEquals(genesisStingWithoutAllocField, genesisStringFromFile); + } } diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/BlockchainImporter.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/BlockchainImporter.java index 2a62331d336..af0a7db1959 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/BlockchainImporter.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/BlockchainImporter.java @@ -44,7 +44,7 @@ public class BlockchainImporter { public BlockchainImporter(final URL blocksUrl, final String genesisJson) throws Exception { protocolSchedule = MainnetProtocolSchedule.fromConfig( - GenesisConfigFile.fromConfig(genesisJson).getConfigOptions(), + GenesisConfigFile.fromConfigWithoutAccounts(genesisJson).getConfigOptions(), MiningParameters.newDefault(), new BadBlockManager()); final BlockHeaderFunctions blockHeaderFunctions = From 8ed9e4c32f203f8bb75479015d3d92e49e6b98b6 Mon Sep 17 00:00:00 2001 From: lyfsn Date: Tue, 16 Apr 2024 20:09:55 +0800 Subject: [PATCH 2/2] genesisConfigString use Suppliers.memoize Signed-off-by: lyfsn --- .../org/hyperledger/besu/cli/BesuCommand.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 9244911eae1..87e9dfc56b0 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2358,12 +2358,9 @@ private GenesisConfigFile getGenesisConfigFile() { return GenesisConfigFile.fromConfigWithoutAccounts(genesisConfig()); } - private String genesisConfigString = ""; + private final Supplier genesisConfigSupplier = Suppliers.memoize(this::loadGenesisConfig); - private String genesisConfig() { - if (!genesisConfigString.isEmpty()) { - return genesisConfigString; - } + private String loadGenesisConfig() { if (genesisStateHashCacheEnabled) { // If the genesis state hash is present in the database, we can use the genesis file without pluginCommonConfiguration.init( @@ -2383,18 +2380,19 @@ private String genesisConfig() { isGenesisStateHashPresent = false; } if (isGenesisStateHashPresent) { - genesisConfigString = JsonUtil.getJsonFromFileWithout(genesisFile, "alloc"); + return JsonUtil.getJsonFromFileWithout(genesisFile, "alloc"); } } } - if (genesisConfigString.isEmpty()) { - try { - genesisConfigString = Resources.toString(genesisFile.toURI().toURL(), UTF_8); - } catch (Exception e) { - throw new RuntimeException(e); - } + try { + return Resources.toString(genesisFile.toURI().toURL(), UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); } - return genesisConfigString; + } + + private String genesisConfig() { + return genesisConfigSupplier.get(); } private static String genesisConfig(final NetworkName networkName) {