diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java index afb97728247e..426b17c4dc46 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java @@ -37,7 +37,6 @@ import com.hedera.node.app.fees.congestion.EntityUtilizationMultiplier; import com.hedera.node.app.fees.congestion.ThrottleMultiplier; import com.hedera.node.app.ids.EntityIdService; -import com.hedera.node.app.ids.WritableEntityIdStore; import com.hedera.node.app.info.CurrentPlatformStatusImpl; import com.hedera.node.app.info.NetworkInfoImpl; import com.hedera.node.app.info.SelfNodeInfoImpl; @@ -57,7 +56,6 @@ import com.hedera.node.app.spi.workflows.record.GenesisRecordsBuilder; import com.hedera.node.app.state.HederaState; import com.hedera.node.app.state.merkle.MerkleHederaState; -import com.hedera.node.app.state.merkle.MerkleSchemaRegistry; import com.hedera.node.app.state.recordcache.RecordCacheService; import com.hedera.node.app.throttle.CongestionThrottleService; import com.hedera.node.app.throttle.SynchronizedThrottleAccumulator; @@ -98,7 +96,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.Objects; import java.util.Set; import java.util.function.IntSupplier; import org.apache.logging.log4j.LogManager; @@ -422,53 +419,8 @@ private void onMigrate( final var selfNodeInfo = SelfNodeInfoImpl.of(nodeAddress, version); final var networkInfo = new NetworkInfoImpl(selfNodeInfo, platform, bootstrapConfigProvider); - logger.info("Migrating Entity ID Service as pre-requisite for other services"); - final var entityIdRegistration = servicesRegistry.registrations().stream() - .filter(service -> EntityIdService.NAME.equals(service.service().getServiceName())) - .findFirst() - .orElseThrow(); - final var entityIdRegistry = (MerkleSchemaRegistry) entityIdRegistration.registry(); - entityIdRegistry.migrate( - state, - previousVersion, - currentVersion, - configProvider.getConfiguration(), - networkInfo, - backendThrottle, - // We call with null here because we're migrating the entity ID service itself - null); - // Now that the Entity ID Service is migrated, migrate the remaining services - servicesRegistry.registrations().stream() - .filter(r -> !Objects.equals(entityIdRegistration, r)) - .forEach(registration -> { - // FUTURE We should have metrics here to keep track of how long it takes to migrate each service - final var service = registration.service(); - final var serviceName = service.getServiceName(); - logger.info("Migrating Service {}", serviceName); - final var registry = (MerkleSchemaRegistry) registration.registry(); - - // The token service has a dependency on the entity ID service during genesis migrations, so we - // CAREFULLY create a different WritableStates specific to the entity ID service. The different - // WritableStates instances won't be able to see the changes made by each other, but there shouldn't - // be any conflicting changes. We'll inject this into the MigrationContext below to enable - // generation of entity IDs. - final var entityIdWritableStates = state.createWritableStates(EntityIdService.NAME); - final var entityIdStore = new WritableEntityIdStore(entityIdWritableStates); - - registry.migrate( - state, - previousVersion, - currentVersion, - configProvider.getConfiguration(), - networkInfo, - backendThrottle, - requireNonNull(entityIdStore)); - // Now commit any changes that were made to the entity ID state (since other service entities could - // depend on newly-generated entity IDs) - if (entityIdWritableStates instanceof MerkleHederaState.MerkleWritableStates mws) { - mws.commit(); - } - }); + final var migrator = new OrderedServiceMigrator(servicesRegistry, backendThrottle); + migrator.doMigrations(state, currentVersion, previousVersion, configProvider.getConfiguration(), networkInfo); logger.info("Migration complete"); } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/OrderedServiceMigrator.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/OrderedServiceMigrator.java new file mode 100644 index 000000000000..f19bacf43fb7 --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/OrderedServiceMigrator.java @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app; + +import static java.util.Objects.requireNonNull; + +import com.hedera.hapi.node.base.SemanticVersion; +import com.hedera.node.app.ids.EntityIdService; +import com.hedera.node.app.ids.WritableEntityIdStore; +import com.hedera.node.app.service.token.impl.TokenServiceImpl; +import com.hedera.node.app.services.ServicesRegistry; +import com.hedera.node.app.spi.info.NetworkInfo; +import com.hedera.node.app.spi.state.SchemaRegistry; +import com.hedera.node.app.state.merkle.MerkleHederaState; +import com.hedera.node.app.state.merkle.MerkleSchemaRegistry; +import com.hedera.node.app.throttle.ThrottleAccumulator; +import com.hedera.node.config.VersionedConfiguration; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.util.Comparator; +import java.util.Objects; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * The entire purpose of this class is to ensure that inter-service dependencies are respected between + * migrations. The only required dependency right now is the {@link EntityIdService}, which is needed + * for genesis blocklist accounts in the token service genesis migration. (See {@link + * TokenServiceImpl#registerSchemas(SchemaRegistry)}). + * + *
Note: there are only two ordering requirements to maintain: first, that the entity ID service
+ * is migrated before the token service; and second, that the remaining services are migrated _in any
+ * deterministic order_. In order to ensure the entity ID service is migrated before the token service,
+ * we'll just migrate the entity ID service first.
+ */
+public class OrderedServiceMigrator {
+ private static final Logger logger = LogManager.getLogger(OrderedServiceMigrator.class);
+ private final ServicesRegistry servicesRegistry;
+ private final ThrottleAccumulator backendThrottle;
+
+ public OrderedServiceMigrator(
+ @NonNull final ServicesRegistry servicesRegistry, @NonNull final ThrottleAccumulator backendThrottle) {
+ this.servicesRegistry = requireNonNull(servicesRegistry);
+ this.backendThrottle = requireNonNull(backendThrottle);
+ }
+
+ /**
+ * Migrates the services registered with the {@link ServicesRegistry}
+ */
+ public void doMigrations(
+ @NonNull final MerkleHederaState state,
+ @NonNull final SemanticVersion currentVersion,
+ @Nullable final SemanticVersion previousVersion,
+ @NonNull final VersionedConfiguration versionedConfiguration,
+ @NonNull final NetworkInfo networkInfo) {
+ requireNonNull(state);
+ requireNonNull(currentVersion);
+ requireNonNull(versionedConfiguration);
+ requireNonNull(networkInfo);
+
+ logger.info("Migrating Entity ID Service as pre-requisite for other services");
+ final var entityIdRegistration = servicesRegistry.registrations().stream()
+ .filter(service -> EntityIdService.NAME.equals(service.service().getServiceName()))
+ .findFirst()
+ .orElseThrow();
+ final var entityIdRegistry = (MerkleSchemaRegistry) entityIdRegistration.registry();
+ entityIdRegistry.migrate(
+ state,
+ previousVersion,
+ currentVersion,
+ versionedConfiguration,
+ networkInfo,
+ backendThrottle,
+ // We call with null here because we're migrating the entity ID service itself
+ null);
+
+ // Now that the Entity ID Service is migrated, migrate the remaining services in name order. Note: the name
+ // ordering itself isn't important, just that the ordering is deterministic
+ servicesRegistry.registrations().stream()
+ .filter(r -> !Objects.equals(entityIdRegistration, r))
+ .sorted(Comparator.comparing(
+ (ServicesRegistry.Registration r) -> r.service().getServiceName()))
+ .forEach(registration -> {
+ // FUTURE We should have metrics here to keep track of how long it takes to
+ // migrate each service
+ final var service = registration.service();
+ final var serviceName = service.getServiceName();
+ logger.info("Migrating Service {}", serviceName);
+ final var registry = (MerkleSchemaRegistry) registration.registry();
+
+ // The token service has a dependency on the entity ID service during genesis migrations, so we
+ // CAREFULLY create a different WritableStates specific to the entity ID service. The different
+ // WritableStates instances won't be able to "see" the changes made by each other, meaning that a
+ // change made with WritableStates instance X would _not_ be read by a separate WritableStates
+ // instance Y. However, since the inter-service dependencies are limited to the EntityIdService,
+ // there shouldn't be any changes made in any single WritableStates instance that would need to be
+ // read by any other separate WritableStates instances. This should hold true as long as the
+ // EntityIdService is not directly injected into any genesis generation code. Instead, we'll inject
+ // this entity ID writable states instance into the MigrationContext below, to enable generation of
+ // entity IDs through an appropriate API.
+ final var entityIdWritableStates = state.createWritableStates(EntityIdService.NAME);
+ final var entityIdStore = new WritableEntityIdStore(entityIdWritableStates);
+
+ registry.migrate(
+ state,
+ previousVersion,
+ currentVersion,
+ versionedConfiguration,
+ networkInfo,
+ backendThrottle,
+ // If we have reached this point in the code, entityIdStore should not be null because the
+ // EntityIdService should have been migrated already. We enforce with requireNonNull in case
+ // there are scenarios we haven't considered.
+ requireNonNull(entityIdStore));
+ // Now commit any changes that were made to the entity ID state (since other service entities could
+ // depend on newly-generated entity IDs)
+ if (entityIdWritableStates instanceof MerkleHederaState.MerkleWritableStates mws) {
+ mws.commit();
+ }
+ });
+ }
+}
diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/state/merkle/DependencyMigrationTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/state/merkle/DependencyMigrationTest.java
new file mode 100644
index 000000000000..dd116398a493
--- /dev/null
+++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/state/merkle/DependencyMigrationTest.java
@@ -0,0 +1,305 @@
+/*
+ * Copyright (C) 2023 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hedera.node.app.state.merkle;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+
+import com.hedera.hapi.node.base.SemanticVersion;
+import com.hedera.hapi.node.state.common.EntityNumber;
+import com.hedera.node.app.OrderedServiceMigrator;
+import com.hedera.node.app.ids.EntityIdService;
+import com.hedera.node.app.services.ServicesRegistryImpl;
+import com.hedera.node.app.spi.Service;
+import com.hedera.node.app.spi.fixtures.state.NoOpGenesisRecordsBuilder;
+import com.hedera.node.app.spi.info.NetworkInfo;
+import com.hedera.node.app.spi.state.MigrationContext;
+import com.hedera.node.app.spi.state.Schema;
+import com.hedera.node.app.spi.state.SchemaRegistry;
+import com.hedera.node.app.spi.state.StateDefinition;
+import com.hedera.node.app.spi.state.WritableStates;
+import com.hedera.node.app.throttle.ThrottleAccumulator;
+import com.hedera.node.config.VersionedConfigImpl;
+import com.swirlds.common.constructable.ConstructableRegistry;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import org.assertj.core.api.Assertions;
+import org.jetbrains.annotations.NotNull;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@ExtendWith(MockitoExtension.class)
+class DependencyMigrationTest extends MerkleTestBase {
+ private static final long INITIAL_ENTITY_ID = 5;
+
+ @Mock
+ private ThrottleAccumulator accumulator;
+
+ @Mock
+ private VersionedConfigImpl versionedConfig;
+
+ @Mock
+ private NetworkInfo networkInfo;
+
+ private MerkleHederaState merkleTree;
+
+ @BeforeEach
+ void setUp() {
+ registry = mock(ConstructableRegistry.class);
+ merkleTree = new MerkleHederaState((tree, state) -> {}, (e, m, s) -> {}, (s, p, ds, t, dv) -> {});
+ }
+
+ @Nested
+ @SuppressWarnings("DataFlowIssue")
+ final class ConstructorTests {
+ @Test
+ void servicesRegistryRequired() {
+ Assertions.assertThatThrownBy(() -> new OrderedServiceMigrator(null, accumulator))
+ .isInstanceOf(NullPointerException.class);
+ }
+
+ @Test
+ void throttleAccumulatorRequired() {
+ Assertions.assertThatThrownBy(() -> new OrderedServiceMigrator(mock(ServicesRegistryImpl.class), null))
+ .isInstanceOf(NullPointerException.class);
+ }
+ }
+
+ @Nested
+ @SuppressWarnings("DataFlowIssue")
+ @ExtendWith(MockitoExtension.class)
+ final class DoMigrationsNullParams {
+ @Mock
+ private ServicesRegistryImpl servicesRegistry;
+
+ @Test
+ void stateRequired() {
+ final var subject = new OrderedServiceMigrator(servicesRegistry, accumulator);
+ Assertions.assertThatThrownBy(() ->
+ subject.doMigrations(null, SemanticVersion.DEFAULT, null, versionedConfig, networkInfo))
+ .isInstanceOf(NullPointerException.class);
+ }
+
+ @Test
+ void currentVersionRequired() {
+ final var subject = new OrderedServiceMigrator(servicesRegistry, accumulator);
+ Assertions.assertThatThrownBy(
+ () -> subject.doMigrations(merkleTree, null, null, versionedConfig, networkInfo))
+ .isInstanceOf(NullPointerException.class);
+ }
+
+ @Test
+ void versionedConfigRequired() {
+ final var subject = new OrderedServiceMigrator(servicesRegistry, accumulator);
+ Assertions.assertThatThrownBy(
+ () -> subject.doMigrations(merkleTree, SemanticVersion.DEFAULT, null, null, networkInfo))
+ .isInstanceOf(NullPointerException.class);
+ }
+
+ @Test
+ void networkInfoRequired() {
+ final var subject = new OrderedServiceMigrator(servicesRegistry, accumulator);
+ Assertions.assertThatThrownBy(() ->
+ subject.doMigrations(merkleTree, SemanticVersion.DEFAULT, null, versionedConfig, null))
+ .isInstanceOf(NullPointerException.class);
+ }
+ }
+
+ @Test
+ @DisplayName("Genesis inter-service dependency migration works")
+ void genesisWithNullVersion() {
+ // Given: register the EntityIdService and the DependentService (order of registration shouldn't matter)
+ final var servicesRegistry = new ServicesRegistryImpl(registry, new NoOpGenesisRecordsBuilder());
+ final var entityService = new EntityIdService() {
+ @Override
+ public void registerSchemas(@NonNull SchemaRegistry registry) {
+ registry.register(new Schema(SemanticVersion.DEFAULT) {
+ @NonNull
+ @Override
+ public Set