Merge branch 'develop' into 9659-CryptoTransferSuite

.github/workflows/node-zxc-build-release-artifact.yaml

@@ -157,7 +157,7 @@ jobs:
           fi
 
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Setup Java
         uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0
@@ -243,7 +243,7 @@ jobs:
           echo "::endgroup::"
 
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Authenticate to Google Cloud
         uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
@@ -389,7 +389,7 @@ jobs:
       - build-artifact
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Authenticate to Google Cloud
         id: google-auth
@@ -517,7 +517,7 @@ jobs:
       - validate
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Install GnuPG Tools
         if: ${{ inputs.dry-run-enabled != true }}
@@ -609,7 +609,7 @@ jobs:
     if: ${{ inputs.sdk-release-profile != 'none' }}
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Install GnuPG Tools
         if: ${{ inputs.dry-run-enabled != true }}

.github/workflows/node-zxc-compile-application-code.yaml

@@ -117,7 +117,7 @@ jobs:
     runs-on: [self-hosted, Linux, large, ephemeral]
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Expand Shallow Clone for SonarQube and Spotless
         if: ${{ (inputs.enable-sonar-analysis || inputs.enable-unit-tests || inputs.enable-spotless-check) && !cancelled() }}

.github/workflows/node-zxcron-release-branching.yaml

@@ -41,7 +41,7 @@ jobs:
       triggered: ${{ steps.evaluate.outputs.triggered }}
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Read Trigger Time
         id: time
@@ -89,7 +89,7 @@ jobs:
     if: ${{ needs.check-trigger.outputs.triggered == 'true' && !cancelled() }}
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Branch Creation Check
         id: branch-creation
@@ -105,7 +105,7 @@ jobs:
     if: ${{ needs.check-branch.outputs.branch-create == 'true' && !cancelled() }}
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           fetch-depth: 0
           token: ${{ secrets.GH_ACCESS_TOKEN }}
@@ -194,7 +194,7 @@ jobs:
     if: ${{ needs.check-branch.outputs.tag-create == 'true' && needs.create-branch.result == 'success' && !cancelled() }}
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           ref: ${{ needs.check-branch.outputs.branch-name }}
           fetch-depth: 0

.github/workflows/node-zxcron-release-fsts-regression.yaml

@@ -33,7 +33,7 @@ jobs:
     runs-on: [self-hosted, Linux, small, scheduler, ephemeral]
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           fetch-depth: 0
 

.github/workflows/node-zxf-snyk-monitor.yaml

@@ -32,7 +32,7 @@ jobs:
     runs-on: [self-hosted, Linux, medium, ephemeral]
     steps:
       - name: Checkout
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Setup Java
         uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0

.github/workflows/platform-zxcron-release-jrs-regression.yaml

@@ -33,7 +33,7 @@ jobs:
     runs-on: [self-hosted, Linux, small, scheduler, ephemeral]
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           fetch-depth: 0
 

.github/workflows/zxc-jrs-regression.yaml

@@ -165,7 +165,7 @@ jobs:
     runs-on: [self-hosted, Linux, large, ephemeral]
     steps:
       - name: Checkout Platform Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           ref: ${{ inputs.ref || inputs.branch-name || '' }}
           fetch-depth: 0
@@ -182,7 +182,7 @@ jobs:
           echo "branch-name=${BRANCH_NAME}" >> "${GITHUB_OUTPUT}"
 
       - name: Checkout Regression Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           path: platform-sdk/regression
           repository: swirlds/swirlds-platform-regression
@@ -256,7 +256,7 @@ jobs:
           fi
 
       - name: Install SSH key
-        uses: shimataro/ssh-key-action@685d0f20da72e4b53cc81d373a2ed0a867770e46 # v2.5.1
+        uses: shimataro/ssh-key-action@38b53cb2f445ea2e0eb8872407e366677c41dbc6 # v2.6.1
         with:
           name: jrs-ssh-keyfile
           key: ${{ secrets.jrs-ssh-key-file }}

hedera-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/state/MigrationContext.java

@@ -77,4 +77,10 @@ public interface MigrationContext {
 
     @NonNull
     HandleThrottleParser handleThrottling();
+
+    /**
+     * Consumes and returns the next entity number. For use by migrations that need to create entities.
+     * @return the next entity number
+     */
+    long newEntityNum();
 }

hedera-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/workflows/HandleContext.java

@@ -430,6 +430,37 @@ <T> T dispatchReversiblePrecedingTransaction(
             @NonNull Predicate<Key> verifier,
             AccountID syntheticPayer);
 
+    /**
+     * Dispatches preceding transaction that can be removed.
+     *
+     * <p>A removable preceding transaction depends on the current transaction. That means if the user transaction
+     * fails, a removable preceding transaction is automatically removed and not exported. The state changes introduced by a
+     * removable preceding transaction are automatically committed together with the parent transaction.
+     *
+     * <p>This method can only be called by a {@link TransactionCategory#USER}-transaction and only as long as no state
+     * changes have been introduced by the user transaction (either by storing state or by calling a child
+     * transaction).
+     *
+     * <p>The provided {@link Predicate} callback will be called to verify simple keys when the child transaction calls
+     * any of the {@code verificationFor} methods.
+     *
+     * @param txBody             the {@link TransactionBody} of the transaction to dispatch
+     * @param recordBuilderClass the record builder class of the transaction
+     * @param verifier           a {@link Predicate} that will be used to validate primitive keys
+     * @param syntheticPayer    the payer of the transaction
+     * @return the record builder of the transaction
+     * @throws NullPointerException     if {@code txBody} is {@code null}
+     * @throws IllegalArgumentException if the transaction is not a {@link TransactionCategory#USER}-transaction or if
+     *                                  the record builder type is unknown to the app
+     * @throws IllegalStateException    if the current transaction has already introduced state changes
+     */
+    @NonNull
+    <T> T dispatchRemovablePrecedingTransaction(
+            @NonNull TransactionBody txBody,
+            @NonNull Class<T> recordBuilderClass,
+            @NonNull Predicate<Key> verifier,
+            AccountID syntheticPayer);
+
     /**
      * Dispatches a reversible preceding transaction that already has an ID.
      *
@@ -603,7 +634,7 @@ default <T> T dispatchRemovableChildTransaction(
     SavepointStack savepointStack();
 
     /**
-     * Revert all child records in {@link RecordListBuilder}.
+     * Revert all child records in RecordListBuilder.
      */
     void revertChildRecords();
 

hedera-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/workflows/record/GenesisRecordsBuilder.java

@@ -45,4 +45,9 @@ public interface GenesisRecordsBuilder {
      * Tracks the treasury clones created during node startup
      */
     void treasuryClones(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
+
+    /**
+     * Tracks the blocklist accounts created during node startup
+     */
+    void blocklistAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts);
 }

hedera-node/hedera-app-spi/src/testFixtures/java/com/hedera/node/app/spi/fixtures/state/NoOpGenesisRecordsBuilder.java

@@ -58,4 +58,9 @@ public void miscAccounts(@NonNull final Map<Account, CryptoCreateTransactionBody
     public void treasuryClones(@NonNull final Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
         // Intentional no-op
     }
+
+    @Override
+    public void blocklistAccounts(@NonNull Map<Account, CryptoCreateTransactionBody.Builder> accounts) {
+        // Intentional no-op
+    }
 }

hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java

@@ -37,6 +37,7 @@
 import com.hedera.node.app.fees.congestion.EntityUtilizationMultiplier;
 import com.hedera.node.app.fees.congestion.ThrottleMultiplier;
 import com.hedera.node.app.ids.EntityIdService;
+import com.hedera.node.app.ids.WritableEntityIdStore;
 import com.hedera.node.app.info.CurrentPlatformStatusImpl;
 import com.hedera.node.app.info.NetworkInfoImpl;
 import com.hedera.node.app.info.SelfNodeInfoImpl;
@@ -97,6 +98,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.Objects;
 import java.util.Set;
 import java.util.function.IntSupplier;
 import org.apache.logging.log4j.LogManager;
@@ -240,9 +242,11 @@ public Hedera(@NonNull final ConstructableRegistry constructableRegistry) {
 
         // Create all the service implementations
         logger.info("Registering services");
+
         // FUTURE: Use the service loader framework to load these services!
         this.servicesRegistry = new ServicesRegistryImpl(constructableRegistry, genesisRecordsBuilder);
         Set.of(
+                        new EntityIdService(),
                         new ConsensusServiceImpl(),
                         CONTRACT_SERVICE,
                         new FileServiceImpl(),
@@ -253,7 +257,6 @@ public Hedera(@NonNull final ConstructableRegistry constructableRegistry) {
                         new UtilServiceImpl(),
                         new RecordCacheService(),
                         new BlockRecordService(),
-                        new EntityIdService(),
                         new FeeService(),
                         new CongestionThrottleService())
                 .forEach(servicesRegistry::register);
@@ -418,20 +421,54 @@ private void onMigrate(
         final var nodeAddress = platform.getAddressBook().getAddress(selfId);
         final var selfNodeInfo = SelfNodeInfoImpl.of(nodeAddress, version);
         final var networkInfo = new NetworkInfoImpl(selfNodeInfo, platform, bootstrapConfigProvider);
-        for (final var registration : servicesRegistry.registrations()) {
-            // FUTURE We should have metrics here to keep track of how long it takes to migrate each service
-            final var service = registration.service();
-            final var serviceName = service.getServiceName();
-            logger.info("Migrating Service {}", serviceName);
-            final var registry = (MerkleSchemaRegistry) registration.registry();
-            registry.migrate(
-                    state,
-                    previousVersion,
-                    currentVersion,
-                    configProvider.getConfiguration(),
-                    networkInfo,
-                    backendThrottle);
-        }
+
+        logger.info("Migrating Entity ID Service as pre-requisite for other services");
+        final var entityIdRegistration = servicesRegistry.registrations().stream()
+                .filter(service -> EntityIdService.NAME.equals(service.service().getServiceName()))
+                .findFirst()
+                .orElseThrow();
+        final var entityIdRegistry = (MerkleSchemaRegistry) entityIdRegistration.registry();
+        entityIdRegistry.migrate(
+                state,
+                previousVersion,
+                currentVersion,
+                configProvider.getConfiguration(),
+                networkInfo,
+                backendThrottle,
+                // We call with null here because we're migrating the entity ID service itself
+                null);
+        // Now that the Entity ID Service is migrated, migrate the remaining services
+        servicesRegistry.registrations().stream()
+                .filter(r -> !Objects.equals(entityIdRegistration, r))
+                .forEach(registration -> {
+                    // FUTURE We should have metrics here to keep track of how long it takes to migrate each service
+                    final var service = registration.service();
+                    final var serviceName = service.getServiceName();
+                    logger.info("Migrating Service {}", serviceName);
+                    final var registry = (MerkleSchemaRegistry) registration.registry();
+
+                    // The token service has a dependency on the entity ID service during genesis migrations, so we
+                    // CAREFULLY create a different WritableStates specific to the entity ID service. The different
+                    // WritableStates instances won't be able to see the changes made by each other, but there shouldn't
+                    // be any conflicting changes. We'll inject this into the MigrationContext below to enable
+                    // generation of entity IDs.
+                    final var entityIdWritableStates = state.createWritableStates(EntityIdService.NAME);
+                    final var entityIdStore = new WritableEntityIdStore(entityIdWritableStates);
+
+                    registry.migrate(
+                            state,
+                            previousVersion,
+                            currentVersion,
+                            configProvider.getConfiguration(),
+                            networkInfo,
+                            backendThrottle,
+                            requireNonNull(entityIdStore));
+                    // Now commit any changes that were made to the entity ID state (since other service entities could
+                    // depend on newly-generated entity IDs)
+                    if (entityIdWritableStates instanceof MerkleHederaState.MerkleWritableStates mws) {
+                        mws.commit();
+                    }
+                });
         logger.info("Migration complete");
     }
 

hedera-node/hedera-app/src/main/java/com/hedera/node/app/ids/EntityIdService.java

@@ -34,8 +34,8 @@
 @SuppressWarnings("rawtypes")
 public class EntityIdService implements Service {
     public static final String NAME = "EntityIdService";
+    public static final String ENTITY_ID_STATE_KEY = "ENTITY_ID";
     private static final SemanticVersion GENESIS_VERSION = SemanticVersion.DEFAULT;
-    static final String ENTITY_ID_STATE_KEY = "ENTITY_ID";
 
     /** {@inheritDoc} */
     @NonNull

hedera-node/hedera-app/src/main/java/com/hedera/node/app/state/merkle/MerkleSchemaRegistry.java

@@ -20,6 +20,7 @@
 import static java.util.Objects.requireNonNull;
 
 import com.hedera.hapi.node.base.SemanticVersion;
+import com.hedera.node.app.ids.WritableEntityIdStore;
 import com.hedera.node.app.spi.HapiUtils;
 import com.hedera.node.app.spi.Service;
 import com.hedera.node.app.spi.info.NetworkInfo;
@@ -154,7 +155,8 @@ public void migrate(
             @NonNull final SemanticVersion currentVersion,
             @NonNull final Configuration config,
             @NonNull final NetworkInfo networkInfo,
-            @NonNull final HandleThrottleParser handleThrottling) {
+            @NonNull final HandleThrottleParser handleThrottling,
+            @Nullable final WritableEntityIdStore entityIdStore) {
         requireNonNull(hederaState);
         requireNonNull(currentVersion);
         requireNonNull(config);
@@ -165,6 +167,7 @@ public void migrate(
         // of those schemas, create the new states and remove the old states and migrate the data.
         final var schemasToApply = computeApplicableSchemas(previousVersion, currentVersion);
         final var updateInsteadOfMigrate = isSameVersion(previousVersion, currentVersion);
+
         for (final var schema : schemasToApply) {
             // Now we can migrate the schema and then commit all the changes
             // We just have one merkle tree -- the just-loaded working tree -- to work from.
@@ -221,13 +224,25 @@ public void migrate(
             remainingStates.removeAll(statesToRemove);
             final var newStates = new FilteredWritableStates(writeableStates, remainingStates);
 
+            // For any changes to state that depend on other services outside the current service, we need a reference
+            // to the overall state that we can pass into the context. This reference to overall state will be strictly
+            // controlled via the MigrationContext API so that only changes explicitly specified in the interface can be
+            // made (instead of allowing any arbitrary change to overall state). As above, we won't commit anything
+            // until after this service's migration
             final var migrationContext = new MigrationContextImpl(
-                    previousStates, newStates, config, networkInfo, genesisRecordsBuilder, handleThrottling);
+                    previousStates,
+                    newStates,
+                    config,
+                    networkInfo,
+                    genesisRecordsBuilder,
+                    handleThrottling,
+                    entityIdStore);
             if (updateInsteadOfMigrate) {
                 schema.restart(migrationContext);
             } else {
                 schema.migrate(migrationContext);
             }
+            // Now commit all the service-specific changes made during this service's update or migration
             if (writeableStates instanceof MerkleHederaState.MerkleWritableStates mws) {
                 mws.commit();
             }