Merge branch 'develop' into 10450-use-address-book-to-determine-start…

…up-node-id
hashgraph · Dec 18, 2023 · 70fc065 · 70fc065
2 parents 45980bc + 50a7d5c
commit 70fc065
Show file tree

Hide file tree

Showing 112 changed files with 1,465 additions and 545 deletions.
diff --git a/.github/workflows/flow-node-performance-tests.yaml b/.github/workflows/flow-node-performance-tests.yaml
@@ -17,7 +17,7 @@
 name: "Node: Performance Tests"
 on:
   schedule:
-    - cron: '0 */12 * * *'
+    - cron: '0 */16 * * *'
   workflow_dispatch:
 
 defaults:

diff --git a/.github/workflows/node-flow-fsts-daily-interval-04.yaml b/.github/workflows/node-flow-fsts-daily-interval-04.yaml
@@ -42,11 +42,11 @@ concurrency:
 #   "configs/services/suites/daily/GCP-Daily-Services-Comp-3NReconnect-15N-4C.json"                   ----         90m
 #   "configs/services/suites/daily/GCP-Daily-Services-Global-3NReconnect-15N-4C.json"                 ----         90m
 
-#   TOTAL RUN TIME:             180 minutes, 3 hours
+#   TOTAL RUN TIME:             90 minutes, 1.5 hours
 #
 #   Parallel job(s):
 #
-#     comp-3nreconnect-15n-4c -> global-3nreconnect-15n-4c
+#     comp-3nreconnect-15n-4c
 
 
 jobs:
@@ -71,24 +71,24 @@ jobs:
     if: ${{ !cancelled() && always() }}
 
 
-  global-3nreconnect-15n-4c:
-    name: Global-3NReconnect-15N-4C
-    uses: ./.github/workflows/zxc-jrs-regression.yaml
-    with:
-      ref: ${{ github.event.inputs.ref }}
-      branch-name: ${{ github.event.inputs.branch-name }}
-      hedera-tests-enabled: true
-      use-branch-for-slack-channel: true
-      panel-config: "configs/services/suites/daily/GCP-Daily-Services-Global-3NReconnect-15N-4C.json"
-    secrets:
-      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
-      jrs-ssh-user-name: ${{ secrets.PLATFORM_JRS_SSH_USER_NAME }}
-      jrs-ssh-key-file: ${{ secrets.PLATFORM_JRS_SSH_KEY_FILE }}
-      gcp-project-number: ${{ secrets.PLATFORM_GCP_PROJECT_NUMBER }}
-      gcp-sa-key-contents: ${{ secrets.PLATFORM_GCP_KEY_FILE }}
-      slack-api-token: ${{ secrets.PLATFORM_SLACK_API_TOKEN }}
-      gradle-cache-username: ${{ secrets.GRADLE_CACHE_USERNAME }}
-      gradle-cache-password: ${{ secrets.GRADLE_CACHE_PASSWORD }}
-    needs:
-      - comp-3nreconnect-15n-4c
-    if: ${{ !cancelled() && always() }}
+#  global-3nreconnect-15n-4c:
+#    name: Global-3NReconnect-15N-4C
+#    uses: ./.github/workflows/zxc-jrs-regression.yaml
+#    with:
+#      ref: ${{ github.event.inputs.ref }}
+#      branch-name: ${{ github.event.inputs.branch-name }}
+#      hedera-tests-enabled: true
+#      use-branch-for-slack-channel: true
+#      panel-config: "configs/services/suites/daily/GCP-Daily-Services-Global-3NReconnect-15N-4C.json"
+#    secrets:
+#      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
+#      jrs-ssh-user-name: ${{ secrets.PLATFORM_JRS_SSH_USER_NAME }}
+#      jrs-ssh-key-file: ${{ secrets.PLATFORM_JRS_SSH_KEY_FILE }}
+#      gcp-project-number: ${{ secrets.PLATFORM_GCP_PROJECT_NUMBER }}
+#      gcp-sa-key-contents: ${{ secrets.PLATFORM_GCP_KEY_FILE }}
+#      slack-api-token: ${{ secrets.PLATFORM_SLACK_API_TOKEN }}
+#      gradle-cache-username: ${{ secrets.GRADLE_CACHE_USERNAME }}
+#      gradle-cache-password: ${{ secrets.GRADLE_CACHE_PASSWORD }}
+#    needs:
+#      - comp-3nreconnect-15n-4c
+#    if: ${{ !cancelled() && always() }}
diff --git a/.github/workflows/node-flow-fsts-daily-regression.yaml b/.github/workflows/node-flow-fsts-daily-regression.yaml
@@ -43,37 +43,37 @@ jobs:
     secrets:
       access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
 
-  interval-02:
-    name: Interval 2
-    uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
-    needs:
-      - interval-01
-    with:
-      ref: ${{ github.event.inputs.ref || github.sha }}
-      branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
-      workflow-file: node-flow-fsts-daily-interval-02.yaml
-      concurrency-group: ${{ github.event.inputs.ref || github.sha }}-node-flow-fsts-daily-group-02
-    secrets:
-      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
+#  interval-02:
+#    name: Interval 2
+#    uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
+#    needs:
+#      - interval-01
+#    with:
+#      ref: ${{ github.event.inputs.ref || github.sha }}
+#      branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
+#      workflow-file: node-flow-fsts-daily-interval-02.yaml
+#      concurrency-group: ${{ github.event.inputs.ref || github.sha }}-node-flow-fsts-daily-group-02
+#    secrets:
+#      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
 
-  interval-03:
-    name: Interval 3
-    uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
-    needs:
-      - interval-02
-    with:
-      ref: ${{ github.event.inputs.ref || github.sha }}
-      branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
-      workflow-file: node-flow-fsts-daily-interval-03.yaml
-      concurrency-group: ${{ github.event.inputs.ref || github.sha }}-node-flow-fsts-daily-group-03
-    secrets:
-      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
+#  interval-03:
+#    name: Interval 3
+#    uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
+#    needs:
+#      - interval-02
+#    with:
+#      ref: ${{ github.event.inputs.ref || github.sha }}
+#      branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
+#      workflow-file: node-flow-fsts-daily-interval-03.yaml
+#      concurrency-group: ${{ github.event.inputs.ref || github.sha }}-node-flow-fsts-daily-group-03
+#    secrets:
+#      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
 
   interval-04:
     name: Interval 4
     uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
     needs:
-      - interval-03
+      - interval-01
     with:
       ref: ${{ github.event.inputs.ref || github.sha }}
       branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
@@ -82,28 +82,28 @@ jobs:
     secrets:
       access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
 
-  interval-05:
-    name: Interval 5
-    uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
-    needs:
-      - interval-04
-    with:
-      ref: ${{ github.event.inputs.ref || github.sha }}
-      branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
-      workflow-file: node-flow-fsts-daily-interval-05.yaml
-      concurrency-group: ${{ github.event.inputs.ref || github.sha }}-node-flow-fsts-daily-group-05
-    secrets:
-      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
+#  interval-05:
+#    name: Interval 5
+#    uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
+#    needs:
+#      - interval-04
+#    with:
+#      ref: ${{ github.event.inputs.ref || github.sha }}
+#      branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
+#      workflow-file: node-flow-fsts-daily-interval-05.yaml
+#      concurrency-group: ${{ github.event.inputs.ref || github.sha }}-node-flow-fsts-daily-group-05
+#    secrets:
+#      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
 
-  interval-06:
-    name: Interval 6
-    uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
-    needs:
-      - interval-05
-    with:
-      ref: ${{ github.event.inputs.ref || github.sha }}
-      branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
-      workflow-file: node-flow-fsts-daily-interval-06.yaml
-      concurrency-group: ${{ github.event.inputs.ref || github.sha }}-node-flow-fsts-daily-group-06
-    secrets:
-      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
+#  interval-06:
+#    name: Interval 6
+#    uses: ./.github/workflows/platform-zxc-launch-jrs-workflow.yaml
+#    needs:
+#      - interval-05
+#    with:
+#      ref: ${{ github.event.inputs.ref || github.sha }}
+#      branch-name: ${{ github.event.inputs.branch-name || github.ref_name }}
+#      workflow-file: node-flow-fsts-daily-interval-06.yaml
+#      concurrency-group: ${{ github.event.inputs.ref || github.sha }}-node-flow-fsts-daily-group-06
+#    secrets:
+#      access-token: ${{ secrets.PLATFORM_GH_ACCESS_TOKEN }}
diff --git a/...ra-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/workflows/HandleContext.java b/...ra-node/hedera-app-spi/src/main/java/com/hedera/node/app/spi/workflows/HandleContext.java
@@ -37,6 +37,7 @@
 import com.hedera.node.app.spi.validation.AttributeValidator;
 import com.hedera.node.app.spi.validation.ExpiryValidator;
 import com.hedera.node.app.spi.workflows.record.ExternalizedRecordCustomizer;
+import com.hedera.node.app.spi.workflows.record.RecordListCheckPoint;
 import com.hedera.pbj.runtime.io.buffer.Bytes;
 import com.swirlds.config.api.Configuration;
 import edu.umd.cs.findbugs.annotations.NonNull;
@@ -396,7 +397,9 @@ <T> T dispatchPrecedingTransaction(
      * @throws IllegalArgumentException if the transaction body did not have an id
      */
     default <T> T dispatchPrecedingTransaction(
-            @NonNull TransactionBody txBody, @NonNull Class<T> recordBuilderClass, @NonNull Predicate<Key> verifier) {
+            @NonNull final TransactionBody txBody,
+            @NonNull final Class<T> recordBuilderClass,
+            @NonNull final Predicate<Key> verifier) {
         throwIfMissingPayerId(txBody);
         return dispatchPrecedingTransaction(
                 txBody,
@@ -478,7 +481,9 @@ <T> T dispatchRemovablePrecedingTransaction(
      * @throws IllegalArgumentException if the transaction body did not have an id
      */
     default <T> T dispatchReversiblePrecedingTransaction(
-            @NonNull TransactionBody txBody, @NonNull Class<T> recordBuilderClass, @NonNull Predicate<Key> verifier) {
+            @NonNull final TransactionBody txBody,
+            @NonNull final Class<T> recordBuilderClass,
+            @NonNull final Predicate<Key> verifier) {
         throwIfMissingPayerId(txBody);
         return dispatchReversiblePrecedingTransaction(
                 txBody,
@@ -536,7 +541,9 @@ <T> T dispatchChildTransaction(
      */
     @NonNull
     default <T> T dispatchScheduledChildTransaction(
-            @NonNull TransactionBody txBody, @NonNull Class<T> recordBuilderClass, @NonNull Predicate<Key> callback) {
+            @NonNull final TransactionBody txBody,
+            @NonNull final Class<T> recordBuilderClass,
+            @NonNull final Predicate<Key> callback) {
         throwIfMissingPayerId(txBody);
         return dispatchChildTransaction(
                 txBody,
@@ -588,7 +595,9 @@ <T> T dispatchRemovableChildTransaction(
      */
     @NonNull
     default <T> T dispatchRemovableChildTransaction(
-            @NonNull TransactionBody txBody, @NonNull Class<T> recordBuilderClass, @NonNull Predicate<Key> callback) {
+            @NonNull final TransactionBody txBody,
+            @NonNull final Class<T> recordBuilderClass,
+            @NonNull final Predicate<Key> callback) {
         throwIfMissingPayerId(txBody);
         return dispatchRemovableChildTransaction(
                 txBody,
@@ -647,9 +656,18 @@ default <T> T dispatchRemovableChildTransaction(
     SavepointStack savepointStack();
 
     /**
-     * Revert all child records in RecordListBuilder.
+     * Revert the childRecords from the checkpoint.
      */
-    void revertChildRecords();
+    void revertRecordsFrom(@NonNull RecordListCheckPoint recordListCheckPoint);
+
+    /**
+     * Create a checkpoint for the current childRecords.
+     *
+     * @return the checkpoint for the current childRecords, containing the first preceding record and the last following
+     * record.
+     */
+    @NonNull
+    RecordListCheckPoint createRecordListCheckPoint();
 
     /**
      * A stack of savepoints.

diff --git a/...-app-spi/src/main/java/com/hedera/node/app/spi/workflows/record/RecordListCheckPoint.java b/...-app-spi/src/main/java/com/hedera/node/app/spi/workflows/record/RecordListCheckPoint.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2023 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hedera.node.app.spi.workflows.record;
+
+import edu.umd.cs.findbugs.annotations.Nullable;
+
+public record RecordListCheckPoint(
+        @Nullable SingleTransactionRecordBuilder firstPrecedingRecord,
+        @Nullable SingleTransactionRecordBuilder lastFollowingRecord) {}
diff --git a/hedera-node/hedera-app/build.gradle.kts b/hedera-node/hedera-app/build.gradle.kts
@@ -141,32 +141,6 @@ tasks.jar {
     }
 }
 
-// Define build directory
-val buildDir = layout.projectDirectory.dir("./build/node")
-
-// Copy everything from hedera-node/data
-val copyNodeData =
-    tasks.register<Sync>("copyNodeData") {
-        dependsOn(copyLib)
-        dependsOn(copyApp)
-        from(layout.projectDirectory.dir("../data/keys")) { into("data/keys") }
-        from(layout.projectDirectory.dir("../data"))
-        into(buildDir)
-        exclude("config", "keys") // Exclude config directory
-        shouldRunAfter(tasks.named("copyApp"))
-        shouldRunAfter(tasks.named("copyLib"))
-    }
-
-//// Copy hedera-node/configuration/dev as hedera-node/hedera-app/build/node/data/config  }
-val copyConfig =
-    tasks.register<Copy>("copyConfig") {
-        from(layout.projectDirectory.dir("../configuration/dev")) { into("data/config") }
-        from(layout.projectDirectory.file("../config.txt"))
-        from(layout.projectDirectory.file("../log4j2.xml"))
-        into(buildDir)
-        shouldRunAfter(tasks.named("copyNodeData"))
-    }
-
 // Copy dependencies into `data/lib`
 val copyLib =
     tasks.register<Sync>("copyLib") {
@@ -183,27 +157,46 @@ val copyApp =
         shouldRunAfter(tasks.named("copyLib"))
     }
 
+// Working directory for 'run' tasks
+val nodeWorkingDir = layout.buildDirectory.dir("node")
+
+val copyNodeData =
+    tasks.register<Sync>("copyNodeDataAndConfig") {
+        into(nodeWorkingDir)
+
+        // Copy things from hedera-node/data
+        into("data/lib") { from(copyLib) }
+        into("data/apps") { from(copyApp) }
+        into("data/onboard") { from(layout.projectDirectory.dir("../data/onboard")) }
+        into("data/keys") { from(layout.projectDirectory.dir("../data/keys")) }
+
+        // Copy hedera-node/configuration/dev as hedera-node/hedera-app/build/node/data/config  }
+        from(layout.projectDirectory.dir("../configuration/dev")) { into("data/config") }
+        from(layout.projectDirectory.file("../config.txt"))
+        from(layout.projectDirectory.file("../log4j2.xml"))
+        from(layout.projectDirectory.file("../configuration/dev/settings.txt"))
+    }
+
 tasks.assemble {
     dependsOn(copyLib)
     dependsOn(copyApp)
     dependsOn(copyNodeData)
-    dependsOn(copyConfig)
 }
 
 // Create the "run" task for running a Hedera consensus node
 tasks.register<JavaExec>("run") {
     group = "application"
     dependsOn(tasks.assemble)
-    workingDir = layout.projectDirectory.dir("./build/node").asFile
-    jvmArgs = listOf("-cp", "lib/*")
+    workingDir = nodeWorkingDir.get().asFile
+    jvmArgs = listOf("-cp", "data/lib/*")
     mainClass.set("com.swirlds.platform.Browser")
 }
 
 tasks.register<JavaExec>("modrun") {
     group = "application"
     dependsOn(tasks.assemble)
-    workingDir = layout.projectDirectory.dir("./build/node").asFile
-    jvmArgs = listOf("-cp", "lib/*:apps/*", "-Dhedera.workflows.enabled=true")
+    workingDir = nodeWorkingDir.get().asFile
+    jvmArgs = listOf("-cp", "data/lib/*:data/apps/*", "-Dhedera.workflows.enabled=true")
     mainClass.set("com.hedera.node.app.ServicesMain")
 }
 

diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/OrderedServiceMigrator.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/OrderedServiceMigrator.java
@@ -77,9 +77,7 @@ public void doMigrations(
                 .filter(service -> EntityIdService.NAME.equals(service.service().getServiceName()))
                 .findFirst()
                 .orElseThrow();
-        final var entityIdService = new EntityIdService();
         final var entityIdRegistry = (MerkleSchemaRegistry) entityIdRegistration.registry();
-        entityIdService.registerSchemas(entityIdRegistry, currentVersion);
         entityIdRegistry.migrate(
                 state,
                 previousVersion,