Merge remote-tracking branch 'origin/develop' into check-eth-e2es

.env

@@ -1,2 +1,2 @@
-TAG=0.32.0-SNAPSHOT
+TAG=0.33.0-SNAPSHOT
 REGISTRY_PREFIX=

Dockerfile

@@ -38,12 +38,6 @@ WORKDIR /opt/hedera/services
 COPY .env /opt/hedera/services
 RUN for PIECE in $(cat .env | head -1 | tr '=' ' '); do \
   if [ "$IS_VERSION" = "true" ]; then echo $PIECE >> .VERSION ; else IS_VERSION=true; fi done
-RUN mkdir /opt/hedera/services/hapi-utils
-COPY hapi-utils /opt/hedera/services/hapi-utils
-RUN mkdir /opt/hedera/services/hapi-fees
-COPY hapi-fees /opt/hedera/services/hapi-fees
-RUN mkdir /opt/hedera/services/modules
-COPY modules /opt/hedera/services/modules
 RUN mkdir /opt/hedera/services/hedera-node
 COPY hedera-node /opt/hedera/services/hedera-node
 RUN mkdir /opt/hedera/services/test-clients

buildSrc/src/main/kotlin/com.hedera.hashgraph.jpms-modules.gradle.kts

@@ -22,19 +22,22 @@ plugins {
 }
 
 extraJavaModuleInfo {
-    failOnMissingModuleInfo.set(false)
+    failOnMissingModuleInfo.set(true)
     automaticModule("com.hedera.hashgraph:protobuf-java", "com.hedera.hashgraph.protobuf.java.api")
 
     automaticModule("com.goterl:lazysodium-java", "lazysodium.java")
     automaticModule("com.goterl:resource-loader", "resource.loader")
 
     automaticModule("io.grpc:grpc-api", "io.grpc.api")
+    automaticModule("io.grpc:grpc-core", "io.grpc.core")
     automaticModule("io.grpc:grpc-netty", "io.grpc.netty")
+    automaticModule("io.grpc:grpc-context", "io.grpc.context")
     automaticModule("io.grpc:grpc-stub", "grpc.stub")
     automaticModule("io.grpc:grpc-protobuf", "grpc.protobuf")
     automaticModule("io.grpc:grpc-services", "grpc.services")
-    automaticModule("io.grpc:grpc-protobuf-lite", "grpc.protobuf.lite")
-    automaticModule("io.grpc:grpc-netty", "grpc.netty")
+    automaticModule("io.grpc:grpc-protobuf-lite", "io.grpc.protobuf.lite")
+    automaticModule("io.grpc:grpc-netty", "io.grpc.netty")
+    automaticModule("io.grpc:grpc-testing", "io.grpc.testing")
 
     automaticModule("org.openjdk.jmh:jmh-core", "jmh.core")
     automaticModule("org.openjdk.jmh:jmh-generator-asm", "jmh.generator.asm")
@@ -72,6 +75,39 @@ extraJavaModuleInfo {
     automaticModule("com.google.guava:listenablefuture", "listenablefuture")
     automaticModule("com.google.guava:failureaccess", "failureaccess")
     automaticModule("com.google.auto.value:auto-value-annotations", "auto.value.annotations")
+    automaticModule("com.google.api.grpc:proto-google-common-protos", "com.google.api.grpc.proto.common")
+
+    automaticModule("org.connid:framework", "org.connid.framework")
+    automaticModule("org.connid:framework-internal", "org.connid.framework.internal")
+
+    automaticModule("org.jetbrains.kotlin:kotlin-stdlib-common", "org.jetbrains.kotlin.stdlib.common")
+    automaticModule("junit:junit", "junit.old")
+    automaticModule("org.codehaus.mojo:animal-sniffer-annotations", "org.codehaus.mojo.animalsniffer.annotations")
+    automaticModule("org.checkerframework:checker-compat-qual", "org.checkerframework.checker.compat.qual")
+    automaticModule("com.google.code.gson:gson", "com.google.code.gson")
+    automaticModule("com.google.android:annotations", "com.google.android.annotations")
+    automaticModule("com.google.errorprone:error_prone_annotations", "com.google.errorprone.annotations")
+    automaticModule("com.google.j2objc:j2objc-annotations", "com.google.j2objc.annotations")
+    automaticModule("com.google.dagger:dagger-compiler", "dagger.compiler")
+    automaticModule("com.google.dagger:dagger-spi", "dagger.spi")
+    automaticModule("com.google.dagger:dagger-producers", "dagger.producers")
+    automaticModule("com.google.googlejavaformat:google-java-format", "com.google.googlejavaformat")
+    automaticModule("com.google.devtools.ksp:symbol-processing-api", "com.google.devtools.ksp.symbolprocessingapi")
+    automaticModule("org.jetbrains.kotlinx:kotlinx-metadata-jvm", "org.jetbrains.kotlinx.metadata.jvm")
+    automaticModule("net.ltgt.gradle.incap:incap", "net.ltgt.gradle.incap")
+    automaticModule("com.google.errorprone:javac-shaded", "com.google.errorprone.javac.shaded")
+    automaticModule("org.hyperledger.besu:bls12-381", "org.hyperledger.besu.bls12.for381")
+    automaticModule("org.hyperledger.besu:secp256r1", "org.hyperledger.besu.secp256r1")
+    automaticModule("org.hyperledger.besu:blake2bf", "org.hyperledger.besu.blake2bf")
+    automaticModule("com.google.truth.extensions:truth-java8-extension", "com.google.truth.extensions.java8")
+    automaticModule("org.apache.commons:commons-math3", "org.apache.commons.math3")
+
+    automaticModule("com.github.docker-java:docker-java-transport-zerodep", "com.github.docker.transport.zerodep")
+    automaticModule("org.rnorth.duct-tape:duct-tape", "org.rnorth.ducttape")
+    automaticModule("io.opencensus:opencensus-api", "io.opencensus.api")
+    automaticModule("org.hyperledger.besu.internal:util", "org.hyperledger.besu.internal.util")
+    automaticModule("org.testcontainers:junit-jupiter", "org.testcontainers.junit.jupiter")
+
 
     // Test Related Modules
     automaticModule("com.github.docker-java:docker-java-transport", "com.github.docker.java.transport")

docs/design/app/workflows.md

@@ -0,0 +1,18 @@
+### Pre-Handle Workflow
+
+The `prehandle` package contains the workflow for pre-handling transactions. A rough overview can be seen in the diagram below.
+
+![Diagram of pre-handle transaction workflow](images/Pre-Handle%20Transaction%20Workflow.png)
+
+An `Event` at a time is sent to the `prehandle` with a reference to the latest immutable state. It iterates through each transaction and initiates the pre-handle workflow in a separate thread. The workflow consists of the following steps:
+
+1. **Parse Transaction.** The transaction arrives as a byte-array. The required parts are parsed and the common information is validated.
+2. **Call PreTransactionHandler.** Depending on the type of transaction, a specific `PreTransactionHandler` is called. It validates the transaction-specific parts and pre-loads data into the cache. It also creates a `TransactionMetadata` and sets the required keys.
+3. **Prepare Signature-Data.** The data for all signatures is loaded into memory. A signature consists of three parts:
+   1. Some bytes that are signed; in our case, either the `bodyBytes` for an Ed25519 signature or the Keccak256 hash of the `bodyBytes` for an ECDSA(secp256k1) signature.
+   2. An Ed25519 or secp256k1 public key that is supposed to have signed these bytes (these public keys come from e.g. the Hedera key of some `0.0.X` account).
+   3. The signature itself---which comes from the `SignatureMap`, based on existence of a unique `SignaturePair` entry whose `pubKeyPrefix` matches the public key in (ii.).
+4. **Verify Signatures.** The information prepared in the previous step is sent to the platform to validate the signatures.
+5. **Transaction Metadata.** The `TransactionMetadata` generated by the `PreTransactionHandler` is attached to the `SwirldsTransaction`.
+
+If all checks have been successful, the status of the created `TransactionMetadata` will be `OK`. Otherwise, the status is set to the response code providing the failure reason. If the workflow terminates early (either because the parsing step (1.) fails or an unexpected `Exception` occurs) an `ErrorTransactionMetadata` is attached to the `SwirldsTransaction` that contains the causing `Exception`.

docs/dev/testnet-rekey.md

@@ -1,61 +1,67 @@
 # :old_key: Re-keying treasury on a testnet saved state
 
-This document describes how you use your local environment to 
-change the `0.0.2` key on a saved state from stable testnet. 
-(We can then update our migration tests to use the new state 
+This document describes how you use your local environment to
+change the `0.0.2` key on a saved state from stable testnet.
+(We can then update our migration tests to use the new state
 and key, without any security concerns.)
 
 :bangbang:  There is one important prerequisite to this guide:
-  1. Since only `0.0.2` can update the treasury key, you must 
-     have a local PEM/passphrase combination with the current 
-     `0.0.2` key on stable testnet. 
 
-:warning:  Always follow these steps **on the tag** matching 
+1. Since only `0.0.2` can update the treasury key, you must
+   have a local PEM/passphrase combination with the current
+   `0.0.2` key on stable testnet.
+
+:warning:  Always follow these steps **on the tag** matching
 the version of testnet state that DevOps provided. For example, if
 DevOps provided a 0.14.0 state, do everything that follows on the
-`v0.14.0` tag! 
+`v0.14.0` tag!
 
 Also, note that at the time of writing, stable testnet has five nodes;
-this number appears below in a couple places which should be easy to 
+this number appears below in a couple places which should be easy to
 adapt if needed.
 
 ## Preparing to start Services from the saved state
 
 ### Setting up data/saved/
 
 Let's suppose you have the _SignedState.swh_ and _PostgresBackup.tar.gz_
-from round 123456789 on testnet in a directory 
-_~/dev/states/stabletestnet/current_. We begin by running, from the 
+from round 123456789 on testnet in a directory
+_~/dev/states/stabletestnet/current_. We begin by running, from the
 root of the Services project,
+
 ```
 $ run/reset-data-saved-from.sh \
 > ~/dev/states/stabletestnet/current \
 > 123456789 \
 > 5
 ```
-This should copy the state files into five node-scoped subdirectories under 
+
+This should copy the state files into five node-scoped subdirectories under
 _hedera-node/data/saved/com.hedera.services.ServicesMain_.
 
 ### Restoring the PostgreSQL state
 
 Open a new terminal at _hedera-node/data/backup_. Then run,
+
 ```
 $ ./pg_restore_manual.sh \
 > ~/dev/states/stabletestnet/current/PostgresBackup.tar.gz
 ```
 
-:mantelpiece_clock: Don't worry if it appears to hang, 
-this will take several minutes in the best case! 
+:mantelpiece_clock: Don't worry if it appears to hang,
+this will take several minutes in the best case!
 
 ### Setting Platform config
 
 Ensure your _hedera-node/settings.txt_ contains the lines,
+
 ```
 dbRestore.active, 0
 dbBackup.active, 0
 ```
 
 And that you have enough nodes in your _config.txt_ address book,
+
 ```
 address,  A, Alice,    1, 127.0.0.1, 50204, 127.0.0.1, 50204, 0.0.3
 address,  B, Bob,      1, 127.0.0.1, 50205, 127.0.0.1, 50205, 0.0.4
@@ -67,20 +73,21 @@ address,  E, Eric,     1, 127.0.0.1, 50208, 127.0.0.1, 50208, 0.0.7
 ## Starting Services
 
 Whether you are starting Services from IntelliJ or command line,
-make sure you give it sufficient heap (8GB to be safe). In IntelliJ 
+make sure you give it sufficient heap (8GB to be safe). In IntelliJ
 this looks like,
 
 ![](../assets/VM-options-for-local-testnet-rekey.png)
 
-Then start Services and wait for a long time until you see Platform 
+Then start Services and wait for a long time until you see Platform
 is `ACTIVE`
 
 ### Rekeying and freezing
 
 :old_key: Put your PEM with the current testnet treasury key
 at _test-clients/stabletestnet-account2.pem_. Update the EET suite
-`com.hedera.services.bdd.suites.misc.RekeySavedStateTreasury` with 
+`com.hedera.services.bdd.suites.misc.RekeySavedStateTreasury` with
 the passphrase to your PEM file on L68, changing,
+
 ```
 ...
 final var passphraseForOriginalPemLoc = "<SECRET>";
@@ -89,13 +96,15 @@ final var passphraseForOriginalPemLoc = "<SECRET>";
 
 Now run the `RekeySavedStateTreasury` suite. The new treasury key
 will be saved at,
- 1. _test-clients/DevStableTestnetStartUpAccount.txt_
- 2. _test-clients/dev-stabletestnet-account2.pem_
+
+1. _test-clients/DevStableTestnetStartUpAccount.txt_
+2. _test-clients/dev-stabletestnet-account2.pem_
 
 ...where the first is a legacy Base64-encoded "startup account"; and
 the PEM file has a passphrase of "swirlds".
 
 Next, run the `FreezeRekeyedState` EET suite. It should end with logs like,
+
 ```
 ...
 2021-05-04 09:47:26.187 INFO   311  HapiApiSpec - 'FreezeRekeyedState' finished initial execution of HapiFreeze{sigs=1, node=0.0.3, start=14:48, end=14:49}
@@ -104,15 +113,17 @@ Next, run the `FreezeRekeyedState` EET suite. It should end with logs like,
 
 ## Capturing the new state
 
-:bangbang:&nbsp;Now we need a minute or two for Services logs to go 
-through the `MAINTENANCE -> ACTIVE` cycle. 
+:bangbang:&nbsp;Now we need a minute or two for Services logs to go
+through the `MAINTENANCE -> ACTIVE` cycle.
 
-After the `Freeze` has written a new signed state to disk, you can 
+After the `Freeze` has written a new signed state to disk, you can
 stop Services and archive the state with the new treasury key,
+
 ```
 $ tar -cvf rekeyed-testnet-round65464591.tar.gz \
-> hedera-node/data/saved/com.hedera.services.ServicesMain/0/123/65464591
+> hedera-node/data/saved/com.hedera.node.app.service.mono.ServicesMain/0/123/65464591
 ```
-(Here 65464591 was the round of state saved by the `Freeze`.)  
+
+(Here 65464591 was the round of state saved by the `Freeze`.)
 
 You're done! Enjoy your shiny new treasury key.