diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/GenesisPlatformStateCommand.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/GenesisPlatformStateCommand.java index 69d4721948b6..28567778d6de 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/GenesisPlatformStateCommand.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/GenesisPlatformStateCommand.java @@ -95,7 +95,7 @@ public Integer call() throws IOException, ExecutionException, InterruptedExcepti .digestTreeAsync(reservedSignedState.get().getState()) .get(); System.out.printf("Writing modified state to %s %n", outputDir.toAbsolutePath()); - writeSignedStateFilesToDirectory(NO_NODE_ID, outputDir, reservedSignedState.get(), configuration); + writeSignedStateFilesToDirectory(platformContext, NO_NODE_ID, outputDir, reservedSignedState.get()); } return 0; diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PreconsensusEventFileManager.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PreconsensusEventFileManager.java index 0488130e4e52..160ebcefab1c 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PreconsensusEventFileManager.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PreconsensusEventFileManager.java @@ -162,7 +162,7 @@ public PreconsensusEventFileManager( * @return the directory where event files are stored */ @NonNull - private static Path getDatabaseDirectory( + public static Path getDatabaseDirectory( @NonNull final PlatformContext platformContext, @NonNull final NodeId selfId) throws IOException { final StateConfig stateConfig = platformContext.getConfiguration().getConfigData(StateConfig.class); diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PreconsensusEventStreamConfig.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PreconsensusEventStreamConfig.java index cd4e1346fc51..a0600cc1bd43 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PreconsensusEventStreamConfig.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PreconsensusEventStreamConfig.java @@ -77,7 +77,8 @@ * Both. Use this with caution. * @param databaseDirectory the directory where preconsensus events will be stored, * relative to - * {@link com.swirlds.common.config.StateConfig#savedStateDirectory()}. + * {@link + * com.swirlds.common.config.StateConfig#savedStateDirectory()}. * @param enableStorage if true, then stream preconsensus events to files on disk. If * this is disabled then a network wide crash (perhaps due to a * bug) can cause transactions that previously reached consensus @@ -90,6 +91,10 @@ * @param replayQueueSize the size of the queue used for holding preconsensus events * that are waiting to be replayed * @param replayHashPoolSize the number of threads used for hashing events during replay + * @param copyRecentStreamToStateSnapshots if true, then copy recent PCES files into the saved state + * snapshot directories every time we take a state snapshot. The + * files copied are guaranteed to contain all non-ancient events + * w.r.t. the state snapshot. */ @ConfigData("event.preconsensus") public record PreconsensusEventStreamConfig( @@ -106,4 +111,5 @@ public record PreconsensusEventStreamConfig( @ConfigProperty(defaultValue = "true") boolean enableStorage, @ConfigProperty(defaultValue = "true") boolean enableReplay, @ConfigProperty(defaultValue = "1024") int replayQueueSize, - @ConfigProperty(defaultValue = "8") int replayHashPoolSize) {} + @ConfigProperty(defaultValue = "8") int replayHashPoolSize, + @ConfigProperty(defaultValue = "true") boolean copyRecentStreamToStateSnapshots) {} diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/recovery/EventRecoveryWorkflow.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/recovery/EventRecoveryWorkflow.java index c87990429f13..3f716b855bf5 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/recovery/EventRecoveryWorkflow.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/recovery/EventRecoveryWorkflow.java @@ -168,7 +168,10 @@ public static void recoverState( resultingStateDirectory); SignedStateFileWriter.writeSignedStateFilesToDirectory( - selfId, resultingStateDirectory, recoveredState.state().get(), platformContext.getConfiguration()); + platformContext, + selfId, + resultingStateDirectory, + recoveredState.state().get()); final StateConfig stateConfig = platformContext.getConfiguration().getConfigData(StateConfig.class); updateEmergencyRecoveryFile( stateConfig, resultingStateDirectory, initialState.get().getConsensusTimestamp()); diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/editor/StateEditorSave.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/editor/StateEditorSave.java index a9cd6bcebc4f..e4eb4cbae559 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/editor/StateEditorSave.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/editor/StateEditorSave.java @@ -22,7 +22,11 @@ import static com.swirlds.platform.state.signed.SignedStateFileWriter.writeSignedStateFilesToDirectory; import com.swirlds.cli.utility.SubcommandOf; +import com.swirlds.common.context.DefaultPlatformContext; +import com.swirlds.common.context.PlatformContext; +import com.swirlds.common.crypto.CryptographyHolder; import com.swirlds.common.merkle.crypto.MerkleCryptoFactory; +import com.swirlds.common.metrics.noop.NoOpMetrics; import com.swirlds.config.api.Configuration; import com.swirlds.logging.legacy.LogMarker; import com.swirlds.platform.config.DefaultConfiguration; @@ -69,8 +73,12 @@ public void run() { } final Configuration configuration = DefaultConfiguration.buildBasicConfiguration(); + + final PlatformContext platformContext = + new DefaultPlatformContext(configuration, new NoOpMetrics(), CryptographyHolder.get()); + try (final ReservedSignedState signedState = getStateEditor().getSignedStateCopy()) { - writeSignedStateFilesToDirectory(NO_NODE_ID, directory, signedState.get(), configuration); + writeSignedStateFilesToDirectory(platformContext, NO_NODE_ID, directory, signedState.get()); } } catch (final IOException e) { diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileManager.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileManager.java index 4dc7b6509754..00af87b0bdfd 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileManager.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileManager.java @@ -103,6 +103,7 @@ public class SignedStateFileManager implements Startable { private final SignedStateMetrics metrics; private final Configuration configuration; + private final PlatformContext platformContext; /** * Provides system time @@ -162,6 +163,7 @@ public SignedStateFileManager( this.mainClassName = mainClassName; this.swirldName = swirldName; this.stateToDiskAttemptConsumer = stateToDiskAttemptConsumer; + this.platformContext = Objects.requireNonNull(context); this.configuration = Objects.requireNonNull(context).getConfiguration(); this.minimumGenerationNonAncientConsumer = Objects.requireNonNull( minimumGenerationNonAncientConsumer, "minimumGenerationNonAncientConsumer must not be null"); @@ -288,7 +290,7 @@ private void saveStateTask( if (outOfBand) { // states requested to be written out-of-band are always written to disk SignedStateFileWriter.writeSignedStateToDisk( - selfId, directory, reservedSignedState.get(), reason, configuration); + platformContext, selfId, directory, reservedSignedState.get(), reason); success = true; } else { @@ -303,7 +305,7 @@ private void saveStateTask( } SignedStateFileWriter.writeSignedStateToDisk( - selfId, directory, reservedSignedState.get(), reason, configuration); + platformContext, selfId, directory, reservedSignedState.get(), reason); stateWrittenToDiskInBand(reservedSignedState.get(), directory, start); success = true; @@ -408,8 +410,8 @@ public boolean saveSignedStateToDisk(@NonNull final SignedState signedState, fin * Dump a state to disk out-of-band. *
* Writing a state "out-of-band" means the state is being written for the sake of a human, whether for debug - * purposes, or because of a fault. States written out-of-band will not be read automatically by the platform, - * and will not be used as an initial state at boot time. + * purposes, or because of a fault. States written out-of-band will not be read automatically by the platform, and + * will not be used as an initial state at boot time. *
* A dumped state will be saved in a subdirectory of the signed states base directory, with the subdirectory being * named after the reason the state is being written out-of-band. diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileWriter.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileWriter.java index d29ea314a10b..34057ed51a76 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileWriter.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/SignedStateFileWriter.java @@ -29,12 +29,15 @@ import com.swirlds.common.config.StateConfig; import com.swirlds.common.config.singleton.ConfigurationHolder; +import com.swirlds.common.context.PlatformContext; import com.swirlds.common.io.streams.MerkleDataOutputStream; import com.swirlds.common.merkle.utility.MerkleTreeVisualizer; import com.swirlds.common.system.NodeId; import com.swirlds.common.system.address.AddressBook; -import com.swirlds.config.api.Configuration; import com.swirlds.logging.legacy.payload.StateSavedToDiskPayload; +import com.swirlds.platform.event.preconsensus.PreconsensusEventFile; +import com.swirlds.platform.event.preconsensus.PreconsensusEventFileManager; +import com.swirlds.platform.event.preconsensus.PreconsensusEventStreamConfig; import com.swirlds.platform.recovery.emergencyfile.EmergencyRecoveryFile; import com.swirlds.platform.state.State; import edu.umd.cs.findbugs.annotations.NonNull; @@ -42,9 +45,14 @@ import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Objects; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -135,27 +143,257 @@ public static void writeStateFile(final Path directory, final SignedState signed /** * Write all files that belong in the signed state directory into a directory. * - * @param selfId the id of the platform - * @param directory the directory where all files should be placed - * @param signedState the signed state being written to disk - * @param configuration the configuration used + * @param platformContext the platform context + * @param selfId the id of the platform + * @param directory the directory where all files should be placed + * @param signedState the signed state being written to disk */ public static void writeSignedStateFilesToDirectory( - @Nullable NodeId selfId, + @Nullable final PlatformContext platformContext, + @Nullable final NodeId selfId, @NonNull final Path directory, - @NonNull final SignedState signedState, - @NonNull final Configuration configuration) + @NonNull final SignedState signedState) throws IOException { - Objects.requireNonNull(directory, "directory must not be null"); - Objects.requireNonNull(signedState, "signedState must not be null"); - Objects.requireNonNull(configuration, "configuration must not be null"); + Objects.requireNonNull(platformContext); + Objects.requireNonNull(directory); + Objects.requireNonNull(signedState); writeStateFile(directory, signedState); writeHashInfoFile(directory, signedState.getState()); writeMetadataFile(selfId, directory, signedState); writeEmergencyRecoveryFile(directory, signedState); writeStateAddressBookFile(directory, signedState.getAddressBook()); - writeSettingsUsed(directory, configuration); + writeSettingsUsed(directory, platformContext.getConfiguration()); + + if (selfId != null) { + copyPreconsensusEventStreamFiles( + platformContext, + selfId, + directory, + signedState.getState().getPlatformState().getPlatformData().getMinimumGenerationNonAncient()); + } + } + + /** + * Copy preconsensus event files into the signed state directory. These files are necessary for the platform to use + * the state file as a starting point. Note: starting a node using the PCES files in the state directory does not + * guarantee that there is no data loss (i.e. there may be transactions that reach consensus after the state + * snapshot), but it does allow a node to start up and participate in gossip. + * + *
+ * This general strategy is not very elegant is very much a hack. But it will allow us to do migration testing using
+ * real production states and streams, in the short term. In the longer term we should consider alternate and
+ * cleaner strategies.
+ *
+ * @param platformContext the platform context
+ * @param destinationDirectory the directory where the state is being written
+ * @param minimumGenerationNonAncient the minimum generation of events that are not ancient, with respect to the
+ * state that is being written
+ */
+ private static void copyPreconsensusEventStreamFiles(
+ @NonNull final PlatformContext platformContext,
+ @NonNull final NodeId selfId,
+ @NonNull final Path destinationDirectory,
+ final long minimumGenerationNonAncient)
+ throws IOException {
+
+ final boolean copyPreconsensusStream = platformContext
+ .getConfiguration()
+ .getConfigData(PreconsensusEventStreamConfig.class)
+ .copyRecentStreamToStateSnapshots();
+ if (!copyPreconsensusStream) {
+ // PCES copying is disabled
+ return;
+ }
+
+ // The PCES files will be copied into this directory
+ final Path pcesDestination =
+ destinationDirectory.resolve("preconsensus-events").resolve(Long.toString(selfId.id()));
+ Files.createDirectories(pcesDestination);
+
+ final List