diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java index b6a06ef05681..2550497b22b6 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java @@ -18,22 +18,28 @@ import static com.hedera.node.app.bbm.associations.TokenAssociationsDumpUtils.dumpModTokenRelations; import static com.hedera.node.app.bbm.associations.TokenAssociationsDumpUtils.dumpMonoTokenRelations; +import static com.hedera.node.app.bbm.files.FilesDumpUtils.dumpModFiles; +import static com.hedera.node.app.bbm.files.FilesDumpUtils.dumpMonoFiles; import static com.hedera.node.app.bbm.nfts.UniqueTokenDumpUtils.dumpModUniqueTokens; import static com.hedera.node.app.bbm.nfts.UniqueTokenDumpUtils.dumpMonoUniqueTokens; import static com.hedera.node.app.records.BlockRecordService.BLOCK_INFO_STATE_KEY; import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.NETWORK_CTX; +import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.STORAGE; import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.TOKEN_ASSOCIATIONS; import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.UNIQUE_TOKENS; import static com.hedera.node.app.service.token.impl.TokenServiceImpl.NFTS_KEY; import static com.hedera.node.app.service.token.impl.TokenServiceImpl.TOKEN_RELS_KEY; import static java.util.Objects.requireNonNull; +import com.hedera.hapi.node.base.FileID; import com.hedera.hapi.node.base.NftID; import com.hedera.hapi.node.base.TokenAssociation; import com.hedera.hapi.node.state.blockrecords.BlockInfo; import com.hedera.hapi.node.state.token.Nft; import com.hedera.hapi.node.state.token.TokenRelation; import com.hedera.node.app.records.BlockRecordService; +import com.hedera.node.app.service.file.FileService; +import com.hedera.node.app.service.file.impl.FileServiceImpl; import com.hedera.node.app.service.mono.state.merkle.MerkleNetworkContext; import com.hedera.node.app.service.token.TokenService; import com.hedera.node.app.state.merkle.MerkleHederaState; @@ -54,6 +60,7 @@ public class StateDumper { private static final String SEMANTIC_UNIQUE_TOKENS = "uniqueTokens.txt"; private static final String SEMANTIC_TOKEN_RELATIONS = "tokenRelations.txt"; + private static final String SEMANTIC_FILES = "files.txt"; public static void dumpMonoChildrenFrom( @NonNull final MerkleHederaState state, @NonNull final DumpCheckpoint checkpoint) { @@ -62,6 +69,7 @@ public static void dumpMonoChildrenFrom( dumpMonoUniqueTokens(Paths.get(dumpLoc, SEMANTIC_UNIQUE_TOKENS), state.getChild(UNIQUE_TOKENS), checkpoint); dumpMonoTokenRelations( Paths.get(dumpLoc, SEMANTIC_TOKEN_RELATIONS), state.getChild(TOKEN_ASSOCIATIONS), checkpoint); + dumpMonoFiles(Paths.get(dumpLoc, SEMANTIC_FILES), state.getChild(STORAGE), checkpoint); } public static void dumpModChildrenFrom( @@ -74,12 +82,18 @@ public static void dumpModChildrenFrom( Optional.ofNullable(blockInfo.consTimeOfLastHandledTxn()) .map(then -> Instant.ofEpochSecond(then.seconds(), then.nanos())) .orElse(null)); + final VirtualMap, OnDiskValue> uniqueTokens = requireNonNull(state.getChild(state.findNodeIndex(TokenService.NAME, NFTS_KEY))); dumpModUniqueTokens(Paths.get(dumpLoc, SEMANTIC_UNIQUE_TOKENS), uniqueTokens, checkpoint); + final VirtualMap, OnDiskValue> tokenRelations = requireNonNull(state.getChild(state.findNodeIndex(TokenService.NAME, TOKEN_RELS_KEY))); dumpModTokenRelations(Paths.get(dumpLoc, SEMANTIC_TOKEN_RELATIONS), tokenRelations, checkpoint); + + final VirtualMap, OnDiskValue> files = + requireNonNull(state.getChild(state.findNodeIndex(FileService.NAME, FileServiceImpl.BLOBS_KEY))); + dumpModFiles(Paths.get(dumpLoc, SEMANTIC_FILES), files, checkpoint); } private static String getExtantDumpLoc( diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java new file mode 100644 index 000000000000..502aa3c3622d --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.bbm.files; + +import com.google.common.collect.ComparisonChain; +import com.hedera.hapi.node.base.FileID; +import com.hedera.node.app.bbm.utils.Writer; +import edu.umd.cs.findbugs.annotations.NonNull; + +record FileId(long shardNum, long realmNum, long fileNum) implements Comparable { + + static FileId fromMod(@NonNull final FileID fileID) { + return new FileId(fileID.shardNum(), fileID.realmNum(), fileID.fileNum()); + } + + static FileId fromMono(@NonNull final Integer fileNum) { + return new FileId(0, 0, fileNum); + } + + @Override + public String toString() { + return "%d%s%d%s%d".formatted(shardNum, Writer.FIELD_SEPARATOR, realmNum, Writer.FIELD_SEPARATOR, fileNum); + } + + @Override + public int compareTo(FileId o) { + return ComparisonChain.start() + .compare(this.shardNum, o.shardNum) + .compare(this.realmNum, o.realmNum) + .compare(this.fileNum, o.fileNum) + .result(); + } +} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileStore.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileStore.java new file mode 100644 index 000000000000..1bd2c68d34d5 --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileStore.java @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.bbm.files; + +enum FileStore { + ORDINARY, + SPECIAL +} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java new file mode 100644 index 000000000000..eeb552565441 --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java @@ -0,0 +1,219 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.bbm.files; + +import static com.hedera.node.app.bbm.utils.ThingsToStrings.quoteForCsv; +import static com.hedera.node.app.bbm.utils.ThingsToStrings.squashLinesToEscapes; +import static com.hedera.node.app.bbm.utils.ThingsToStrings.toStringOfByteArray; +import static com.swirlds.common.threading.manager.AdHocThreadManager.getStaticThreadManager; + +import com.hedera.hapi.node.base.FileID; +import com.hedera.hapi.node.state.file.File; +import com.hedera.node.app.bbm.DumpCheckpoint; +import com.hedera.node.app.bbm.utils.Writer; +import com.hedera.node.app.service.mono.files.HFileMeta; +import com.hedera.node.app.service.mono.files.MetadataMapFactory; +import com.hedera.node.app.service.mono.state.adapters.VirtualMapLike; +import com.hedera.node.app.service.mono.state.virtual.VirtualBlobKey; +import com.hedera.node.app.service.mono.state.virtual.VirtualBlobValue; +import com.hedera.node.app.service.mono.utils.MiscUtils; +import com.hedera.node.app.state.merkle.disk.OnDiskKey; +import com.hedera.node.app.state.merkle.disk.OnDiskValue; +import com.swirlds.base.utility.Pair; +import com.swirlds.virtualmap.VirtualMap; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.file.Path; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +public class FilesDumpUtils { + + private FilesDumpUtils() { + // Utility class + } + + public static void dumpModFiles( + @NonNull final Path path, + @NonNull final VirtualMap, OnDiskValue> files, + @NonNull final DumpCheckpoint checkpoint) { + try (@NonNull final var writer = new Writer(path)) { + final var dumpableFiles = gatherModFiles(files); + reportOnFiles(writer, dumpableFiles); + System.out.printf( + "=== mod files report is %d bytes at checkpoint %s%n", writer.getSize(), checkpoint.name()); + } + } + + public static void dumpMonoFiles( + @NonNull final Path path, + @NonNull final VirtualMap files, + @NonNull final DumpCheckpoint checkpoint) { + try (@NonNull final var writer = new Writer(path)) { + final var dumpableFiles = gatherMonoFiles(files); + reportOnFiles(writer, dumpableFiles); + System.out.printf( + "=== mono files report is %d bytes at checkpoint %s%n", writer.getSize(), checkpoint.name()); + } + } + + @NonNull + private static Map gatherModFiles(VirtualMap, OnDiskValue> source) { + final var r = new HashMap(); + final var threadCount = 8; + final var files = new ConcurrentLinkedQueue>(); + try { + VirtualMapLike.from(source) + .extractVirtualMapData( + getStaticThreadManager(), + p -> files.add(Pair.of(FileId.fromMod(p.left().getKey()), HederaFile.fromMod(p.right()))), + threadCount); + } catch (final InterruptedException ex) { + System.err.println("*** Traversal of files virtual map interrupted!"); + Thread.currentThread().interrupt(); + } + files.forEach(filePair -> r.put(filePair.key(), filePair.value())); + return r; + } + + /** Collects the information for each data file in the file store, also the summaries of all files of all types. */ + @SuppressWarnings("java:S108") // "Nested blocks of code should not be left empty" - not for switches on an enum + @NonNull + private static Map gatherMonoFiles( + @NonNull final VirtualMap source) { + final var foundFiles = new ConcurrentHashMap(); + final var foundMetadata = new ConcurrentHashMap(); + + final var nType = new ConcurrentHashMap(); + final var nNullValues = new ConcurrentHashMap(); + final var nNullMetadataValues = new AtomicInteger(); + + Stream.of(nType, nNullValues) + .forEach(m -> EnumSet.allOf(VirtualBlobKey.Type.class).forEach(t -> m.put(t, 0))); + + final int THREAD_COUNT = 8; + boolean didRunToCompletion = true; + try { + VirtualMapLike.from(source) + .extractVirtualMapDataC( + getStaticThreadManager(), + entry -> { + final var contractId = entry.key().getEntityNumCode(); + + final var type = entry.key().getType(); + nType.merge(type, 1, Integer::sum); + + final var value = entry.value().getData(); + if (null != value) { + switch (type) { + case FILE_DATA -> foundFiles.put(contractId, value); + + case FILE_METADATA -> { + final var metadata = MetadataMapFactory.toAttr(value); + if (null != metadata) { + foundMetadata.put(contractId, metadata); + } else { + nNullMetadataValues.incrementAndGet(); + + System.err.printf( + "*** collectFiles file metadata (HFileMeta) null for contract id %d, type %s%n", + contractId, type); + } + } + case CONTRACT_BYTECODE, SYSTEM_DELETED_ENTITY_EXPIRY -> {} + } + } else { + nNullValues.merge(type, 1, Integer::sum); + + System.err.printf( + "*** collectFiles file value (bytes) null for contract id %d, type %s%n", + contractId, type); + } + }, + THREAD_COUNT); + } catch (final InterruptedException ex) { + Thread.currentThread().interrupt(); + didRunToCompletion = false; + } + + if (!didRunToCompletion) { + System.err.printf("*** collectFiles interrupted (did not run to completion)%n"); + } + + final var r = new HashMap(); + for (@NonNull final var e : foundFiles.entrySet()) { + final var contractId = e.getKey(); + final var contents = e.getValue(); + final var metadata = foundMetadata.getOrDefault(contractId, null); + r.put( + FileId.fromMono(contractId), + null != metadata + ? HederaFile.of(contractId, contents, metadata) + : HederaFile.of(contractId, contents)); + } + + return r; + } + + private static void reportOnFiles(@NonNull final Writer writer, @NonNull final Map files) { + reportFileContentsHeader(writer); + reportFileContents(writer, files); + writer.writeln(""); + } + + /** Emits the CSV header line for the file contents - **KEEP IN SYNC WITH reportFileContents!!!** */ + private static void reportFileContentsHeader(@NonNull final Writer writer) { + final var header = "fileId,PRESENT/DELETED,SPECIAL file,SYSTEM file,length(bytes),expiry,memo,content,key"; + writer.write("%s%n", header); + } + + /** Emits the actual content (hexified) for each file, and it's full key */ + private static void reportFileContents( + @NonNull final Writer writer, @NonNull final Map allFiles) { + for (@NonNull + final var file : + allFiles.entrySet().stream().sorted(Map.Entry.comparingByKey()).toList()) { + final var fileId = file.getKey().fileNum(); + final var hf = file.getValue(); + if (hf.isActive()) { + final var sb = new StringBuilder(); + toStringOfByteArray(sb, hf.contents()); + writer.write( + "%d,PRESENT,%s,%s,%d,%s,%s,%s,%s%n", + fileId, + hf.fileStore() == FileStore.SPECIAL ? "SPECIAL" : "", + hf.systemFileType() != null ? hf.systemFileType().name() : "", + hf.contents().length, + hf.metadata() != null ? Long.toString(hf.metadata().getExpiry()) : "", + hf.metadata() != null ? quoteForCsv(",", hf.metadata().getMemo()) : "", + sb, + hf.metadata() != null + ? quoteForCsv( + ",", + squashLinesToEscapes( + MiscUtils.describe(hf.metadata().getWacl()))) + : ""); + } else { + writer.write("%d,DELETED%n", fileId); + } + } + } +} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java new file mode 100644 index 000000000000..6bb722457821 --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.bbm.files; + +import com.hedera.hapi.node.state.file.File; +import com.hedera.node.app.service.mono.files.HFileMeta; +import com.hedera.node.app.state.merkle.disk.OnDiskValue; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; + +/** Holds the content and the metadata for a single data file in the store */ +@SuppressWarnings("java:S6218") // "Equals/hashcode methods should be overridden in records containing array fields" +// not using this with equals +record HederaFile( + @NonNull FileStore fileStore, + @NonNull Integer fileId, + @NonNull byte[] contents, + @Nullable HFileMeta metadata, + @Nullable SystemFileType systemFileType) { + + static HederaFile fromMod(@NonNull final OnDiskValue wrapper) { + final var value = wrapper.getValue(); + return new HederaFile( + FileStore.ORDINARY, + (int) value.fileId().fileNum(), + value.contents().toByteArray(), + null, + SystemFileType.byId.get((int) value.fileId().fileNum())); + } + + @NonNull + static HederaFile of(final int fileId, @NonNull final byte[] contents) { + return new HederaFile(FileStore.ORDINARY, fileId, contents, null, SystemFileType.byId.get(fileId)); + } + + @NonNull + static HederaFile of(final int fileId, @NonNull final byte[] contents, @NonNull final HFileMeta metadata) { + return new HederaFile(FileStore.ORDINARY, fileId, contents, metadata, SystemFileType.byId.get(fileId)); + } + + boolean isActive() { + if (null != systemFileType) { + return true; + } + if (null != metadata) { + return !metadata.isDeleted(); + } + return false; + } +} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/SystemFileType.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/SystemFileType.java new file mode 100644 index 000000000000..567680fa40bc --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/SystemFileType.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.bbm.files; + +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; + +enum SystemFileType { + ADDRESS_BOOK(101), + NODE_DETAILS(102), + FEE_SCHEDULES(111), + EXCHANGE_RATES(112), + NETWORK_PROPERTIES(121), + HAPI_PERMISSIONS(122), + THROTTLE_DEFINITIONS(123), + SOFTWARE_UPDATE0(150), + SOFTWARE_UPDATE1(151), + SOFTWARE_UPDATE2(152), + SOFTWARE_UPDATE3(153), + SOFTWARE_UPDATE4(154), + SOFTWARE_UPDATE5(155), + SOFTWARE_UPDATE6(156), + SOFTWARE_UPDATE7(157), + SOFTWARE_UPDATE8(158), + SOFTWARE_UPDATE9(159), + UNKNOWN(-1); + + public final int id; + + static final Map byId = new HashMap<>(); + + SystemFileType(final int id) { + this.id = id; + } + + static { + EnumSet.allOf(SystemFileType.class).forEach(e -> byId.put(e.id, e)); + } +}