From 2a78e4ea21f7b5a7873ca3701db241945afe2ca3 Mon Sep 17 00:00:00 2001 From: Valentin Tronkov <99957253+vtronkov@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:26:29 +0200 Subject: [PATCH 1/4] feature: add mod file dumper Signed-off-by: Valentin Tronkov <99957253+vtronkov@users.noreply.github.com> --- .../com/hedera/node/app/bbm/StateDumper.java | 12 ++ .../com/hedera/node/app/bbm/files/FileId.java | 48 ++++++ .../hedera/node/app/bbm/files/FileStore.java | 22 +++ .../node/app/bbm/files/FilesDumpUtils.java | 151 ++++++++++++++++++ .../hedera/node/app/bbm/files/HederaFile.java | 66 ++++++++ .../node/app/bbm/files/SystemFileType.java | 54 +++++++ 6 files changed, 353 insertions(+) create mode 100644 hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java create mode 100644 hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileStore.java create mode 100644 hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java create mode 100644 hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java create mode 100644 hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/SystemFileType.java diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java index 60327fff241d..3d014646bbd5 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java @@ -16,18 +16,24 @@ package com.hedera.node.app.bbm; +import static com.hedera.node.app.bbm.files.FilesDumpUtils.dumpModFiles; +import static com.hedera.node.app.bbm.files.FilesDumpUtils.dumpMonoFiles; import static com.hedera.node.app.bbm.nfts.UniqueTokenDumpUtils.dumpModUniqueTokens; import static com.hedera.node.app.bbm.nfts.UniqueTokenDumpUtils.dumpMonoUniqueTokens; import static com.hedera.node.app.records.BlockRecordService.BLOCK_INFO_STATE_KEY; import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.NETWORK_CTX; +import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.STORAGE; import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.UNIQUE_TOKENS; import static com.hedera.node.app.service.token.impl.TokenServiceImpl.NFTS_KEY; import static java.util.Objects.requireNonNull; +import com.hedera.hapi.node.base.FileID; import com.hedera.hapi.node.base.NftID; import com.hedera.hapi.node.state.blockrecords.BlockInfo; import com.hedera.hapi.node.state.token.Nft; import com.hedera.node.app.records.BlockRecordService; +import com.hedera.node.app.service.file.FileService; +import com.hedera.node.app.service.file.impl.FileServiceImpl; import com.hedera.node.app.service.mono.state.merkle.MerkleNetworkContext; import com.hedera.node.app.service.token.TokenService; import com.hedera.node.app.state.merkle.MerkleHederaState; @@ -47,12 +53,14 @@ */ public class StateDumper { private static final String SEMANTIC_UNIQUE_TOKENS = "uniqueTokens.txt"; + private static final String SEMANTIC_FILES = "files.txt"; public static void dumpMonoChildrenFrom( @NonNull final MerkleHederaState state, @NonNull final DumpCheckpoint checkpoint) { final MerkleNetworkContext networkContext = state.getChild(NETWORK_CTX); final var dumpLoc = getExtantDumpLoc("mono", networkContext.consensusTimeOfLastHandledTxn()); dumpMonoUniqueTokens(Paths.get(dumpLoc, SEMANTIC_UNIQUE_TOKENS), state.getChild(UNIQUE_TOKENS), checkpoint); + dumpMonoFiles(Paths.get(dumpLoc, SEMANTIC_FILES), state.getChild(STORAGE), checkpoint); } public static void dumpModChildrenFrom( @@ -68,6 +76,10 @@ public static void dumpModChildrenFrom( final VirtualMap, OnDiskValue> uniqueTokens = requireNonNull(state.getChild(state.findNodeIndex(TokenService.NAME, NFTS_KEY))); dumpModUniqueTokens(Paths.get(dumpLoc, SEMANTIC_UNIQUE_TOKENS), uniqueTokens, checkpoint); + + final VirtualMap, OnDiskValue> test = + requireNonNull(state.getChild(state.findNodeIndex(FileService.NAME, FileServiceImpl.BLOBS_KEY))); + dumpModFiles(Paths.get(dumpLoc, SEMANTIC_FILES), test, checkpoint); } private static String getExtantDumpLoc( diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java new file mode 100644 index 000000000000..db9858715bde --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.bbm.files; + +import com.google.common.collect.ComparisonChain; +import com.hedera.hapi.node.base.FileID; +import com.hedera.node.app.bbm.utils.Writer; +import com.hedera.node.app.service.mono.state.virtual.VirtualBlobKey; +import edu.umd.cs.findbugs.annotations.NonNull; + +record FileId(long shardNum, long realmNum, long fileNum) implements Comparable { + + static FileId fromMod(@NonNull final FileID fileID) { + return new FileId(fileID.shardNum(), fileID.realmNum(), fileID.fileNum()); + } + + static FileId fromMono(@NonNull final VirtualBlobKey key) { + return new FileId(0, 0, key.getEntityNumCode()); + } + + @Override + public String toString() { + return "%d%s%d%s%d".formatted(shardNum, Writer.FIELD_SEPARATOR, realmNum, Writer.FIELD_SEPARATOR, fileNum); + } + + @Override + public int compareTo(FileId o) { + return ComparisonChain.start() + .compare(this.shardNum, o.shardNum) + .compare(this.realmNum, o.realmNum) + .compare(this.fileNum, o.fileNum) + .result(); + } +} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileStore.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileStore.java new file mode 100644 index 000000000000..1bd2c68d34d5 --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileStore.java @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.bbm.files; + +enum FileStore { + ORDINARY, + SPECIAL +} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java new file mode 100644 index 000000000000..b2a5162f3ace --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.bbm.files; + +import static com.hedera.node.app.bbm.utils.ThingsToStrings.quoteForCsv; +import static com.hedera.node.app.bbm.utils.ThingsToStrings.squashLinesToEscapes; +import static com.hedera.node.app.bbm.utils.ThingsToStrings.toStringOfByteArray; +import static com.swirlds.common.threading.manager.AdHocThreadManager.getStaticThreadManager; + +import com.hedera.hapi.node.base.FileID; +import com.hedera.hapi.node.state.file.File; +import com.hedera.node.app.bbm.DumpCheckpoint; +import com.hedera.node.app.bbm.utils.Writer; +import com.hedera.node.app.service.mono.state.adapters.VirtualMapLike; +import com.hedera.node.app.service.mono.state.virtual.VirtualBlobKey; +import com.hedera.node.app.service.mono.state.virtual.VirtualBlobValue; +import com.hedera.node.app.service.mono.utils.MiscUtils; +import com.hedera.node.app.state.merkle.disk.OnDiskKey; +import com.hedera.node.app.state.merkle.disk.OnDiskValue; +import com.swirlds.base.utility.Pair; +import com.swirlds.virtualmap.VirtualMap; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class FilesDumpUtils { + + public static void dumpModFiles( + @NonNull final Path path, + @NonNull final VirtualMap, OnDiskValue> files, + @NonNull final DumpCheckpoint checkpoint) { + try (@NonNull final var writer = new Writer(path)) { + final var dumpableFiles = gatherModFiles(files); + reportOnFiles(writer, dumpableFiles); + System.out.printf( + "=== mod files report is %d bytes at checkpoint %s%n", writer.getSize(), checkpoint.name()); + } + } + + public static void dumpMonoFiles( + @NonNull final Path path, + @NonNull final VirtualMap files, + @NonNull final DumpCheckpoint checkpoint) { + try (@NonNull final var writer = new Writer(path)) { + final var dumpableFiles = gatherMonoFiles(files); + reportOnFiles(writer, dumpableFiles); + System.out.printf( + "=== mono files report is %d bytes at checkpoint %s%n", writer.getSize(), checkpoint.name()); + } + } + + @NonNull + private static Map gatherModFiles(VirtualMap, OnDiskValue> source) { + final var r = new HashMap(); + final var threadCount = 8; + final var files = new ConcurrentLinkedQueue>(); + try { + VirtualMapLike.from(source) + .extractVirtualMapData( + getStaticThreadManager(), + p -> files.add(Pair.of(FileId.fromMod(p.left().getKey()), HederaFile.fromMod(p.right()))), + threadCount); + } catch (final InterruptedException ex) { + System.err.println("*** Traversal of files virtual map interrupted!"); + Thread.currentThread().interrupt(); + } + files.forEach(filePair -> r.put(filePair.key(), filePair.value())); + return r; + } + + @NonNull + private static Map gatherMonoFiles(VirtualMap source) { + final var r = new HashMap(); + final var threadCount = 8; + final var files = new ConcurrentLinkedQueue>(); + try { + VirtualMapLike.from(source) + .extractVirtualMapData( + getStaticThreadManager(), + p -> files.add( + Pair.of(FileId.fromMono(p.left()), HederaFile.fromMono(p.left(), p.right()))), + threadCount); + } catch (final InterruptedException ex) { + System.err.println("*** Traversal of files virtual map interrupted!"); + Thread.currentThread().interrupt(); + } + files.forEach(filePair -> r.put(filePair.key(), filePair.value())); + return r; + } + + private static void reportOnFiles(@NonNull final Writer writer, @NonNull final Map files) { + reportFileContentsHeader(writer); + reportFileContents(writer, files); + writer.writeln(""); + } + + /** Emits the CSV header line for the file contents - **KEEP IN SYNC WITH reportFileContents!!!** */ + private static void reportFileContentsHeader(@NonNull final Writer writer) { + final var header = "fileId,PRESENT/DELETED,SPECIAL file,SYSTEM file,length(bytes),expiry,memo,content,key"; + writer.write("%s%n", header); + } + + /** Emits the actual content (hexified) for each file, and it's full key */ + private static void reportFileContents( + @NonNull final Writer writer, @NonNull final Map allFiles) { + for (@NonNull + final var file : + allFiles.entrySet().stream().sorted(Map.Entry.comparingByKey()).toList()) { + + final var fileId = file.getKey().fileNum(); + final var hf = file.getValue(); + if (hf.isActive()) { + final var sb = new StringBuilder(); + toStringOfByteArray(sb, hf.contents()); + writer.write( + "%d,PRESENT,%s,%s,%d,%s,%s,%s,%s%n", + fileId, + hf.fileStore() == FileStore.SPECIAL ? "SPECIAL" : "", + hf.systemFileType() != null ? hf.systemFileType().name() : "", + hf.contents().length, + hf.metadata() != null ? Long.toString(hf.metadata().getExpiry()) : "", + hf.metadata() != null ? quoteForCsv(",", hf.metadata().getMemo()) : "", + sb, + hf.metadata() != null + ? quoteForCsv( + ",", + squashLinesToEscapes( + MiscUtils.describe(hf.metadata().getWacl()))) + : ""); + } else { + writer.write("%d,DELETED%n", fileId); + } + } + } +} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java new file mode 100644 index 000000000000..1c2f0e81505f --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.bbm.files; + +import com.hedera.hapi.node.state.file.File; +import com.hedera.node.app.service.mono.files.HFileMeta; +import com.hedera.node.app.service.mono.state.virtual.VirtualBlobKey; +import com.hedera.node.app.service.mono.state.virtual.VirtualBlobValue; +import com.hedera.node.app.state.merkle.disk.OnDiskValue; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; + +/** Holds the content and the metadata for a single data file in the store */ +@SuppressWarnings("java:S6218") // "Equals/hashcode methods should be overridden in records containing array fields" +// not using this with equals +record HederaFile( + @NonNull FileStore fileStore, + @NonNull Integer fileId, + @NonNull byte[] contents, + @Nullable HFileMeta metadata, + @Nullable SystemFileType systemFileType) { + + static HederaFile fromMod(@NonNull final OnDiskValue wrapper) { + final var value = wrapper.getValue(); + + return new HederaFile( + FileStore.ORDINARY, + (int) value.fileId().fileNum(), + value.contents().toByteArray(), + null, + SystemFileType.byId.get((int) value.fileId().fileNum())); + } + + static HederaFile fromMono(@NonNull final VirtualBlobKey key, @NonNull final VirtualBlobValue file) { + return new HederaFile( + FileStore.ORDINARY, + key.getEntityNumCode(), + file.getData(), + null, + SystemFileType.byId.get(key.getEntityNumCode())); + } + + boolean isActive() { + if (null != systemFileType) { + return true; + } + if (null != metadata) { + return !metadata.isDeleted(); + } + return false; + } +} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/SystemFileType.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/SystemFileType.java new file mode 100644 index 000000000000..567680fa40bc --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/SystemFileType.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.node.app.bbm.files; + +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; + +enum SystemFileType { + ADDRESS_BOOK(101), + NODE_DETAILS(102), + FEE_SCHEDULES(111), + EXCHANGE_RATES(112), + NETWORK_PROPERTIES(121), + HAPI_PERMISSIONS(122), + THROTTLE_DEFINITIONS(123), + SOFTWARE_UPDATE0(150), + SOFTWARE_UPDATE1(151), + SOFTWARE_UPDATE2(152), + SOFTWARE_UPDATE3(153), + SOFTWARE_UPDATE4(154), + SOFTWARE_UPDATE5(155), + SOFTWARE_UPDATE6(156), + SOFTWARE_UPDATE7(157), + SOFTWARE_UPDATE8(158), + SOFTWARE_UPDATE9(159), + UNKNOWN(-1); + + public final int id; + + static final Map byId = new HashMap<>(); + + SystemFileType(final int id) { + this.id = id; + } + + static { + EnumSet.allOf(SystemFileType.class).forEach(e -> byId.put(e.id, e)); + } +} From e619a743f3f2bc93fdca16095c10b84aba29e140 Mon Sep 17 00:00:00 2001 From: Valentin Tronkov <99957253+vtronkov@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:37:28 +0200 Subject: [PATCH 2/4] nit: polishing Signed-off-by: Valentin Tronkov <99957253+vtronkov@users.noreply.github.com> --- .../src/main/java/com/hedera/node/app/bbm/StateDumper.java | 4 ++-- .../java/com/hedera/node/app/bbm/files/FilesDumpUtils.java | 1 - .../main/java/com/hedera/node/app/bbm/files/HederaFile.java | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java index 3d014646bbd5..a709ddbee565 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java @@ -77,9 +77,9 @@ public static void dumpModChildrenFrom( requireNonNull(state.getChild(state.findNodeIndex(TokenService.NAME, NFTS_KEY))); dumpModUniqueTokens(Paths.get(dumpLoc, SEMANTIC_UNIQUE_TOKENS), uniqueTokens, checkpoint); - final VirtualMap, OnDiskValue> test = + final VirtualMap, OnDiskValue> files = requireNonNull(state.getChild(state.findNodeIndex(FileService.NAME, FileServiceImpl.BLOBS_KEY))); - dumpModFiles(Paths.get(dumpLoc, SEMANTIC_FILES), test, checkpoint); + dumpModFiles(Paths.get(dumpLoc, SEMANTIC_FILES), files, checkpoint); } private static String getExtantDumpLoc( diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java index b2a5162f3ace..664659b3da59 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java @@ -122,7 +122,6 @@ private static void reportFileContents( for (@NonNull final var file : allFiles.entrySet().stream().sorted(Map.Entry.comparingByKey()).toList()) { - final var fileId = file.getKey().fileNum(); final var hf = file.getValue(); if (hf.isActive()) { diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java index 1c2f0e81505f..4eef45a75973 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java @@ -36,7 +36,6 @@ record HederaFile( static HederaFile fromMod(@NonNull final OnDiskValue wrapper) { final var value = wrapper.getValue(); - return new HederaFile( FileStore.ORDINARY, (int) value.fileId().fileNum(), From 7e7ba83fc1b82456c8237d60a6fc06fe2b26f6c2 Mon Sep 17 00:00:00 2001 From: Valentin Tronkov <99957253+vtronkov@users.noreply.github.com> Date: Thu, 8 Feb 2024 14:14:37 +0200 Subject: [PATCH 3/4] fix: fix the mono file gatherer Signed-off-by: Valentin Tronkov <99957253+vtronkov@users.noreply.github.com> --- .../com/hedera/node/app/bbm/files/FileId.java | 5 +- .../node/app/bbm/files/FilesDumpUtils.java | 89 ++++++++++++++++--- .../hedera/node/app/bbm/files/HederaFile.java | 17 ++-- 3 files changed, 89 insertions(+), 22 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java index db9858715bde..502aa3c3622d 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FileId.java @@ -19,7 +19,6 @@ import com.google.common.collect.ComparisonChain; import com.hedera.hapi.node.base.FileID; import com.hedera.node.app.bbm.utils.Writer; -import com.hedera.node.app.service.mono.state.virtual.VirtualBlobKey; import edu.umd.cs.findbugs.annotations.NonNull; record FileId(long shardNum, long realmNum, long fileNum) implements Comparable { @@ -28,8 +27,8 @@ static FileId fromMod(@NonNull final FileID fileID) { return new FileId(fileID.shardNum(), fileID.realmNum(), fileID.fileNum()); } - static FileId fromMono(@NonNull final VirtualBlobKey key) { - return new FileId(0, 0, key.getEntityNumCode()); + static FileId fromMono(@NonNull final Integer fileNum) { + return new FileId(0, 0, fileNum); } @Override diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java index 664659b3da59..eeb552565441 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/FilesDumpUtils.java @@ -25,6 +25,8 @@ import com.hedera.hapi.node.state.file.File; import com.hedera.node.app.bbm.DumpCheckpoint; import com.hedera.node.app.bbm.utils.Writer; +import com.hedera.node.app.service.mono.files.HFileMeta; +import com.hedera.node.app.service.mono.files.MetadataMapFactory; import com.hedera.node.app.service.mono.state.adapters.VirtualMapLike; import com.hedera.node.app.service.mono.state.virtual.VirtualBlobKey; import com.hedera.node.app.service.mono.state.virtual.VirtualBlobValue; @@ -35,12 +37,20 @@ import com.swirlds.virtualmap.VirtualMap; import edu.umd.cs.findbugs.annotations.NonNull; import java.nio.file.Path; +import java.util.EnumSet; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; public class FilesDumpUtils { + private FilesDumpUtils() { + // Utility class + } + public static void dumpModFiles( @NonNull final Path path, @NonNull final VirtualMap, OnDiskValue> files, @@ -84,23 +94,82 @@ private static Map gatherModFiles(VirtualMap gatherMonoFiles(VirtualMap source) { - final var r = new HashMap(); - final var threadCount = 8; - final var files = new ConcurrentLinkedQueue>(); + private static Map gatherMonoFiles( + @NonNull final VirtualMap source) { + final var foundFiles = new ConcurrentHashMap(); + final var foundMetadata = new ConcurrentHashMap(); + + final var nType = new ConcurrentHashMap(); + final var nNullValues = new ConcurrentHashMap(); + final var nNullMetadataValues = new AtomicInteger(); + + Stream.of(nType, nNullValues) + .forEach(m -> EnumSet.allOf(VirtualBlobKey.Type.class).forEach(t -> m.put(t, 0))); + + final int THREAD_COUNT = 8; + boolean didRunToCompletion = true; try { VirtualMapLike.from(source) - .extractVirtualMapData( + .extractVirtualMapDataC( getStaticThreadManager(), - p -> files.add( - Pair.of(FileId.fromMono(p.left()), HederaFile.fromMono(p.left(), p.right()))), - threadCount); + entry -> { + final var contractId = entry.key().getEntityNumCode(); + + final var type = entry.key().getType(); + nType.merge(type, 1, Integer::sum); + + final var value = entry.value().getData(); + if (null != value) { + switch (type) { + case FILE_DATA -> foundFiles.put(contractId, value); + + case FILE_METADATA -> { + final var metadata = MetadataMapFactory.toAttr(value); + if (null != metadata) { + foundMetadata.put(contractId, metadata); + } else { + nNullMetadataValues.incrementAndGet(); + + System.err.printf( + "*** collectFiles file metadata (HFileMeta) null for contract id %d, type %s%n", + contractId, type); + } + } + case CONTRACT_BYTECODE, SYSTEM_DELETED_ENTITY_EXPIRY -> {} + } + } else { + nNullValues.merge(type, 1, Integer::sum); + + System.err.printf( + "*** collectFiles file value (bytes) null for contract id %d, type %s%n", + contractId, type); + } + }, + THREAD_COUNT); } catch (final InterruptedException ex) { - System.err.println("*** Traversal of files virtual map interrupted!"); Thread.currentThread().interrupt(); + didRunToCompletion = false; } - files.forEach(filePair -> r.put(filePair.key(), filePair.value())); + + if (!didRunToCompletion) { + System.err.printf("*** collectFiles interrupted (did not run to completion)%n"); + } + + final var r = new HashMap(); + for (@NonNull final var e : foundFiles.entrySet()) { + final var contractId = e.getKey(); + final var contents = e.getValue(); + final var metadata = foundMetadata.getOrDefault(contractId, null); + r.put( + FileId.fromMono(contractId), + null != metadata + ? HederaFile.of(contractId, contents, metadata) + : HederaFile.of(contractId, contents)); + } + return r; } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java index 4eef45a75973..6bb722457821 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/files/HederaFile.java @@ -18,8 +18,6 @@ import com.hedera.hapi.node.state.file.File; import com.hedera.node.app.service.mono.files.HFileMeta; -import com.hedera.node.app.service.mono.state.virtual.VirtualBlobKey; -import com.hedera.node.app.service.mono.state.virtual.VirtualBlobValue; import com.hedera.node.app.state.merkle.disk.OnDiskValue; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; @@ -44,13 +42,14 @@ static HederaFile fromMod(@NonNull final OnDiskValue wrapper) { SystemFileType.byId.get((int) value.fileId().fileNum())); } - static HederaFile fromMono(@NonNull final VirtualBlobKey key, @NonNull final VirtualBlobValue file) { - return new HederaFile( - FileStore.ORDINARY, - key.getEntityNumCode(), - file.getData(), - null, - SystemFileType.byId.get(key.getEntityNumCode())); + @NonNull + static HederaFile of(final int fileId, @NonNull final byte[] contents) { + return new HederaFile(FileStore.ORDINARY, fileId, contents, null, SystemFileType.byId.get(fileId)); + } + + @NonNull + static HederaFile of(final int fileId, @NonNull final byte[] contents, @NonNull final HFileMeta metadata) { + return new HederaFile(FileStore.ORDINARY, fileId, contents, metadata, SystemFileType.byId.get(fileId)); } boolean isActive() { From cb6f341658e3d8086e47adc4c6676047b6ec9153 Mon Sep 17 00:00:00 2001 From: Valentin Tronkov <99957253+vtronkov@users.noreply.github.com> Date: Thu, 8 Feb 2024 14:17:06 +0200 Subject: [PATCH 4/4] nit: spotless Signed-off-by: Valentin Tronkov <99957253+vtronkov@users.noreply.github.com> --- .../src/main/java/com/hedera/node/app/bbm/StateDumper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java index 4d4ead6ebdee..2550497b22b6 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java @@ -24,8 +24,8 @@ import static com.hedera.node.app.bbm.nfts.UniqueTokenDumpUtils.dumpMonoUniqueTokens; import static com.hedera.node.app.records.BlockRecordService.BLOCK_INFO_STATE_KEY; import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.NETWORK_CTX; -import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.TOKEN_ASSOCIATIONS; import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.STORAGE; +import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.TOKEN_ASSOCIATIONS; import static com.hedera.node.app.service.mono.state.migration.StateChildIndices.UNIQUE_TOKENS; import static com.hedera.node.app.service.token.impl.TokenServiceImpl.NFTS_KEY; import static com.hedera.node.app.service.token.impl.TokenServiceImpl.TOKEN_RELS_KEY; @@ -82,11 +82,11 @@ public static void dumpModChildrenFrom( Optional.ofNullable(blockInfo.consTimeOfLastHandledTxn()) .map(then -> Instant.ofEpochSecond(then.seconds(), then.nanos())) .orElse(null)); - + final VirtualMap, OnDiskValue> uniqueTokens = requireNonNull(state.getChild(state.findNodeIndex(TokenService.NAME, NFTS_KEY))); dumpModUniqueTokens(Paths.get(dumpLoc, SEMANTIC_UNIQUE_TOKENS), uniqueTokens, checkpoint); - + final VirtualMap, OnDiskValue> tokenRelations = requireNonNull(state.getChild(state.findNodeIndex(TokenService.NAME, TOKEN_RELS_KEY))); dumpModTokenRelations(Paths.get(dumpLoc, SEMANTIC_TOKEN_RELATIONS), tokenRelations, checkpoint);