Skip to content

Commit

Permalink
fix: mod dumper
Browse files Browse the repository at this point in the history 
Signed-off-by: Valentin Tronkov <99957253+vtronkov@users.noreply.github.com>
  • Loading branch information
@vtronkov
vtronkov committed Feb 21, 2024
1 parent dfc70e3 commit 3ead2ff
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 80 deletions.
8 changes: 7 additions & 1 deletion hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/StateDumper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,19 @@
import static java.util.Objects.requireNonNull;

import com.hedera.hapi.node.base.AccountID;
import com.hedera.hapi.node.base.ContractID;
import com.hedera.hapi.node.base.FileID;
import com.hedera.hapi.node.base.NftID;
import com.hedera.hapi.node.base.TokenAssociation;
import com.hedera.hapi.node.state.blockrecords.BlockInfo;
import com.hedera.hapi.node.state.contract.Bytecode;
import com.hedera.hapi.node.state.token.Account;
import com.hedera.hapi.node.state.token.Nft;
import com.hedera.hapi.node.state.token.TokenRelation;
import com.hedera.node.app.bbm.contracts.ContractBytecodesDumpUtils;
import com.hedera.node.app.records.BlockRecordService;
import com.hedera.node.app.service.contract.ContractService;
import com.hedera.node.app.service.contract.impl.state.InitialModServiceContractSchema;
import com.hedera.node.app.service.file.FileService;
import com.hedera.node.app.service.file.impl.FileServiceImpl;
import com.hedera.node.app.service.mono.state.merkle.MerkleNetworkContext;
Expand Down Expand Up @@ -116,7 +120,9 @@ public static void dumpModChildrenFrom(
requireNonNull(state.getChild(state.findNodeIndex(TokenService.NAME, ACCOUNTS_KEY)));
dumpModAccounts(Paths.get(dumpLoc, SEMANTIC_ACCOUNTS), accounts, checkpoint);

dumpModContractBytecodes(Paths.get(dumpLoc, SEMANTIC_CONTRACT_BYTECODES), accounts, checkpoint);
final VirtualMap<OnDiskKey<ContractID>, OnDiskValue<Bytecode>> contracts = requireNonNull(state.getChild(
state.findNodeIndex(ContractService.NAME, InitialModServiceContractSchema.BYTECODE_KEY)));
dumpModContractBytecodes(Paths.get(dumpLoc, SEMANTIC_CONTRACT_BYTECODES), contracts, checkpoint);
}

private static String getExtantDumpLoc(
Expand Down
12 changes: 12 additions & 0 deletions hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/contracts/Contract.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@

package com.hedera.node.app.bbm.contracts;

import com.hedera.hapi.node.base.ContractID;
import com.hedera.hapi.node.state.contract.Bytecode;
import com.hedera.node.app.state.merkle.disk.OnDiskKey;
import com.hedera.node.app.state.merkle.disk.OnDiskValue;
import edu.umd.cs.findbugs.annotations.NonNull;
import java.util.Arrays;
import java.util.TreeSet;
Expand All @@ -36,6 +40,14 @@
public record Contract(
@NonNull TreeSet</*@NonNull*/ Integer> ids, @NonNull byte[] bytecode, @NonNull Validity validity) {

public static Contract fromMod(OnDiskKey<ContractID> id, OnDiskValue<Bytecode> bytecode) {
final var c = new Contract(new TreeSet<>(), bytecode.getValue().code().toByteArray(), Validity.ACTIVE);
if (id.getKey().contractNum() != null) {
c.ids().add(id.getKey().contractNum().intValue());
}
return c;
}

// For any set of contract ids with the same bytecode, the lowest contract id is used as the "canonical"
// id for that bytecode (useful for ordering contracts deterministically)
public int canonicalId() {
Expand Down
45 changes: 37 additions & 8 deletions ...edera-app/src/main/java/com/hedera/node/app/bbm/contracts/ContractBytecodesDumpUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,11 @@
package com.hedera.node.app.bbm.contracts;

import static com.hedera.node.app.bbm.contracts.ContractUtils.ESTIMATED_NUMBER_OF_CONTRACTS;
import static com.swirlds.common.threading.manager.AdHocThreadManager.getStaticThreadManager;

import com.hedera.hapi.node.base.AccountID;
import com.hedera.hapi.node.state.token.Account;
import com.hedera.hapi.node.base.ContractID;
import com.hedera.hapi.node.state.contract.Bytecode;
import com.hedera.node.app.bbm.DumpCheckpoint;
import com.hedera.node.app.bbm.accounts.AccountDumpUtils;
import com.hedera.node.app.bbm.accounts.HederaAccount;
import com.hedera.node.app.bbm.utils.Writer;
import com.hedera.node.app.service.mono.state.adapters.VirtualMapLike;
import com.hedera.node.app.service.mono.state.migration.AccountStorageAdapter;
Expand All @@ -41,6 +40,8 @@
import java.util.HexFormat;
import java.util.List;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;

Expand All @@ -52,11 +53,10 @@ private ContractBytecodesDumpUtils() {

public static void dumpModContractBytecodes(
@NonNull final Path path,
@NonNull final VirtualMap<OnDiskKey<AccountID>, OnDiskValue<Account>> accounts,
@NonNull final VirtualMap<OnDiskKey<ContractID>, OnDiskValue<Bytecode>> contracts,
@NonNull final DumpCheckpoint checkpoint) {
final var dumpableAccounts = AccountDumpUtils.gatherAccounts(accounts, HederaAccount::fromMod);
final var contracts = ContractUtils.getModContracts(dumpableAccounts);
final var sb = generateReport(contracts);
final var dumpableAccounts = gatherModContracts(contracts);
final var sb = generateReport(dumpableAccounts);
try (@NonNull final var writer = new Writer(path)) {
writer.writeln(sb.toString());
System.out.printf(
Expand All @@ -65,6 +65,31 @@ public static void dumpModContractBytecodes(
}
}

@NonNull
public static Contracts gatherModContracts(VirtualMap<OnDiskKey<ContractID>, OnDiskValue<Bytecode>> contracts) {
final var contractsToReturn = new ConcurrentLinkedQueue<Contract>();
final var threadCount = 8;
final var processed = new AtomicInteger();

try {
VirtualMapLike.from(contracts)
.extractVirtualMapData(
getStaticThreadManager(),
p -> {
processed.incrementAndGet();
contractsToReturn.add(Contract.fromMod(p.left(), p.right()));
},
threadCount);
} catch (final InterruptedException ex) {
System.err.println("*** Traversal of contracts virtual map interrupted!");
Thread.currentThread().interrupt();
}

final var contractArr = contractsToReturn.toArray(new Contract[0]);
System.out.printf("=== %d contracts iterated over (%d saved)%n", processed.get(), contractArr.length);
return new Contracts(List.of(contractArr), List.of(), contractArr.length);
}

public static void dumpMonoContractBytecodes(
@NonNull final Path path,
@NonNull final VirtualMap<EntityNumVirtualKey, OnDiskAccount> accounts,
Expand All @@ -82,6 +107,10 @@ public static void dumpMonoContractBytecodes(
}

private static StringBuilder generateReport(Contracts knownContracts) {
if (knownContracts.contracts().isEmpty()) {
return new StringBuilder();
}

var r = getNonTrivialContracts(knownContracts);
var contractsWithBytecode = r.getLeft();
var zeroLengthContracts = r.getRight();
Expand Down
71 changes: 0 additions & 71 deletions hedera-node/hedera-app/src/main/java/com/hedera/node/app/bbm/contracts/ContractUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,28 +16,13 @@

package com.hedera.node.app.bbm.contracts;

import com.hedera.hapi.node.base.ContractID;
import com.hedera.hapi.node.base.SemanticVersion;
import com.hedera.hapi.node.state.contract.Bytecode;
import com.hedera.node.app.bbm.accounts.HederaAccount;
import com.hedera.node.app.service.contract.ContractService;
import com.hedera.node.app.service.contract.impl.state.InitialModServiceContractSchema;
import com.hedera.node.app.service.mono.state.adapters.VirtualMapLike;
import com.hedera.node.app.service.mono.state.migration.AccountStorageAdapter;
import com.hedera.node.app.service.mono.state.virtual.VirtualBlobKey;
import com.hedera.node.app.service.mono.state.virtual.VirtualBlobKey.Type;
import com.hedera.node.app.service.mono.state.virtual.VirtualBlobValue;
import com.hedera.node.app.service.mono.utils.NonAtomicReference;
import com.hedera.node.app.spi.state.ReadableKVState;
import com.hedera.node.app.spi.state.StateDefinition;
import com.hedera.node.app.state.merkle.StateMetadata;
import com.hedera.node.app.state.merkle.memory.InMemoryKey;
import com.hedera.node.app.state.merkle.memory.InMemoryValue;
import com.hedera.node.app.state.merkle.memory.InMemoryWritableKVState;
import com.swirlds.merkle.map.MerkleMap;
import edu.umd.cs.findbugs.annotations.NonNull;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Objects;
Expand All @@ -46,8 +31,6 @@

public class ContractUtils {

private static final SemanticVersion CURRENT_VERSION = new SemanticVersion(0, 47, 0, "SNAPSHOT", "");

static final int ESTIMATED_NUMBER_OF_CONTRACTS = 100_000;
static final int ESTIMATED_NUMBER_OF_DELETED_CONTRACTS = 10_000;

Expand Down Expand Up @@ -119,58 +102,4 @@ public static Contracts getMonoContracts(
}
return codes;
}

static Contracts getModContracts(HederaAccount[] dumpableAccounts) {
final var smartContracts = Arrays.stream(dumpableAccounts)
.filter(HederaAccount::smartContract)
.toList();
final var deletedSmartContract =
smartContracts.stream().filter(HederaAccount::deleted).toList();

final var extractedFiles = getContractStore();

final var contractContents = new ArrayList<Contract>(ESTIMATED_NUMBER_OF_CONTRACTS);
for (final var smartContract : smartContracts) {
final var contractId = getContractId(smartContract);
final var fileId = ContractID.newBuilder().contractNum(contractId).build();
if (extractedFiles.contains(fileId)) {
final var bytecode = extractedFiles.get(fileId);
if (null != bytecode) {
final var c = new Contract(
new TreeSet<>(),
bytecode.code().toByteArray(),
deletedSmartContract.contains(smartContract) ? Validity.DELETED : Validity.ACTIVE);
c.ids().add(contractId);
contractContents.add(c);
}
}
}

final var deletedContractIds =
deletedSmartContract.stream().map(ContractUtils::getContractId).toList();
return new Contracts(contractContents, deletedContractIds, smartContracts.size());
}

private static int getContractId(HederaAccount contract) {
if (contract.accountId() == null || contract.accountId().accountNum() == null) {
return 0;
}
return contract.accountId().accountNum().intValue();
}

private static ReadableKVState<ContractID, Bytecode> getContractStore() {
final var contractSchema = new InitialModServiceContractSchema(CURRENT_VERSION);
final var contractSchemas = contractSchema.statesToCreate();
final StateDefinition<ContractID, Bytecode> contractStoreStateDefinition = contractSchemas.stream()
.filter(sd -> sd.stateKey().equals(InitialModServiceContractSchema.BYTECODE_KEY))
.findFirst()
.orElseThrow();
final var contractStoreSchemaMetadata =
new StateMetadata<>(ContractService.NAME, contractSchema, contractStoreStateDefinition);
final var contractMerkleMap = new NonAtomicReference<
MerkleMap<InMemoryKey<ContractID>, InMemoryValue<ContractID, Bytecode>>>(new MerkleMap<>());
final var toStore = new NonAtomicReference<ReadableKVState<ContractID, Bytecode>>(
new InMemoryWritableKVState<>(contractStoreSchemaMetadata, contractMerkleMap.get()));
return toStore.get();
}
}

0 comments on commit 3ead2ff

Please sign in to comment.