netgoal: fix large db generation (#5445)

cmd/netgoal/generate.go

@@ -52,6 +52,8 @@ var accountsCount uint64
 var assetsCount uint64
 var applicationCount uint64
 var balRange []string
+var lastPartKeyRound uint64
+var deterministicKeys bool
 
 func init() {
 	rootCmd.AddCommand(generateCmd)
@@ -77,6 +79,8 @@ func init() {
 	generateCmd.Flags().Uint64VarP(&assetsCount, "nassets", "", 5, "Asset count")
 	generateCmd.Flags().Uint64VarP(&applicationCount, "napps", "", 7, "Application Count")
 	generateCmd.Flags().StringArrayVar(&balRange, "bal", []string{}, "Application Count")
+	generateCmd.Flags().BoolVarP(&deterministicKeys, "deterministic", "", false, "Whether to generate deterministic keys")
+	generateCmd.Flags().Uint64VarP(&lastPartKeyRound, "last-part-key-round", "", gen.DefaultGenesis.LastPartKeyRound, "LastPartKeyRound in genesis.json")
 
 	longParts := make([]string, len(generateTemplateLines)+1)
 	longParts[0] = generateCmd.Long
@@ -184,7 +188,7 @@ template modes for -t:`,
 			if len(balRange) < 2 {
 				reportErrorf("must specify account balance range with --bal.")
 			}
-			err = generateAccountsLoadingFileTemplate(outputFilename, sourceWallet, rounds, roundTxnCount, accountsCount, assetsCount, applicationCount, balRange)
+			err = generateAccountsLoadingFileTemplate(outputFilename, sourceWallet, rounds, roundTxnCount, accountsCount, assetsCount, applicationCount, balRange, deterministicKeys)
 		default:
 			reportInfoln("Please specify a valid template name.\nSupported templates are:")
 			for _, line := range generateTemplateLines {
@@ -528,6 +532,9 @@ func generateWalletGenesis(filename string, wallets, npnNodes int) error {
 }
 
 func saveGenesisDataToDisk(genesisData gen.GenesisData, filename string) error {
+	if lastPartKeyRound != 0 {
+		genesisData.LastPartKeyRound = lastPartKeyRound
+	}
 	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err == nil {
 		defer f.Close()
@@ -538,7 +545,7 @@ func saveGenesisDataToDisk(genesisData gen.GenesisData, filename string) error {
 	return err
 }
 
-func generateAccountsLoadingFileTemplate(templateFilename, sourceWallet string, rounds, roundTxnCount, accountsCount, assetsCount, applicationCount uint64, balRange []string) error {
+func generateAccountsLoadingFileTemplate(templateFilename, sourceWallet string, rounds, roundTxnCount, accountsCount, assetsCount, applicationCount uint64, balRange []string, deterministicKeys bool) error {
 
 	min, err := strconv.ParseInt(balRange[0], 0, 64)
 	if err != nil {
@@ -557,6 +564,7 @@ func generateAccountsLoadingFileTemplate(templateFilename, sourceWallet string,
 		GeneratedApplicationCount: applicationCount,
 		SourceWalletName:          sourceWallet,
 		BalanceRange:              []int64{min, max},
+		DeterministicKeys:         deterministicKeys,
 	}
 	return saveLoadingFileDataToDisk(data, templateFilename)
 }

netdeploy/remote/bootstrappedNetwork.go

@@ -21,7 +21,7 @@ import (
 	"os"
 )
 
-//BootstrappedNetwork contains the specs for generating db files
+// BootstrappedNetwork contains the specs for generating db files
 type BootstrappedNetwork struct {
 	NumRounds                 uint64  `json:"numRounds"`
 	RoundTransactionsCount    uint64  `json:"roundTransactionsCount"`
@@ -30,6 +30,7 @@ type BootstrappedNetwork struct {
 	GeneratedApplicationCount uint64  `json:"generatedApplicationCount"`
 	SourceWalletName          string  `json:"sourceWalletName"`
 	BalanceRange              []int64 `json:"acctBalanceRange"`
+	DeterministicKeys         bool    `json:"deterministicKeys"`
 }
 
 // LoadBootstrappedData loads a bootstrappedFile structure from a json file

netdeploy/remote/deployedNetwork.go

@@ -92,6 +92,9 @@ type netState struct {
 	assetPerAcct int
 	appsPerAcct  int
 
+	deterministicKeys         bool
+	deterministicAccountCount uint64
+
 	genesisID   string
 	genesisHash crypto.Digest
 	poolAddr    basics.Address
@@ -351,7 +354,7 @@ func (cfg DeployedNetwork) BuildNetworkFromTemplate(buildCfg BuildConfig, rootDi
 	return
 }
 
-//GenerateDatabaseFiles generates database files according to the configurations
+// GenerateDatabaseFiles generates database files according to the configurations
 func (cfg DeployedNetwork) GenerateDatabaseFiles(fileCfgs BootstrappedNetwork, genesisFolder string) error {
 
 	accounts := make(map[basics.Address]basics.AccountData)
@@ -391,16 +394,17 @@ func (cfg DeployedNetwork) GenerateDatabaseFiles(fileCfgs BootstrappedNetwork, g
 	log := logging.NewLogger()
 
 	bootstrappedNet := netState{
-		nAssets:       fileCfgs.GeneratedAssetsCount,
-		nApplications: fileCfgs.GeneratedApplicationCount,
-		txnState:      protocol.PaymentTx,
-		roundTxnCnt:   fileCfgs.RoundTransactionsCount,
-		round:         basics.Round(0),
-		genesisID:     genesis.ID(),
-		genesisHash:   genesis.Hash(),
-		poolAddr:      poolAddr,
-		sinkAddr:      sinkAddr,
-		log:           log,
+		nAssets:           fileCfgs.GeneratedAssetsCount,
+		nApplications:     fileCfgs.GeneratedApplicationCount,
+		txnState:          protocol.PaymentTx,
+		roundTxnCnt:       fileCfgs.RoundTransactionsCount,
+		round:             basics.Round(0),
+		genesisID:         genesis.ID(),
+		genesisHash:       genesis.Hash(),
+		poolAddr:          poolAddr,
+		sinkAddr:          sinkAddr,
+		log:               log,
+		deterministicKeys: fileCfgs.DeterministicKeys,
 	}
 
 	var params config.ConsensusParams
@@ -422,6 +426,9 @@ func (cfg DeployedNetwork) GenerateDatabaseFiles(fileCfgs BootstrappedNetwork, g
 	rand.Seed(time.Now().UnixNano())
 	min := fileCfgs.BalanceRange[0]
 	max := fileCfgs.BalanceRange[1]
+	// TODO: Randomly assigning target balance in a range may cause tests to behave unpredictably,
+	// if the randomly selected balance is too low for proper testing.
+	// consider inserting a hardcoded balance sufficient for your tests.
 	bal := rand.Int63n(max-min) + min
 	bootstrappedNet.fundPerAccount = basics.MicroAlgos{Raw: uint64(bal)}
 	srcAcct := accounts[src]
@@ -455,6 +462,10 @@ func (cfg DeployedNetwork) GenerateDatabaseFiles(fileCfgs BootstrappedNetwork, g
 	for i := uint64(bootstrappedNet.round); i < fileCfgs.NumRounds; i++ {
 		bootstrappedNet.round++
 		blk, _ := createBlock(src, prev, fileCfgs.RoundTransactionsCount, &bootstrappedNet, params, log)
+		// don't allow the ledger to fall more than 10 rounds behind before adding more
+		for int(bootstrappedNet.round)-int(l.LatestTrackerCommitted()) > 10 {
+			time.Sleep(100 * time.Millisecond)
+		}
 		err = l.AddBlock(blk, agreement.Certificate{Round: bootstrappedNet.round})
 		if err != nil {
 			fmt.Printf("Error  %v\n", err)
@@ -486,6 +497,15 @@ func getGenesisAlloc(name string, allocation []bookkeeping.GenesisAllocation) bo
 	return bookkeeping.GenesisAllocation{}
 }
 
+// deterministicKeypair returns a key based on the provided index
+func deterministicKeypair(i uint64) *crypto.SignatureSecrets {
+	var seed crypto.Seed
+	binary.LittleEndian.PutUint64(seed[:], i)
+	s := crypto.GenerateSignatureSecrets(seed)
+	return s
+}
+
+// keypair returns a random key
 func keypair() *crypto.SignatureSecrets {
 	var seed crypto.Seed
 	crypto.RandBytes(seed[:])
@@ -576,6 +596,10 @@ func generateAccounts(src basics.Address, roundTxnCnt uint64, prev bookkeeping.B
 		//create accounts
 		bootstrappedNet.round++
 		blk, _ := createBlock(src, prev, roundTxnCnt, bootstrappedNet, csParams, log)
+		// don't allow the ledger to fall more than 10 rounds behind before adding more
+		for int(bootstrappedNet.round)-int(l.LatestTrackerCommitted()) > 10 {
+			time.Sleep(100 * time.Millisecond)
+		}
 		err := l.AddBlock(blk, agreement.Certificate{Round: bootstrappedNet.round})
 		if err != nil {
 			fmt.Printf("Error %v\n", err)
@@ -659,7 +683,13 @@ func createSignedTx(src basics.Address, round basics.Round, params config.Consen
 
 		if !bootstrappedNet.accountsCreated {
 			for i := uint64(0); i < n; i++ {
-				secretDst := keypair()
+				var secretDst *crypto.SignatureSecrets
+				if bootstrappedNet.deterministicKeys {
+					secretDst = deterministicKeypair(bootstrappedNet.deterministicAccountCount)
+					bootstrappedNet.deterministicAccountCount++
+				} else {
+					secretDst = keypair()
+				}
 				dst := basics.Address(secretDst.SignatureVerifier)
 				bootstrappedNet.accounts = append(bootstrappedNet.accounts, dst)
 
@@ -1064,6 +1094,9 @@ func computeRootStorage(nodeCount, relayCount int) int {
 	// 10 per node should be good for a week (add relayCount * 0 so param is used)
 	minGB := 20 + (nodeCount * 10) + (relayCount * 50)
 	return minGB
+	// TODO: this function appears to insufficiently provision EBS nodes in some cases
+	// if your nodes have insufficient storage, consider using a reasonable hardcoded value like
+	// return 256
 }
 
 func computeSSDStorage(nodeCount, relayCount int) int {

test/testdata/deployednettemplates/recipes/bootstrappedScenario/Makefile

@@ -1,6 +1,6 @@
 # bootstrappedScenario is scenario1s but with pre-built 30_000_000 accountdb
-PARAMS=-w 20 -R 8 -N 20 -n 20 --npn-algod-nodes 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json
-FILEPARAMS=--rounds 1600 --ntxns 20000 --naccounts 30000000 --nassets 20000 --napps 20000 --wallet-name "wallet1" --bal 100000 --bal 1000000
+PARAMS=-w 20 -R 8 -N 20 -n 20 --npn-algod-nodes 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json --last-part-key-round 50000
+FILEPARAMS=--rounds 1600 --ntxns 20000 --naccounts 30000000 --nassets 20000 --napps 20000 --wallet-name "wallet1" --bal 50000000 --bal 50000001 --deterministic
 
 all:	net.json genesis.json topology.json boostrappedFile.json
 

test/testdata/deployednettemplates/recipes/bootstrappedScenario/boostrappedFile.json

@@ -6,7 +6,8 @@
 	"generatedApplicationCount": 20000,
 	"sourceWalletName": "wallet1",
 	"acctBalanceRange": [
-		100000,
-		1000000
-	]
+		50000000,
+		50000001
+	],
+	"deterministicKeys": true
 }