tools: enable consecutive test runs in the block generator (#5409)

Co-authored-by: Zeph Grunschlag <tzaffi@users.noreply.github.com>

Makefile

@@ -202,7 +202,7 @@ rebuild_kmd_swagger: deps
 
 # develop
 
-build: buildsrc
+build: buildsrc buildsrc-special
 
 # We're making an empty file in the go-cache dir to
 # get around a bug in go build where it will fail
@@ -213,6 +213,10 @@ buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_e
 	touch "${GOCACHE}"/file.txt && \
 	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
+buildsrc-special:
+	cd tools/block-generator && \
+	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
+
 check-go-version:
 	./scripts/check_golang_version.sh build
 

go.mod

@@ -36,7 +36,6 @@ require (
 	golang.org/x/sys v0.7.0
 	golang.org/x/text v0.9.0
 	gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -72,4 +71,5 @@ require (
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

tools/block-generator/README.md

@@ -5,14 +5,16 @@ This tool is used for testing Conduit import performance. It does this by genera
 ## Scenario Configuration
 
 Block generator uses a YAML config file to describe the composition of each randomly generated block. There are three levels of configuration:
+
 1. Setup
 2. Transaction type distribution
 3. Transaction type specific configuration
 
 At the time of writing, the block generator supports **payment** and **asset** transactions. The settings are hopefully, more or less, obvious. Distributions are specified as fractions of 1.0, and the sum of all options must add up to 1.0.
 
 Here is an example which uses all of the current options. Notice that the synthetic blocks are not required to follow algod limits, in this case the block size is specified as 19999:
-```
+
+```yml
 name: "Mixed (19,999)"
 genesis_accounts: 10000
 genesis_account_balance: 1000000000000
@@ -37,14 +39,16 @@ asset_delete_fraction: 0
 ## Modes
 
 The block generator can run in one of two _modes_:
+
 1. standalone **daemon**
 2. test suite **runner**
 
 ### daemon
 
 In standalone daemon mode, a block-generator process starts and exposes the mock algod endpoints for **/genesis** and **/v2/blocks/{block}**. If you choose to query them manually, it only supports fetching blocks sequentially. This is due to the fact that it generates a pseudorandom stream of transactions and after each random transaction the state increments to the next.
 
-Here is the help output for **daemon**: 
+Here is the help output for **daemon**:
+
 ```bash
 ~$ ./block-generator daemon -h
 Start the generator daemon in standalone mode.
@@ -60,12 +64,13 @@ Flags:
 
 ### runner
 
-The runner mode is well suited for runing the same set of tests consistently across many scenarios and for different releases. The runner mode automates this process by starting the **daemon** with many different configurations, managing a postgres database, and running a separate Conduit process configured to use them.
+The runner mode is well suited for running the same set of tests consistently across many scenarios and for different releases. The runner mode automates this process by starting the **daemon** with many different configurations, managing a postgres database, and running a separate Conduit process configured to use them.
 
 The results of the testing are written to the directory specified by the **--report-directory** option, and include many different metrics. In addition to the report, the Conduit log is written to this directory. The files are named according to the scenario file, and end in "report" or "log".
 
 Here is an example report from running with a test duration of "1h":
-```
+
+```json
 test_duration_seconds:3600
 test_duration_actual_seconds:3600.056457
 transaction_pay_total:30024226
@@ -95,6 +100,7 @@ final_uptime_seconds:3600.06
 ```
 
 Here is the help output for **runner**:
+
 ```bash
 ~$ ./block-generator runner -h
 Run an automated test suite using the block-generator daemon and a provided conduit binary. Results are captured to a specified output directory.
@@ -117,15 +123,17 @@ Flags:
       --validate                            If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure.
 ```
 
-## Example Scenario Run using Conduit and Postgres - `run_runner.sh`
+## Example Run using Conduit and Postgres in **bash** via `run_runner.sh`
 
 A typical **runner** scenario involves:
+
 * a [scenario configuration](#scenario-configuration) file, e.g. [test_config.yml](./test_config.yml)
 * access to a `conduit` binary to query the block generator's mock Algod endpoint and ingest the synthetic blocks
 * a datastore -such as a postgres database- to collect `conduit`'s output
 * a `conduit` config file to define its import/export behavior
 
 `run_runner.sh` makes the following choices for the previous bullet points:
+
 * it can accept any scenario as its second argument, but defaults to [test_config.yml](./test_config.yml) when this isn't provided (this is a scenario with a lifetime of ~30 seconds)
 * knows how to import through a mock Algod running on port 11112 (which is the port the runner avails)
 * sets up a dockerized postgres database to receive conduit's output
@@ -137,16 +145,17 @@ First you'll need to get a `conduit` binary. For example you can follow the [dev
 
 Assume you've navigated to the `tools/block-generator` directory of 
 the `go-algorand` repo, and:
+
 * saved the conduit binary to `tools/block-generator/conduit`
 * created a block generator scenario config at `tools/block-generator/scenario.yml`
 
 Then you can execute the following command to run the scenario:
+
 ```sh
 ./run_runner.sh ./conduit scenario.yml 
 ```
 
 ### Scenario Report
 
-If all goes well, the run will generate a directory `tools/block-generator/OUTPUT_RUN_RUNNER_TEST` and in that directory you can see the statistics 
-of the run in `scenario.report`.
-
+If all goes well, the run will generate a directory `tmp/OUTPUT_RUN_RUNNER_TEST`
+and in that directory you can see the statisticsn of the run in `scenario.report`.

tools/block-generator/generator/generate.go

@@ -130,7 +130,7 @@ func MakeGenerator(dbround uint64, bkGenesis bookkeeping.Genesis, config Generat
 		rewardsRate:               0,
 		rewardsRecalculationRound: 0,
 		reportData:                make(map[TxTypeID]TxData),
-		nextdbround:               dbround,
+		roundOffset:               dbround,
 	}
 
 	gen.feeSink[31] = 1
@@ -242,8 +242,7 @@ type generator struct {
 	// ledger
 	ledger *ledger.Ledger
 
-	// next_account_round in the preloaded database
-	nextdbround uint64
+	roundOffset uint64
 }
 
 type assetData struct {
@@ -286,7 +285,7 @@ func (g *generator) WriteReport(output io.Writer) error {
 
 func (g *generator) WriteStatus(output io.Writer) error {
 	response := model.NodeStatusResponse{
-		LastRound: g.round + g.nextdbround,
+		LastRound: g.round + g.roundOffset,
 	}
 	return json.NewEncoder(output).Encode(response)
 }
@@ -378,24 +377,21 @@ func (g *generator) finishRound(txnCount uint64) {
 
 // WriteBlock generates a block full of new transactions and writes it to the writer.
 func (g *generator) WriteBlock(output io.Writer, round uint64) error {
-	if round < g.nextdbround {
+	if round < g.roundOffset {
 		return fmt.Errorf("cannot generate block for round %d, already in database", round)
 	}
-	if round-g.nextdbround != g.round {
-		return fmt.Errorf("generator only supports sequential block access. Expected %d but received request for %d", g.round+g.nextdbround, round)
+	if round-g.roundOffset != g.round {
+		return fmt.Errorf("generator only supports sequential block access. Expected %d but received request for %d", g.round+g.roundOffset, round)
 	}
 	numTxnForBlock := g.txnForRound(g.round)
 
 	// return genesis block. offset round for non-empty database
-	if round-g.nextdbround == 0 {
+	if round-g.roundOffset == 0 {
 		// write the msgpack bytes for a block
-		block, cert, _ := g.ledger.BlockCert(basics.Round(round - g.nextdbround))
+		block, _, _ := g.ledger.BlockCert(basics.Round(round - g.roundOffset))
 		// return the block with the requested round number
 		block.BlockHeader.Round = basics.Round(round)
-		encodedblock := rpcs.EncodedBlockCert{
-			Block:       block,
-			Certificate: cert,
-		}
+		encodedblock := rpcs.EncodedBlockCert{Block: block}
 		blk := protocol.EncodeMsgp(&encodedblock)
 		// write the msgpack bytes for a block
 		_, err := output.Write(blk)
@@ -478,16 +474,16 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error {
 
 // WriteDeltas generates returns the deltas for payset.
 func (g *generator) WriteDeltas(output io.Writer, round uint64) error {
-	// offset round for non-empty database
-	if round-g.nextdbround == 0 {
+	// the first generated round has no statedelta.
+	if round-g.roundOffset == 0 {
 		data, _ := encode(protocol.CodecHandle, ledgercore.StateDelta{})
 		_, err := output.Write(data)
 		if err != nil {
 			return err
 		}
 		return nil
 	}
-	delta, err := g.ledger.GetStateDeltaForRound(basics.Round(round - g.nextdbround))
+	delta, err := g.ledger.GetStateDeltaForRound(basics.Round(round - g.roundOffset))
 	if err != nil {
 		return fmt.Errorf("err getting state delta for round %d: %w", round, err)
 	}

tools/block-generator/go.mod

@@ -0,0 +1,50 @@
+module github.com/algorand/go-algorand/tools/block-generator
+
+replace github.com/algorand/go-algorand => ../..
+
+go 1.17
+
+require (
+	github.com/algorand/go-algorand v0.0.0-00010101000000-000000000000
+	github.com/algorand/go-codec/codec v1.1.9
+	github.com/algorand/go-deadlock v0.2.2
+	github.com/lib/pq v1.10.9
+	github.com/spf13/cobra v1.7.0
+	github.com/stretchr/testify v1.8.3
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	github.com/DataDog/zstd v1.5.2 // indirect
+	github.com/algorand/avm-abi v0.2.0 // indirect
+	github.com/algorand/falcon v0.0.0-20220727072124-02a2a64c4414 // indirect
+	github.com/algorand/go-sumhash v0.1.0 // indirect
+	github.com/algorand/msgp v1.1.53 // indirect
+	github.com/algorand/oapi-codegen v1.12.0-algorand.0 // indirect
+	github.com/algorand/websocket v1.4.6 // indirect
+	github.com/aws/aws-sdk-go v1.33.0 // indirect
+	github.com/consensys/gnark-crypto v0.7.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 // indirect
+	github.com/dchest/siphash v1.2.1 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmespath/go-jmespath v0.3.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-sqlite3 v1.10.0 // indirect
+	github.com/miekg/dns v1.1.41 // indirect
+	github.com/mmcloughlin/addchain v0.4.0 // indirect
+	github.com/olivere/elastic v6.2.14+incompatible // indirect
+	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/crypto v0.1.0 // indirect
+	golang.org/x/net v0.9.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+	gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 // indirect
+)