tools: update block-generator to use conduit binary

tools/block-generator/generator/generate.go

@@ -72,7 +72,7 @@ type GenerationConfig struct {
 	GenesisAccountInitialBalance uint64 `yaml:"genesis_account_balance"`
 
 	// Block generation
-	TxnPerBlock uint64 `mapstructure:"tx_per_block"`
+	TxnPerBlock uint64 `yaml:"tx_per_block"`
 
 	// TX Distribution
 	PaymentTransactionFraction float32 `yaml:"tx_pay_fraction"`
@@ -120,7 +120,7 @@ func MakeGenerator(config GenerationConfig) (Generator, error) {
 		genesisHash:               [32]byte{},
 		genesisID:                 "blockgen-test",
 		prevBlockHash:             "",
-		round:                     1,
+		round:                     0,
 		txnCounter:                0,
 		timestamp:                 0,
 		rewardsLevel:              0,
@@ -312,7 +312,9 @@ func (g *generator) WriteGenesis(output io.Writer) error {
 		FeeSink:     g.feeSink.String(),
 		Timestamp:   g.timestamp,
 	}
-	return json.NewEncoder(output).Encode(gen)
+
+	_, err := output.Write(protocol.EncodeJSON(gen))
+	return err
 }
 
 func getTransactionOptions() []interface{} {
@@ -357,12 +359,28 @@ func (g *generator) finishRound(txnCount uint64) {
 
 // WriteBlock generates a block full of new transactions and writes it to the writer.
 func (g *generator) WriteBlock(output io.Writer, round uint64) error {
+
 	if round != g.round {
 		fmt.Printf("Generator only supports sequential block access. Expected %d but received request for %d.\n", g.round, round)
 	}
 
 	numTxnForBlock := g.txnForRound(round)
 
+	// return genesis block
+	if round == 0 {
+		// write the msgpack bytes for a block
+		block, err := rpcs.RawBlockBytes(g.ledger, basics.Round(round))
+		if err != nil {
+			return err
+		}
+		_, err = output.Write(block)
+		if err != nil {
+			return err
+		}
+		g.finishRound(numTxnForBlock)
+		return nil
+	}
+
 	header := bookkeeping.BlockHeader{
 		Round:          basics.Round(g.round),
 		Branch:         bookkeeping.BlockHash{},
@@ -414,15 +432,19 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error {
 		Certificate: agreement.Certificate{},
 	}
 
-	err := json.NewEncoder(output).Encode(cert)
+	err := g.ledger.AddBlock(cert.Block, cert.Certificate)
+	if err != nil {
+		return err
+	}
+	// write the msgpack bytes for a block
+	block, err := rpcs.RawBlockBytes(g.ledger, basics.Round(round))
 	if err != nil {
 		return err
 	}
-	err = g.ledger.AddBlock(cert.Block, agreement.Certificate{})
+	_, err = output.Write(block)
 	if err != nil {
 		return err
 	}
-	g.ledger.WaitForCommit(basics.Round(g.round))
 	g.finishRound(numTxnForBlock)
 	return nil
 }

tools/block-generator/generator/generate_test.go

@@ -203,6 +203,7 @@ func TestWriteRound(t *testing.T) {
 	g := makePrivateGenerator(t)
 	var data []byte
 	writer := bytes.NewBuffer(data)
+	g.WriteBlock(writer, 0)
 	g.WriteBlock(writer, 1)
 	var block rpcs.EncodedBlockCert
 	protocol.Decode(data, &block)

tools/block-generator/generator/server_test.go

@@ -53,61 +53,61 @@ func TestParseURL(t *testing.T) {
 	var testcases = []struct {
 		name          string
 		url           string
-		expectedRound string
+		expectedParam string
 		err           string
 	}{
 		{
 			name:          "no block",
 			url:           "/v2/blocks/",
-			expectedRound: "",
+			expectedParam: "",
 			err:           "invalid request path, /v2/blocks/",
 		},
 		{
 			name:          "normal one digit",
 			url:           fmt.Sprintf("%s1", blockQueryPrefix),
-			expectedRound: "1",
+			expectedParam: "1",
 			err:           "",
 		},
 		{
 			name:          "normal long number",
 			url:           fmt.Sprintf("%s12345678", blockQueryPrefix),
-			expectedRound: "12345678",
+			expectedParam: "12345678",
 			err:           "",
 		},
 		{
 			name:          "with query parameters",
 			url:           fmt.Sprintf("%s1234?pretty", blockQueryPrefix),
-			expectedRound: "1234",
+			expectedParam: "1234",
 			err:           "",
 		},
 		{
 			name:          "with query parameters",
 			url:           fmt.Sprintf("%s1234?pretty", blockQueryPrefix),
-			expectedRound: "1234",
+			expectedParam: "1234",
 			err:           "",
 		},
 		{
 			name:          "no deltas",
 			url:           "/v2/deltas/",
-			expectedRound: "",
+			expectedParam: "",
 			err:           "invalid request path, /v2/deltas/",
 		},
 		{
 			name:          "deltas",
 			url:           fmt.Sprintf("%s123?Format=msgp", deltaQueryPrefix),
-			expectedRound: "123",
+			expectedParam: "123",
 			err:           "",
 		},
 		{
 			name:          "no account",
 			url:           "/v2/accounts/",
-			expectedRound: "",
+			expectedParam: "",
 			err:           "invalid request path, /v2/accounts/",
 		},
 		{
 			name:          "accounts",
 			url:           fmt.Sprintf("%sAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4", accountQueryPrefix),
-			expectedRound: "AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4",
+			expectedParam: "AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4",
 			err:           "",
 		},
 	}
@@ -117,9 +117,9 @@ func TestParseURL(t *testing.T) {
 			round, err := parseURL(testcase.url)
 			if len(testcase.err) == 0 {
 				msg := fmt.Sprintf("Unexpected error parsing '%s', expected round '%s' received error: %v",
-					testcase.url, testcase.expectedRound, err)
+					testcase.url, testcase.expectedParam, err)
 				require.NoError(t, err, msg)
-				assert.Equal(t, testcase.expectedRound, round)
+				assert.Equal(t, testcase.expectedParam, round)
 			} else {
 				require.Error(t, err, fmt.Sprintf("Expected an error containing: %s", testcase.err))
 				require.True(t, strings.Contains(err.Error(), testcase.err))

tools/block-generator/run_postgres.sh

@@ -4,7 +4,7 @@
 # in a debugger. Simply start this script and run with:
 # ./block-generator runner \
 #       -d 5s \
-#       -i ./../algorand-indexer/algorand-indexer \
+#       -i <path to conduit binary> \
 #       -c "host=localhost user=algorand password=algorand dbname=algorand port=15432 sslmode=disable" \
 #       -r results \
 #       -s scenarios/config.payment.small.yml

tools/block-generator/run_runner.sh

@@ -4,10 +4,16 @@
 
 set -e
 
+CONDUIT_BINARY=$1
+if [ -z "$CONDUIT_BINARY" ]; then
+  echo "path to conduit binary is required"
+  exit 1
+fi
+
 POSTGRES_CONTAINER=generator-test-container
 POSTGRES_PORT=15432
 POSTGRES_DATABASE=generator_db
-CONFIG=${1:-"$(dirname $0)/test_config.yml"}
+CONFIG=${2:-"$(dirname $0)/test_config.yml"}
 echo "Using config file: $CONFIG"
 
 function start_postgres() {
@@ -38,15 +44,12 @@ rm -rf OUTPUT_RUN_RUNNER_TEST > /dev/null 2>&1
 echo "Building generator."
 pushd $(dirname "$0") > /dev/null
 go build
-cd ../.. > /dev/null
-echo "Building indexer."
-make
 popd
 echo "Starting postgres container."
 start_postgres
 echo "Starting test runner"
 $(dirname "$0")/block-generator runner \
-	--indexer-binary ../algorand-indexer/algorand-indexer \
+	--conduit-binary "$CONDUIT_BINARY" \
 	--report-directory OUTPUT_RUN_RUNNER_TEST \
 	--test-duration 30s \
 	--log-level trace \

tools/block-generator/run_tests.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 CONNECTION_STRING=""
-INDEXER_BINARY=""
+CONDUIT_BINARY=""
 REPORT_DIR=""
 DURATION="1h"
 LOG_LEVEL="error"
@@ -17,7 +17,7 @@ help() {
   echo " -r|--report-dir directory where the report should be written."
   echo " -d|--duration   test duration."
   echo " -l|--level      log level to pass to Indexer."
-  echo " -g|--generator  use a different indexer binary to run the generator."
+  echo " -g|--generator  block-generator binary to run the generator."
   exit
 }
 
@@ -34,7 +34,7 @@ while :; do
     shift
     ;;
   -i | --indexer)
-    INDEXER_BINARY="${2-}"
+    CONDUIT_BINARY="${2-}"
     shift
     ;;
   -r | --report-dir)
@@ -66,7 +66,7 @@ if [ -z "$CONNECTION_STRING" ]; then
   exit 1
 fi
 
-if [ -z "$INDEXER_BINARY" ]; then
+if [ -z "$CONDUIT_BINARY" ]; then
   echo "Missing required indexer binary parameter (-i / --indexer)."
   exit 1
 fi
@@ -77,18 +77,17 @@ if [ -z "$SCENARIOS" ]; then
 fi
 
 if [ -z "$GENERATOR_BINARY" ]; then
-  echo "Using indexer binary for generator, override with (-g / --generator)."
-  GENERATOR_BINARY="$INDEXER_BINARY"
+  echo "path to block-generator binary is required"
+  exit 1
 fi
 
-echo "Running with binary: $INDEXER_BINARY"
+echo "Running with binary: $CONDUIT_BINARY"
 echo "Report directory: $REPORT_DIR"
 echo "Duration: $DURATION"
 echo "Log Level: $LOG_LEVEL"
 
-"$GENERATOR_BINARY" \
-         util block-generator runner \
-  -i "$INDEXER_BINARY" \
+"$GENERATOR_BINARY" runner \
+  -i "$CONDUIT_BINARY" \
   -s "$SCENARIOS" \
   -d "$DURATION" \
   -c "$CONNECTION_STRING" \