Conduit: Merge conduit develop

.version

@@ -1 +1 @@
-2.12.4
+2.13.0

README.md

@@ -1,5 +1,15 @@
-| master <br> [![CircleCI](https://circleci.com/gh/algorand/indexer/tree/master.svg?style=svg)](https://circleci.com/gh/algorand/indexer/tree/master) | develop <br> [![CircleCI](https://circleci.com/gh/algorand/indexer/tree/develop.svg?style=svg)](https://circleci.com/gh/algorand/indexer/tree/develop) |
-| --- | --- |
+<div style="text-align:center" align="center">
+  <picture>
+    <img src="./docs/assets/algorand_logo_mark_black.svg" alt="Algorand" width="400">
+    <source media="(prefers-color-scheme: dark)" srcset="./docs/assets/algorand_logo_mark_white.svg">
+    <source media="(prefers-color-scheme: light)" srcset="./docs/assets/algorand_logo_mark_black.svg">
+  </picture>
+
+[![CircleCI](https://img.shields.io/circleci/build/github/algorand/indexer/develop?label=develop)](https://circleci.com/gh/algorand/indexer/tree/develop)
+[![CircleCI](https://img.shields.io/circleci/build/github/algorand/indexer/master?label=master)](https://circleci.com/gh/algorand/indexer/tree/master)
+![Github](https://img.shields.io/github/license/algorand/indexer)
+[![Contribute](https://img.shields.io/badge/contributor-guide-blue?logo=github)](https://github.com/algorand/go-algorand/blob/master/CONTRIBUTING.md)
+</div>
 
 # Algorand Indexer
 
@@ -25,6 +35,7 @@ For a simple deployment the following configuration works well:
 * Network: Indexer, Algod and PostgreSQL should all be on the same network.
 * Indexer: 2 CPU and 8 GB of ram.
 * Database: When hosted on AWS a `db.r5.xlarge` instance works well.
+* Storage: 20 GiB
 
 A database with replication can be used to scale read volume. Configure multiple Indexer daemons with a single writer and multiple readers.
 
@@ -199,6 +210,7 @@ Settings can be provided from the command line, a configuration file, or an envi
 | max-applications-limit        |         | max-applications-limit        | INDEXER_MAX_APPLICATIONS_LIMIT        |
 | default-applications-limit    |         | default-applications-limit    | INDEXER_DEFAULT_APPLICATIONS_LIMIT    |
 | enable-all-parameters         |         | enable-all-parameters         | INDEXER_ENABLE_ALL_PARAMETERS         |
+| catchpoint                    |         | catchpoint                    | INDEXER_CATCHPOINT                    |
 
 ## Command line
 
@@ -212,17 +224,19 @@ The command line arguments always take priority over the config file and environ
 
 The Indexer data directory is the location where the Indexer can store and/or load data needed for runtime operation and configuration.
 
-**It is a required argument for Indexer daemon operation. Supply it to the Indexer via the `--data-dir` flag.**
+**It is a required argument for Indexer daemon operation. Supply it to the Indexer via the `--data-dir`/`-i` flag.**
 
 **It is HIGHLY recommended placing the data directory in a separate, stateful directory for production usage of the Indexer.**
 
+For more information on the data directory see [Indexer Data Directory](docs/DataDirectory.md).
+
 ### Auto-Loading Configuration
 
 The Indexer will scan the data directory at startup and load certain configuration files if they are present.  The files are as follows:
 
 - `indexer.yml` - Indexer Configuration File
 - `api_config.yml` - API Parameter Enable/Disable Configuration File
-- 
+
 **NOTE:** It is not allowed to supply both the command line flag AND have an auto-loading configuration file in the data directory.  Doing so will result in an error.
 
 To see an example of how to use the data directory to load a configuration file check out the [Disabling Parameters Guide](docs/DisablingParametersGuide.md).

accounting/eval_preload.go

@@ -1,6 +1,8 @@
 package accounting
 
 import (
+	"math"
+
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/transactions"
 	"github.com/algorand/go-algorand/ledger"
@@ -34,9 +36,15 @@ func addToCreatorsRequest(stxnad *transactions.SignedTxnWithAD, assetsReq map[ba
 			appsReq[fields.ApplicationID] = struct{}{}
 		}
 		for _, index := range fields.ForeignApps {
+			if index > basics.AppIndex(math.MaxInt64) {
+				continue
+			}
 			appsReq[index] = struct{}{}
 		}
 		for _, index := range fields.ForeignAssets {
+			if index > basics.AssetIndex(math.MaxInt64) {
+				continue
+			}
 			assetsReq[index] = struct{}{}
 		}
 	}
@@ -161,6 +169,9 @@ func addToAccountsResourcesRequest(stxnad *transactions.SignedTxnWithAD, assetCr
 			addressesReq[address] = struct{}{}
 		}
 		for _, index := range fields.ForeignApps {
+			if index > basics.AppIndex(math.MaxInt64) {
+				continue
+			}
 			if creator := appCreators[index]; creator.Exists {
 				creatable := ledger.Creatable{
 					Index: basics.CreatableIndex(index),
@@ -170,6 +181,9 @@ func addToAccountsResourcesRequest(stxnad *transactions.SignedTxnWithAD, assetCr
 			}
 		}
 		for _, index := range fields.ForeignAssets {
+			if index > basics.AssetIndex(math.MaxInt64) {
+				continue
+			}
 			if creator := assetCreators[index]; creator.Exists {
 				creatable := ledger.Creatable{
 					Index: basics.CreatableIndex(index),

api/server.go

@@ -148,7 +148,9 @@ func Serve(ctx context.Context, serveAddr string, db idb.IndexerDb, fetcherError
 	}
 
 	go func() {
-		log.Fatal(e.StartServer(s))
+		if err := e.StartServer(s); err != nil {
+			log.Fatalf("Serve() err: %s", err)
+		}
 	}()
 
 	<-ctx.Done()

cmd/algorand-indexer/main.go

@@ -12,7 +12,6 @@ import (
 	"github.com/spf13/viper"
 
 	bg "github.com/algorand/indexer/cmd/block-generator/core"
-	iv "github.com/algorand/indexer/cmd/import-validator/core"
 	v "github.com/algorand/indexer/cmd/validator/core"
 	"github.com/algorand/indexer/config"
 	"github.com/algorand/indexer/idb"
@@ -101,7 +100,6 @@ func init() {
 		Short: "Utilities for testing Indexer operation and correctness.",
 		Long:  "Utilities used for Indexer development. These are low level tools that may require low level knowledge of Indexer deployment and operation. They are included as part of this binary for ease of deployment and automation, and to publicize their existance to people who may find them useful. More detailed documention may be found on github in README files located the different 'cmd' directories.",
 	}
-	utilsCmd.AddCommand(iv.ImportValidatorCmd)
 	utilsCmd.AddCommand(v.ValidatorCmd)
 	utilsCmd.AddCommand(bg.BlockGenerator)
 	rootCmd.AddCommand(utilsCmd)

cmd/block-generator/generator/generate.go

@@ -101,7 +101,7 @@ func MakeGenerator(config GenerationConfig) (Generator, error) {
 		genesisHash:               [32]byte{},
 		genesisID:                 "blockgen-test",
 		prevBlockHash:             "",
-		round:                     0,
+		round:                     1,
 		txnCounter:                0,
 		timestamp:                 0,
 		rewardsLevel:              0,

cmd/block-generator/run_tests.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+CONNECTION_STRING=""
+INDEXER_BINARY=""
+REPORT_DIR=""
+DURATION="1h"
+LOG_LEVEL="error"
+SCENARIOS=""
+
+help() {
+  echo "Usage:"
+  echo " -v|--verbose    enable verbose script output."
+  echo " -c|--connection-string"
+  echo "                 PostgreSQL connection string."
+  echo " -i|--indexer    path to indexer binary."
+  echo " -s|--scenarios  path to indexer test scenarios."
+  echo " -r|--report-dir directory where the report should be written."
+  echo " -d|--duration   test duration."
+  echo " -l|--level      log level to pass to Indexer."
+  echo " -g|--generator  use a different indexer binary to run the generator."
+  exit
+}
+
+while :; do
+  case "${1-}" in
+  -h | --help) help ;;
+  -v | --verbose) set -x ;;
+  -c | --connection-string)
+    CONNECTION_STRING="${2-}"
+    shift
+    ;;
+  -g | --generator)
+    GENERATOR_BINARY="${2-}"
+    shift
+    ;;
+  -i | --indexer)
+    INDEXER_BINARY="${2-}"
+    shift
+    ;;
+  -r | --report-dir)
+    REPORT_DIR="${2-}"
+    shift
+    ;;
+  -s | --scenarios)
+    SCENARIOS="${2-}"
+    shift
+    ;;
+  -d | --duration)
+    DURATION="${2-}"
+    shift
+    ;;
+  -l | --level)
+    LOG_LEVEL="${2-}"
+    shift
+    ;;
+  -?*) echo "Unknown option: $1" && exit 1;;
+  *) break ;;
+  esac
+  shift
+done
+
+args=("$@")
+
+if [ -z "$CONNECTION_STRING" ]; then
+  echo "Missing required connection string parameter (-c / --connection-string)."
+  exit 1
+fi
+
+if [ -z "$INDEXER_BINARY" ]; then
+  echo "Missing required indexer binary parameter (-i / --indexer)."
+  exit 1
+fi
+
+if [ -z "$SCENARIOS" ]; then
+  echo "Missing required indexer test scenario parameter (-s / --scenarios)."
+  exit 1
+fi
+
+if [ -z "$GENERATOR_BINARY" ]; then
+  echo "Using indexer binary for generator, override with (-g / --generator)."
+  GENERATOR_BINARY="$INDEXER_BINARY"
+fi
+
+echo "Running with binary: $INDEXER_BINARY"
+echo "Report directory: $REPORT_DIR"
+echo "Duration: $DURATION"
+echo "Log Level: $LOG_LEVEL"
+
+"$GENERATOR_BINARY" \
+         util block-generator runner \
+  -i "$INDEXER_BINARY" \
+  -s "$SCENARIOS" \
+  -d "$DURATION" \
+  -c "$CONNECTION_STRING" \
+  --report-directory "$REPORT_DIR" \
+  --log-level "$LOG_LEVEL" \
+  --reset
+

cmd/block-generator/runner/run.go

@@ -13,7 +13,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"syscall"
 	"time"
 
 	"github.com/jackc/pgx/v4"
@@ -37,6 +36,7 @@ type Args struct {
 	ReportDirectory          string
 	ResetReportDir           bool
 	RunValidation            bool
+	KeepDataDir              bool
 }
 
 // Run is a public helper to run the tests.
@@ -73,6 +73,10 @@ func (r *Args) run() error {
 	baseNameNoExt := strings.TrimSuffix(baseName, filepath.Ext(baseName))
 	reportfile := path.Join(r.ReportDirectory, fmt.Sprintf("%s.report", baseNameNoExt))
 	logfile := path.Join(r.ReportDirectory, fmt.Sprintf("%s.indexer-log", baseNameNoExt))
+	dataDir := path.Join(r.ReportDirectory, fmt.Sprintf("%s_data", baseNameNoExt))
+	if !r.KeepDataDir {
+		defer os.RemoveAll(dataDir)
+	}
 
 	// This middleware allows us to lock the block endpoint
 	var freezeMutex sync.Mutex
@@ -94,7 +98,7 @@ func (r *Args) run() error {
 		}
 	}()
 
-	indexerShutdownFunc, err := startIndexer(logfile, r.LogLevel, r.IndexerBinary, algodNet, indexerNet, r.PostgresConnectionString, r.CPUProfilePath)
+	indexerShutdownFunc, err := startIndexer(dataDir, logfile, r.LogLevel, r.IndexerBinary, algodNet, indexerNet, r.PostgresConnectionString, r.CPUProfilePath)
 	if err != nil {
 		return fmt.Errorf("failed to start indexer: %w", err)
 	}
@@ -270,15 +274,17 @@ func (r *Args) runTest(report *os.File, indexerURL string, generatorURL string)
 
 	// Run for r.RunDuration
 	start := time.Now()
+	count := 1
 	for time.Since(start) < r.RunDuration {
 		time.Sleep(r.RunDuration / 10)
 
 		if err := collector.Collect(metrics.AllMetricNames...); err != nil {
-			return fmt.Errorf("problem collecting metrics: %w", err)
+			return fmt.Errorf("problem collecting metrics (%d / %s): %w", count, time.Since(start), err)
 		}
+		count++
 	}
 	if err := collector.Collect(metrics.AllMetricNames...); err != nil {
-		return fmt.Errorf("problem collecting metrics: %w", err)
+		return fmt.Errorf("problem collecting final metrics (%d / %s): %w", count, time.Since(start), err)
 	}
 
 	// Collect results.
@@ -396,7 +402,7 @@ func startGenerator(configFile string, addr string, blockMiddleware func(http.Ha
 
 // startIndexer resets the postgres database and executes the indexer binary. It performs some simple verification to
 // ensure that the service has started properly.
-func startIndexer(logfile string, loglevel string, indexerBinary string, algodNet string, indexerNet string, postgresConnectionString string, cpuprofile string) (func() error, error) {
+func startIndexer(dataDir string, logfile string, loglevel string, indexerBinary string, algodNet string, indexerNet string, postgresConnectionString string, cpuprofile string) (func() error, error) {
 	{
 		conn, err := pgx.Connect(context.Background(), postgresConnectionString)
 		if err != nil {
@@ -421,7 +427,8 @@ func startIndexer(logfile string, loglevel string, indexerBinary string, algodNe
 		"--server", indexerNet,
 		"--logfile", logfile,
 		"--loglevel", loglevel,
-		"--cpuprofile", cpuprofile)
+		"--cpuprofile", cpuprofile,
+		"--data-dir", dataDir)
 
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
@@ -444,10 +451,15 @@ func startIndexer(logfile string, loglevel string, indexerBinary string, algodNe
 	resp.Body.Close()
 
 	return func() error {
-		if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
-			return fmt.Errorf("failed to kill indexer process: %w", err)
+		if err := cmd.Process.Signal(os.Interrupt); err != nil {
+			fmt.Printf("failed to kill indexer process: %s\n", err)
+			if err := cmd.Process.Kill(); err != nil {
+				return fmt.Errorf("failed to kill indexer process: %w", err)
+			}
+		}
+		if err := cmd.Wait(); err != nil {
+			fmt.Printf("ignoring error while waiting for process to stop: %s\n", err)
 		}
-
 		return nil
 	}, nil
 }

cmd/block-generator/runner/runner.go

@@ -36,6 +36,7 @@ func init() {
 	RunnerCmd.Flags().StringVarP(&runnerArgs.CPUProfilePath, "cpuprofile", "", "", "Path where Indexer writes its CPU profile.")
 	RunnerCmd.Flags().BoolVarP(&runnerArgs.ResetReportDir, "reset", "", false, "If set any existing report directory will be deleted before running tests.")
 	RunnerCmd.Flags().BoolVarP(&runnerArgs.RunValidation, "validate", "", false, "If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure.")
+	RunnerCmd.Flags().BoolVarP(&runnerArgs.KeepDataDir, "keep-data-dir", "k", false, "If set the validator will not delete the data directory after tests complete.")
 
 	RunnerCmd.MarkFlagRequired("scenario")
 	RunnerCmd.MarkFlagRequired("indexer-binary")