From 4fcbd3a7c4c06091cbc0c1e1127509896d2778e8 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 14 Apr 2023 15:40:30 -0400 Subject: [PATCH 01/14] use conduit binary --- tools/block-generator/data/conduit.yml | 61 +++++++++ tools/block-generator/generator/generate.go | 28 +++- .../block-generator/generator/server_test.go | 24 ++-- tools/block-generator/metrics/metrics.go | 42 ------ tools/block-generator/run_tests.sh | 2 +- .../runner/metrics_collector.go | 25 ++++ tools/block-generator/runner/run.go | 128 ++++++++++++++---- tools/block-generator/runner/runner.go | 8 +- 8 files changed, 225 insertions(+), 93 deletions(-) create mode 100644 tools/block-generator/data/conduit.yml delete mode 100644 tools/block-generator/metrics/metrics.go diff --git a/tools/block-generator/data/conduit.yml b/tools/block-generator/data/conduit.yml new file mode 100644 index 0000000000..c361426ee6 --- /dev/null +++ b/tools/block-generator/data/conduit.yml @@ -0,0 +1,61 @@ +# Log verbosity: PANIC, FATAL, ERROR, WARN, INFO, DEBUG, TRACE +log-level: {{.LogLevel}} + +# If no log file is provided logs are written to stdout. +log-file: {{.LogFile}} + +# Number of retries to perform after a pipeline plugin error. +retry-count: 10 + +# Time duration to wait between retry attempts. +retry-delay: "1s" + +# Optional filepath to use for pidfile. +#pid-filepath: /path/to/pidfile + +# Whether or not to print the conduit banner on startup. +hide-banner: false + +# When enabled prometheus metrics are available on '/metrics' +metrics: + mode: ON + addr: "{{.MetricsPort}}" + prefix: "conduit" + +# The importer is typically an algod follower node. +importer: + name: algod + config: + # The mode of operation, either "archival" or "follower". + # * archival mode allows you to start processing on any round but does not + # contain the ledger state delta objects required for the postgres writer. + # * follower mode allows you to use a lightweight non-archival node as the + # data source. In addition, it will provide ledger state delta objects to + # the processors and exporter. + mode: "follower" + + # Algod API address. + netaddr: "{{.AlgodNet}}" + + # Algod API token. + token: "" + + +# Zero or more processors may be defined to manipulate what data +# reaches the exporter. +processors: + +# An exporter is defined to do something with the data. +exporter: + name: postgresql + config: + # Pgsql connection string + # See https://github.com/jackc/pgconn for more details + connection-string: "{{ .PostgresConnectionString }}" + + # Maximum connection number for connection pool + # This means the total number of active queries that can be running + # concurrently can never be more than this + max-conn: 20 + + diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index f1961587bb..6080c35e93 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -72,7 +72,7 @@ type GenerationConfig struct { GenesisAccountInitialBalance uint64 `yaml:"genesis_account_balance"` // Block generation - TxnPerBlock uint64 `mapstructure:"tx_per_block"` + TxnPerBlock uint64 `yaml:"tx_per_block"` // TX Distribution PaymentTransactionFraction float32 `yaml:"tx_pay_fraction"` @@ -120,7 +120,7 @@ func MakeGenerator(config GenerationConfig) (Generator, error) { genesisHash: [32]byte{}, genesisID: "blockgen-test", prevBlockHash: "", - round: 1, + round: 0, txnCounter: 0, timestamp: 0, rewardsLevel: 0, @@ -312,7 +312,9 @@ func (g *generator) WriteGenesis(output io.Writer) error { FeeSink: g.feeSink.String(), Timestamp: g.timestamp, } - return json.NewEncoder(output).Encode(gen) + + _, err := output.Write(protocol.EncodeJSON(gen)) + return err } func getTransactionOptions() []interface{} { @@ -357,12 +359,25 @@ func (g *generator) finishRound(txnCount uint64) { // WriteBlock generates a block full of new transactions and writes it to the writer. func (g *generator) WriteBlock(output io.Writer, round uint64) error { + if round != g.round { fmt.Printf("Generator only supports sequential block access. Expected %d but received request for %d.\n", g.round, round) } numTxnForBlock := g.txnForRound(round) + // return genesis block + if round == 0 { + // write the msgpack bytes for a block + block, err := rpcs.RawBlockBytes(g.ledger, basics.Round(round)) + if err != nil { + return err + } + output.Write(block) + g.finishRound(numTxnForBlock) + return nil + } + header := bookkeeping.BlockHeader{ Round: basics.Round(g.round), Branch: bookkeeping.BlockHash{}, @@ -414,15 +429,16 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { Certificate: agreement.Certificate{}, } - err := json.NewEncoder(output).Encode(cert) + err := g.ledger.AddBlock(cert.Block, cert.Certificate) if err != nil { return err } - err = g.ledger.AddBlock(cert.Block, agreement.Certificate{}) + // write the msgpack bytes for a block + block, err := rpcs.RawBlockBytes(g.ledger, basics.Round(round)) if err != nil { return err } - g.ledger.WaitForCommit(basics.Round(g.round)) + output.Write(block) g.finishRound(numTxnForBlock) return nil } diff --git a/tools/block-generator/generator/server_test.go b/tools/block-generator/generator/server_test.go index 6e21d77586..7007db00f0 100644 --- a/tools/block-generator/generator/server_test.go +++ b/tools/block-generator/generator/server_test.go @@ -53,61 +53,61 @@ func TestParseURL(t *testing.T) { var testcases = []struct { name string url string - expectedRound string + expectedParam string err string }{ { name: "no block", url: "/v2/blocks/", - expectedRound: "", + expectedParam: "", err: "invalid request path, /v2/blocks/", }, { name: "normal one digit", url: fmt.Sprintf("%s1", blockQueryPrefix), - expectedRound: "1", + expectedParam: "1", err: "", }, { name: "normal long number", url: fmt.Sprintf("%s12345678", blockQueryPrefix), - expectedRound: "12345678", + expectedParam: "12345678", err: "", }, { name: "with query parameters", url: fmt.Sprintf("%s1234?pretty", blockQueryPrefix), - expectedRound: "1234", + expectedParam: "1234", err: "", }, { name: "with query parameters", url: fmt.Sprintf("%s1234?pretty", blockQueryPrefix), - expectedRound: "1234", + expectedParam: "1234", err: "", }, { name: "no deltas", url: "/v2/deltas/", - expectedRound: "", + expectedParam: "", err: "invalid request path, /v2/deltas/", }, { name: "deltas", url: fmt.Sprintf("%s123?Format=msgp", deltaQueryPrefix), - expectedRound: "123", + expectedParam: "123", err: "", }, { name: "no account", url: "/v2/accounts/", - expectedRound: "", + expectedParam: "", err: "invalid request path, /v2/accounts/", }, { name: "accounts", url: fmt.Sprintf("%sAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4", accountQueryPrefix), - expectedRound: "AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4", + expectedParam: "AIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFFWAF4", err: "", }, } @@ -117,9 +117,9 @@ func TestParseURL(t *testing.T) { round, err := parseURL(testcase.url) if len(testcase.err) == 0 { msg := fmt.Sprintf("Unexpected error parsing '%s', expected round '%s' received error: %v", - testcase.url, testcase.expectedRound, err) + testcase.url, testcase.expectedParam, err) require.NoError(t, err, msg) - assert.Equal(t, testcase.expectedRound, round) + assert.Equal(t, testcase.expectedParam, round) } else { require.Error(t, err, fmt.Sprintf("Expected an error containing: %s", testcase.err)) require.True(t, strings.Contains(err.Error(), testcase.err)) diff --git a/tools/block-generator/metrics/metrics.go b/tools/block-generator/metrics/metrics.go deleted file mode 100644 index 6c60fbabc2..0000000000 --- a/tools/block-generator/metrics/metrics.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (C) 2019-2023 Algorand, Inc. -// This file is part of go-algorand -// -// go-algorand is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// go-algorand is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with go-algorand. If not, see . - -package metrics - -// Prometheus metrics collected in Conduit. -const ( - BlockImportTimeName = "import_time_sec" - ImportedTxnsPerBlockName = "imported_tx_per_block" - ImportedRoundGaugeName = "imported_round" - GetAlgodRawBlockTimeName = "get_algod_raw_block_time_sec" - ImportedTxnsName = "imported_txns" - ImporterTimeName = "importer_time_sec" - ProcessorTimeName = "processor_time_sec" - ExporterTimeName = "exporter_time_sec" - PipelineRetryCountName = "pipeline_retry_count" -) - -// AllMetricNames is a reference for all the custom metric names. -var AllMetricNames = []string{ - BlockImportTimeName, - ImportedTxnsPerBlockName, - ImportedRoundGaugeName, - GetAlgodRawBlockTimeName, - ImporterTimeName, - ProcessorTimeName, - ExporterTimeName, - PipelineRetryCountName, -} diff --git a/tools/block-generator/run_tests.sh b/tools/block-generator/run_tests.sh index 0a1c6df49e..8befa41dc4 100755 --- a/tools/block-generator/run_tests.sh +++ b/tools/block-generator/run_tests.sh @@ -17,7 +17,7 @@ help() { echo " -r|--report-dir directory where the report should be written." echo " -d|--duration test duration." echo " -l|--level log level to pass to Indexer." - echo " -g|--generator use a different indexer binary to run the generator." + echo " -g|--generator use a different block-generator binary to run the generator." exit } diff --git a/tools/block-generator/runner/metrics_collector.go b/tools/block-generator/runner/metrics_collector.go index 327ca9e33b..394bf32cba 100644 --- a/tools/block-generator/runner/metrics_collector.go +++ b/tools/block-generator/runner/metrics_collector.go @@ -24,6 +24,31 @@ import ( "time" ) +// Prometheus metrics collected in Conduit. +const ( + BlockImportTimeName = "import_time_sec" + ImportedTxnsPerBlockName = "imported_tx_per_block" + ImportedRoundGaugeName = "imported_round" + GetAlgodRawBlockTimeName = "get_algod_raw_block_time_sec" + ImportedTxnsName = "imported_txns" + ImporterTimeName = "importer_time_sec" + ProcessorTimeName = "processor_time_sec" + ExporterTimeName = "exporter_time_sec" + PipelineRetryCountName = "pipeline_retry_count" +) + +// AllMetricNames is a reference for all the custom metric names. +var AllMetricNames = []string{ + BlockImportTimeName, + ImportedTxnsPerBlockName, + ImportedRoundGaugeName, + GetAlgodRawBlockTimeName, + ImporterTimeName, + ProcessorTimeName, + ExporterTimeName, + PipelineRetryCountName, +} + // MetricsCollector queries a /metrics endpoint for prometheus style metrics and saves metrics matching a pattern. type MetricsCollector struct { // MetricsURL where metrics can be queried. diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 9604212da1..0b1d49c518 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -17,19 +17,21 @@ package runner import ( + "bytes" "context" "encoding/json" "fmt" "net/http" "os" + "os/exec" "path" "path/filepath" "strconv" "strings" + "text/template" "time" "github.com/algorand/go-algorand/tools/block-generator/generator" - "github.com/algorand/go-algorand/tools/block-generator/metrics" "github.com/algorand/go-algorand/tools/block-generator/util" "github.com/algorand/go-deadlock" ) @@ -38,8 +40,8 @@ import ( type Args struct { // Path is a directory when passed to RunBatch, otherwise a file path. Path string - IndexerBinary string - IndexerPort uint64 + ConduitBinary string + MetricsPort uint64 PostgresConnectionString string CPUProfilePath string RunDuration time.Duration @@ -50,7 +52,15 @@ type Args struct { KeepDataDir bool } -// Run is a publi8c helper to run the tests. +type config struct { + LogLevel string + LogFile string + MetricsPort string + AlgodNet string + PostgresConnectionString string +} + +// Run is a public helper to run the tests. // The test will run against the generator configuration file specified by 'args.Path'. // If 'args.Path' is a directory it should contain generator configuration files, a test will run using each file. func Run(args Args) error { @@ -86,8 +96,12 @@ func (r *Args) run() error { baseName := filepath.Base(r.Path) baseNameNoExt := strings.TrimSuffix(baseName, filepath.Ext(baseName)) reportfile := path.Join(r.ReportDirectory, fmt.Sprintf("%s.report", baseNameNoExt)) - //logfile := path.Join(r.ReportDirectory, fmt.Sprintf("%s.indexer-log", baseNameNoExt)) + logfile := path.Join(r.ReportDirectory, fmt.Sprintf("%s.indexer-log", baseNameNoExt)) dataDir := path.Join(r.ReportDirectory, fmt.Sprintf("%s_data", baseNameNoExt)) + // create the data directory. + if err := os.Mkdir(dataDir, os.ModeDir|os.ModePerm); err != nil { + return fmt.Errorf("failed to create data directory: %w", err) + } if !r.KeepDataDir { defer os.RemoveAll(dataDir) } @@ -103,7 +117,7 @@ func (r *Args) run() error { } // Start services algodNet := fmt.Sprintf("localhost:%d", 11112) - indexerNet := fmt.Sprintf("localhost:%d", r.IndexerPort) + metricsNet := fmt.Sprintf("localhost:%d", r.MetricsPort) generatorShutdownFunc, _ := startGenerator(r.Path, algodNet, blockMiddleware) defer func() { // Shutdown generator. @@ -111,17 +125,43 @@ func (r *Args) run() error { fmt.Printf("Failed to shutdown generator: %s\n", err) } }() + time.Sleep(1 * time.Second) + // write conduit config file + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("current working directory: %w", err) + } + t, err := template.ParseFiles(path.Join(cwd, "data/conduit.yml")) + if err != nil { + return fmt.Errorf("unable to open config template file: %w", err) + } - //indexerShutdownFunc, err := startIndexer(dataDir, logfile, r.LogLevel, r.IndexerBinary, algodNet, indexerNet, r.PostgresConnectionString, r.CPUProfilePath) - //if err != nil { - // return fmt.Errorf("failed to start indexer: %w", err) - //} - //defer func() { - // // Shutdown indexer - // if err := indexerShutdownFunc(); err != nil { - // fmt.Printf("Failed to shutdown indexer: %s\n", err) - // } - //}() + f, err := os.Create(path.Join(dataDir, "conduit.yml")) + if err != nil { + return fmt.Errorf("creating conduit.yml: ", err) + } + defer f.Close() + + conduitConfig := config{r.LogLevel, logfile, + fmt.Sprintf(":%d", r.MetricsPort), + algodNet, r.PostgresConnectionString, + } + + err = t.Execute(f, conduitConfig) + if err != nil { + return fmt.Errorf("execute template file: ", err) + } + + indexerShutdownFunc, err := startIndexer(dataDir, r.ConduitBinary) + if err != nil { + return fmt.Errorf("failed to start indexer: %w", err) + } + defer func() { + // Shutdown indexer + if err := indexerShutdownFunc(); err != nil { + fmt.Printf("Failed to shutdown indexer: %s\n", err) + } + }() // Create the report file report, err := os.Create(reportfile) @@ -131,7 +171,7 @@ func (r *Args) run() error { defer report.Close() // Run the test, collecting results. - if err := r.runTest(report, indexerNet, algodNet); err != nil { + if err := r.runTest(report, metricsNet, algodNet); err != nil { return err } @@ -158,23 +198,23 @@ func recordDataToFile(start time.Time, entry Entry, prefix string, out *os.File) } } - record("_average", metrics.BlockImportTimeName, rate) - record("_cumulative", metrics.BlockImportTimeName, floatTotal) - record("_average", metrics.ImportedTxnsPerBlockName, rate) - record("_cumulative", metrics.ImportedTxnsPerBlockName, intTotal) - record("", metrics.ImportedRoundGaugeName, intTotal) + record("_average", BlockImportTimeName, rate) + record("_cumulative", BlockImportTimeName, floatTotal) + record("_average", ImportedTxnsPerBlockName, rate) + record("_cumulative", ImportedTxnsPerBlockName, intTotal) + record("", ImportedRoundGaugeName, intTotal) if len(writeErrors) > 0 { return fmt.Errorf("error writing metrics (%s): %w", strings.Join(writeErrors, ", "), writeErr) } // Calculate import transactions per second. - totalTxn, err := getMetric(entry, metrics.ImportedTxnsPerBlockName, false) + totalTxn, err := getMetric(entry, ImportedTxnsPerBlockName, false) if err != nil { return err } - importTimeS, err := getMetric(entry, metrics.BlockImportTimeName, false) + importTimeS, err := getMetric(entry, BlockImportTimeName, false) if err != nil { return err } @@ -266,8 +306,8 @@ func getMetric(entry Entry, suffix string, rateMetric bool) (float64, error) { } // Run the test for 'RunDuration', collect metrics and write them to the 'ReportDirectory' -func (r *Args) runTest(report *os.File, indexerURL string, generatorURL string) error { - collector := &MetricsCollector{MetricsURL: fmt.Sprintf("http://%s/metrics", indexerURL)} +func (r *Args) runTest(report *os.File, metricsURL string, generatorURL string) error { + collector := &MetricsCollector{MetricsURL: fmt.Sprintf("http://%s/metrics", metricsURL)} // Run for r.RunDuration start := time.Now() @@ -275,12 +315,12 @@ func (r *Args) runTest(report *os.File, indexerURL string, generatorURL string) for time.Since(start) < r.RunDuration { time.Sleep(r.RunDuration / 10) - if err := collector.Collect(metrics.AllMetricNames...); err != nil { + if err := collector.Collect(AllMetricNames...); err != nil { return fmt.Errorf("problem collecting metrics (%d / %s): %w", count, time.Since(start), err) } count++ } - if err := collector.Collect(metrics.AllMetricNames...); err != nil { + if err := collector.Collect(AllMetricNames...); err != nil { return fmt.Errorf("problem collecting final metrics (%d / %s): %w", count, time.Since(start), err) } @@ -335,6 +375,7 @@ func startGenerator(configFile string, addr string, blockMiddleware func(http.Ha // Start the server go func() { // always returns error. ErrServerClosed on graceful close + fmt.Printf("generator serving on %s\n", server.Addr) if err := server.ListenAndServe(); err != http.ErrServerClosed { util.MaybeFail(err, "ListenAndServe() failure to start with config file '%s'", configFile) } @@ -350,3 +391,34 @@ func startGenerator(configFile string, addr string, blockMiddleware func(http.Ha return nil }, generator } + +// startIndexer starts the indexer. +func startIndexer(dataDir string, conduitBinary string) (func() error, error) { + cmd := exec.Command( + conduitBinary, + "-d", dataDir, + ) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failure calling Start(): %w", err) + } + // conduit doesn't have health check endpoint. so, no health check for now + + return func() error { + if err := cmd.Process.Signal(os.Interrupt); err != nil { + fmt.Printf("failed to kill indexer process: %s\n", err) + if err := cmd.Process.Kill(); err != nil { + return fmt.Errorf("failed to kill indexer process: %w", err) + } + } + if err := cmd.Wait(); err != nil { + fmt.Printf("ignoring error while waiting for process to stop: %s\n", err) + } + return nil + }, nil +} diff --git a/tools/block-generator/runner/runner.go b/tools/block-generator/runner/runner.go index 94df877660..2eab45fea5 100644 --- a/tools/block-generator/runner/runner.go +++ b/tools/block-generator/runner/runner.go @@ -34,7 +34,7 @@ func init() { RunnerCmd = &cobra.Command{ Use: "runner", Short: "Run test suite and collect results.", - Long: "Run an automated test suite using the block-generator daemon and a provided algorand-indexer binary. Results are captured to a specified output directory.", + Long: "Run an automated test suite using the block-generator daemon and a provided conduit binary. Results are captured to a specified output directory.", Run: func(cmd *cobra.Command, args []string) { if err := Run(runnerArgs); err != nil { fmt.Println(err) @@ -43,12 +43,12 @@ func init() { } RunnerCmd.Flags().StringVarP(&runnerArgs.Path, "scenario", "s", "", "Directory containing scenarios, or specific scenario file.") - RunnerCmd.Flags().StringVarP(&runnerArgs.IndexerBinary, "indexer-binary", "i", "", "Path to indexer binary.") - RunnerCmd.Flags().Uint64VarP(&runnerArgs.IndexerPort, "indexer-port", "p", 4010, "Port to start the server at. This is useful if you have a prometheus server for collecting additional data.") + RunnerCmd.Flags().StringVarP(&runnerArgs.ConduitBinary, "conduit-binary", "i", "", "Path to conduit binary.") + RunnerCmd.Flags().Uint64VarP(&runnerArgs.MetricsPort, "metrics-port", "p", 9999, "Port to start the metrics server at.") RunnerCmd.Flags().StringVarP(&runnerArgs.PostgresConnectionString, "postgres-connection-string", "c", "", "Postgres connection string.") RunnerCmd.Flags().DurationVarP(&runnerArgs.RunDuration, "test-duration", "d", 5*time.Minute, "Duration to use for each scenario.") RunnerCmd.Flags().StringVarP(&runnerArgs.ReportDirectory, "report-directory", "r", "", "Location to place test reports.") - RunnerCmd.Flags().StringVarP(&runnerArgs.LogLevel, "log-level", "l", "error", "LogLevel to use when starting Indexer. [error, warn, info, debug, trace]") + RunnerCmd.Flags().StringVarP(&runnerArgs.LogLevel, "log-level", "l", "ERROR", "LogLevel to use when starting Indexer. [PANIC, FATAL, ERROR, WARN, INFO, DEBUG, TRACE]") RunnerCmd.Flags().StringVarP(&runnerArgs.CPUProfilePath, "cpuprofile", "", "", "Path where Indexer writes its CPU profile.") RunnerCmd.Flags().BoolVarP(&runnerArgs.ResetReportDir, "reset", "", false, "If set any existing report directory will be deleted before running tests.") RunnerCmd.Flags().BoolVarP(&runnerArgs.RunValidation, "validate", "", false, "If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure.") From 4cd3fe1342c10f0c8632ebd8a6b8c127d5f84fb8 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 14 Apr 2023 16:00:04 -0400 Subject: [PATCH 02/14] updates --- tools/block-generator/runner/run.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 0b1d49c518..05199bee73 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -125,8 +125,8 @@ func (r *Args) run() error { fmt.Printf("Failed to shutdown generator: %s\n", err) } }() - time.Sleep(1 * time.Second) - // write conduit config file + + // get conduit config template cwd, err := os.Getwd() if err != nil { return fmt.Errorf("current working directory: %w", err) @@ -136,6 +136,7 @@ func (r *Args) run() error { return fmt.Errorf("unable to open config template file: %w", err) } + // create config file in the right data directory f, err := os.Create(path.Join(dataDir, "conduit.yml")) if err != nil { return fmt.Errorf("creating conduit.yml: ", err) @@ -152,6 +153,7 @@ func (r *Args) run() error { return fmt.Errorf("execute template file: ", err) } + // Start indexer indexerShutdownFunc, err := startIndexer(dataDir, r.ConduitBinary) if err != nil { return fmt.Errorf("failed to start indexer: %w", err) From 0f7285b7d134ac700be728d1cbf5135858a1f81a Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 14 Apr 2023 16:02:55 -0400 Subject: [PATCH 03/14] lint fix --- tools/block-generator/generator/generate.go | 10 ++++++++-- tools/block-generator/runner/run.go | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index 6080c35e93..a8c245a488 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -373,7 +373,10 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { if err != nil { return err } - output.Write(block) + _, err = output.Write(block) + if err != nil { + return err + } g.finishRound(numTxnForBlock) return nil } @@ -438,7 +441,10 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { if err != nil { return err } - output.Write(block) + _, err = output.Write(block) + if err != nil { + return err + } g.finishRound(numTxnForBlock) return nil } diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 05199bee73..6117add0aa 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -139,7 +139,7 @@ func (r *Args) run() error { // create config file in the right data directory f, err := os.Create(path.Join(dataDir, "conduit.yml")) if err != nil { - return fmt.Errorf("creating conduit.yml: ", err) + return fmt.Errorf("creating conduit.yml: %v", err) } defer f.Close() @@ -150,7 +150,7 @@ func (r *Args) run() error { err = t.Execute(f, conduitConfig) if err != nil { - return fmt.Errorf("execute template file: ", err) + return fmt.Errorf("execute template file: %v", err) } // Start indexer From e6a45f916f0edc66f34b3acc30a55b8de604e0b8 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 14 Apr 2023 16:42:42 -0400 Subject: [PATCH 04/14] fix test --- tools/block-generator/generator/generate_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go index 78ca9e0323..e57d3f8006 100644 --- a/tools/block-generator/generator/generate_test.go +++ b/tools/block-generator/generator/generate_test.go @@ -203,6 +203,7 @@ func TestWriteRound(t *testing.T) { g := makePrivateGenerator(t) var data []byte writer := bytes.NewBuffer(data) + g.WriteBlock(writer, 0) g.WriteBlock(writer, 1) var block rpcs.EncodedBlockCert protocol.Decode(data, &block) From d9d8b67f794a37e6c28dfe009e76e15a0b327f33 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 19 Apr 2023 12:08:16 -0400 Subject: [PATCH 05/14] address PR comments --- tools/block-generator/runner/run.go | 29 ++++++++++--------- tools/block-generator/runner/runner.go | 2 +- .../template/conduit.yml.tmpl} | 0 3 files changed, 16 insertions(+), 15 deletions(-) rename tools/block-generator/{data/conduit.yml => runner/template/conduit.yml.tmpl} (100%) diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 6117add0aa..407a1b48bd 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -19,6 +19,8 @@ package runner import ( "bytes" "context" + // embed conduit template config file + _ "embed" "encoding/json" "fmt" "net/http" @@ -36,6 +38,9 @@ import ( "github.com/algorand/go-deadlock" ) +//go:embed template/conduit.yml.tmpl +var conduitConfigTmpl string + // Args are all the things needed to run a performance test. type Args struct { // Path is a directory when passed to RunBatch, otherwise a file path. @@ -127,13 +132,9 @@ func (r *Args) run() error { }() // get conduit config template - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("current working directory: %w", err) - } - t, err := template.ParseFiles(path.Join(cwd, "data/conduit.yml")) + t, err := template.New("conduit").Parse(conduitConfigTmpl) if err != nil { - return fmt.Errorf("unable to open config template file: %w", err) + return fmt.Errorf("unable to parse conduit config template: %w", err) } // create config file in the right data directory @@ -153,15 +154,15 @@ func (r *Args) run() error { return fmt.Errorf("execute template file: %v", err) } - // Start indexer - indexerShutdownFunc, err := startIndexer(dataDir, r.ConduitBinary) + // Start conduit + conduitShutdownFunc, err := startConduit(dataDir, r.ConduitBinary) if err != nil { - return fmt.Errorf("failed to start indexer: %w", err) + return fmt.Errorf("failed to start Conduit: %w", err) } defer func() { - // Shutdown indexer - if err := indexerShutdownFunc(); err != nil { - fmt.Printf("Failed to shutdown indexer: %s\n", err) + // Shutdown conduit + if err := conduitShutdownFunc(); err != nil { + fmt.Printf("Failed to shutdown Conduit: %s\n", err) } }() @@ -394,8 +395,8 @@ func startGenerator(configFile string, addr string, blockMiddleware func(http.Ha }, generator } -// startIndexer starts the indexer. -func startIndexer(dataDir string, conduitBinary string) (func() error, error) { +// startConduit starts the indexer. +func startConduit(dataDir string, conduitBinary string) (func() error, error) { cmd := exec.Command( conduitBinary, "-d", dataDir, diff --git a/tools/block-generator/runner/runner.go b/tools/block-generator/runner/runner.go index 2eab45fea5..6c653efaec 100644 --- a/tools/block-generator/runner/runner.go +++ b/tools/block-generator/runner/runner.go @@ -48,7 +48,7 @@ func init() { RunnerCmd.Flags().StringVarP(&runnerArgs.PostgresConnectionString, "postgres-connection-string", "c", "", "Postgres connection string.") RunnerCmd.Flags().DurationVarP(&runnerArgs.RunDuration, "test-duration", "d", 5*time.Minute, "Duration to use for each scenario.") RunnerCmd.Flags().StringVarP(&runnerArgs.ReportDirectory, "report-directory", "r", "", "Location to place test reports.") - RunnerCmd.Flags().StringVarP(&runnerArgs.LogLevel, "log-level", "l", "ERROR", "LogLevel to use when starting Indexer. [PANIC, FATAL, ERROR, WARN, INFO, DEBUG, TRACE]") + RunnerCmd.Flags().StringVarP(&runnerArgs.LogLevel, "log-level", "l", "error", "LogLevel to use when starting Indexer. [panic, fatal, error, warn, info, debug, trace]") RunnerCmd.Flags().StringVarP(&runnerArgs.CPUProfilePath, "cpuprofile", "", "", "Path where Indexer writes its CPU profile.") RunnerCmd.Flags().BoolVarP(&runnerArgs.ResetReportDir, "reset", "", false, "If set any existing report directory will be deleted before running tests.") RunnerCmd.Flags().BoolVarP(&runnerArgs.RunValidation, "validate", "", false, "If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure.") diff --git a/tools/block-generator/data/conduit.yml b/tools/block-generator/runner/template/conduit.yml.tmpl similarity index 100% rename from tools/block-generator/data/conduit.yml rename to tools/block-generator/runner/template/conduit.yml.tmpl From 1980571ec5fa4d2400ae0952cb8f3ec8010f2efa Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 19 Apr 2023 14:07:37 -0400 Subject: [PATCH 06/14] update scripts --- tools/block-generator/run_generator.sh | 56 -------------------------- tools/block-generator/run_postgres.sh | 2 +- tools/block-generator/run_runner.sh | 13 +++--- tools/block-generator/run_tests.sh | 19 +++++---- 4 files changed, 18 insertions(+), 72 deletions(-) delete mode 100755 tools/block-generator/run_generator.sh diff --git a/tools/block-generator/run_generator.sh b/tools/block-generator/run_generator.sh deleted file mode 100755 index 47e87b2e0d..0000000000 --- a/tools/block-generator/run_generator.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash - -# Demonstrate how to run the generator and connect it to indexer. - -set -e - -POSTGRES_CONTAINER=generator-test-container -POSTGRES_PORT=15432 -POSTGRES_DATABASE=generator_db -CONFIG=${1:-"$(dirname $0)/test_config.yml"} -echo "Using config file: $CONFIG" - -function start_postgres() { - docker rm -f $POSTGRES_CONTAINER > /dev/null 2>&1 || true - - # Start postgres container... - docker run \ - -d \ - --name $POSTGRES_CONTAINER \ - -e POSTGRES_USER=algorand \ - -e POSTGRES_PASSWORD=algorand \ - -e PGPASSWORD=algorand \ - -p $POSTGRES_PORT:5432 \ - postgres - - sleep 5 - - docker exec -it $POSTGRES_CONTAINER psql -Ualgorand -c "create database $POSTGRES_DATABASE" -} - -function shutdown() { - docker rm -f $POSTGRES_CONTAINER > /dev/null 2>&1 || true - kill -9 $GENERATOR_PID -} - -trap shutdown EXIT - -echo "Building generator." -pushd $(dirname "$0") > /dev/null -go build -cd ../.. > /dev/null -echo "Building indexer." -make -popd -echo "Starting postgres container." -start_postgres -echo "Starting block generator (see generator.log)" -$(dirname "$0")/block-generator daemon --port 11111 --config "${CONFIG}" & -GENERATOR_PID=$! -echo "Starting indexer" -$(dirname "$0")/../../cmd/algorand-indexer/algorand-indexer daemon \ - -S localhost:8980 \ - --algod-net localhost:11111 \ - --algod-token security-is-our-number-one-priority \ - --metrics-mode VERBOSE \ - -P "host=localhost user=algorand password=algorand dbname=generator_db port=15432 sslmode=disable" diff --git a/tools/block-generator/run_postgres.sh b/tools/block-generator/run_postgres.sh index c6a967132b..4817961220 100755 --- a/tools/block-generator/run_postgres.sh +++ b/tools/block-generator/run_postgres.sh @@ -4,7 +4,7 @@ # in a debugger. Simply start this script and run with: # ./block-generator runner \ # -d 5s \ -# -i ./../algorand-indexer/algorand-indexer \ +# -i \ # -c "host=localhost user=algorand password=algorand dbname=algorand port=15432 sslmode=disable" \ # -r results \ # -s scenarios/config.payment.small.yml diff --git a/tools/block-generator/run_runner.sh b/tools/block-generator/run_runner.sh index d90749b96b..7f1205cae5 100755 --- a/tools/block-generator/run_runner.sh +++ b/tools/block-generator/run_runner.sh @@ -4,10 +4,16 @@ set -e +CONDUIT_BINARY=$1 +if [ -z "$CONDUIT_BINARY" ]; then + echo "path to conduit binary is required" + exit 1 +fi + POSTGRES_CONTAINER=generator-test-container POSTGRES_PORT=15432 POSTGRES_DATABASE=generator_db -CONFIG=${1:-"$(dirname $0)/test_config.yml"} +CONFIG=${2:-"$(dirname $0)/test_config.yml"} echo "Using config file: $CONFIG" function start_postgres() { @@ -38,15 +44,12 @@ rm -rf OUTPUT_RUN_RUNNER_TEST > /dev/null 2>&1 echo "Building generator." pushd $(dirname "$0") > /dev/null go build -cd ../.. > /dev/null -echo "Building indexer." -make popd echo "Starting postgres container." start_postgres echo "Starting test runner" $(dirname "$0")/block-generator runner \ - --indexer-binary ../algorand-indexer/algorand-indexer \ + --conduit-binary "$CONDUIT_BINARY" \ --report-directory OUTPUT_RUN_RUNNER_TEST \ --test-duration 30s \ --log-level trace \ diff --git a/tools/block-generator/run_tests.sh b/tools/block-generator/run_tests.sh index 8befa41dc4..5d25d50af4 100755 --- a/tools/block-generator/run_tests.sh +++ b/tools/block-generator/run_tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash CONNECTION_STRING="" -INDEXER_BINARY="" +CONDUIT_BINARY="" REPORT_DIR="" DURATION="1h" LOG_LEVEL="error" @@ -17,7 +17,7 @@ help() { echo " -r|--report-dir directory where the report should be written." echo " -d|--duration test duration." echo " -l|--level log level to pass to Indexer." - echo " -g|--generator use a different block-generator binary to run the generator." + echo " -g|--generator block-generator binary to run the generator." exit } @@ -34,7 +34,7 @@ while :; do shift ;; -i | --indexer) - INDEXER_BINARY="${2-}" + CONDUIT_BINARY="${2-}" shift ;; -r | --report-dir) @@ -66,7 +66,7 @@ if [ -z "$CONNECTION_STRING" ]; then exit 1 fi -if [ -z "$INDEXER_BINARY" ]; then +if [ -z "$CONDUIT_BINARY" ]; then echo "Missing required indexer binary parameter (-i / --indexer)." exit 1 fi @@ -77,18 +77,17 @@ if [ -z "$SCENARIOS" ]; then fi if [ -z "$GENERATOR_BINARY" ]; then - echo "Using indexer binary for generator, override with (-g / --generator)." - GENERATOR_BINARY="$INDEXER_BINARY" + echo "path to block-generator binary is required" + exit 1 fi -echo "Running with binary: $INDEXER_BINARY" +echo "Running with binary: $CONDUIT_BINARY" echo "Report directory: $REPORT_DIR" echo "Duration: $DURATION" echo "Log Level: $LOG_LEVEL" -"$GENERATOR_BINARY" \ - util block-generator runner \ - -i "$INDEXER_BINARY" \ +"$GENERATOR_BINARY" runner \ + -i "$CONDUIT_BINARY" \ -s "$SCENARIOS" \ -d "$DURATION" \ -c "$CONNECTION_STRING" \ From 32ad557438a9f33e2678d3ae1e324544104bcc4a Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 19 Apr 2023 14:47:12 -0400 Subject: [PATCH 07/14] wait for /metrics to be available --- tools/block-generator/runner/run.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 407a1b48bd..5e68cd49ae 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -174,7 +174,20 @@ func (r *Args) run() error { defer report.Close() // Run the test, collecting results. - if err := r.runTest(report, metricsNet, algodNet); err != nil { + // check /metrics endpoint is available before running the test + var resp *http.Response + for retry := 0; retry < 10; retry++ { + resp, err = http.Get(fmt.Sprintf("http://%s/metrics", metricsNet)) + if err == nil { + resp.Body.Close() + break + } + time.Sleep(3 * time.Second) + } + if err != nil { + return fmt.Errorf("failed to query metrics endpoint: %w", err) + } + if err = r.runTest(report, metricsNet, algodNet); err != nil { return err } From c59c4770c1cbdf65fa49279f3a1c2bc0261a24ed Mon Sep 17 00:00:00 2001 From: Zeph Grunschlag Date: Thu, 20 Apr 2023 16:05:03 -0500 Subject: [PATCH 08/14] tools: copy/replace indexer/conduit in block-generator --- tools/block-generator/README.md | 67 ++++++++++++++++++++------ tools/block-generator/run_tests.sh | 12 ++--- tools/block-generator/runner/run.go | 17 ++++--- tools/block-generator/runner/runner.go | 6 +-- 4 files changed, 70 insertions(+), 32 deletions(-) diff --git a/tools/block-generator/README.md b/tools/block-generator/README.md index 5caff65225..4b2fd9b5a6 100644 --- a/tools/block-generator/README.md +++ b/tools/block-generator/README.md @@ -1,6 +1,6 @@ # Block Generator -This tool is used for testing Indexer import performance. It does this by generating synthetic blocks which are sent by mocking the Algod REST API endpoints that Indexer uses. +This tool is used for testing Conduit import performance. It does this by generating synthetic blocks which are sent by mocking the Algod REST API endpoints that Conduit uses. ## Scenario Configuration @@ -9,11 +9,11 @@ Block generator uses a YAML config file to describe the composition of each rand 2. Transaction type distribution 3. Transaction type specific configuration -At the time of writing, the block generator supports **payment** and **asset** transactions. The settings are hopefully, more or less, obvious. Distributions are specified as fractions of one, and the sum of all options must add up to one. +At the time of writing, the block generator supports **payment** and **asset** transactions. The settings are hopefully, more or less, obvious. Distributions are specified as fractions of 1.0, and the sum of all options must add up to 1.0. -Here is an example which uses all of the current options. Notice that the synthetic blocks are not required to follow algod limits, in this case the block size is specified as 19999, or four times larger than the current block size limit: +Here is an example which uses all of the current options. Notice that the synthetic blocks are not required to follow algod limits, in this case the block size is specified as 19999: ``` -name: "Mixed (jumbo)" +name: "Mixed (19,999)" genesis_accounts: 10000 genesis_account_balance: 1000000000000 tx_per_block: 19999 @@ -36,13 +36,15 @@ asset_delete_fraction: 0 ## Modes -The block generator can run in one of two modes, a standalone **daemon**, or a test suite **runner** +The block generator can run in one of two _modes_: +1. standalone **daemon** +2. test suite **runner** ### daemon -In standalone mode, a block-generator process starts and exposes the mock algod endpoints for **/genesis** and **/v2/blocks/{block}**. If you choose to query them manually, it only supports fetching blocks sequentially. This is due to the fact that it generates a pseudorandom stream of transactions and after each random transaction the state increments to the next. +In standalone daemon mode, a block-generator process starts and exposes the mock algod endpoints for **/genesis** and **/v2/blocks/{block}**. If you choose to query them manually, it only supports fetching blocks sequentially. This is due to the fact that it generates a pseudorandom stream of transactions and after each random transaction the state increments to the next. -Here is the help output: +Here is the help output for **daemon**: ```bash ~$ ./block-generator daemon -h Start the generator daemon in standalone mode. @@ -58,9 +60,9 @@ Flags: ### runner -For our usage, we want to run the same set of tests consistently across many scenarios and with many different releases. The runner mode automates this process by starting the **daemon** with many different configurations, managing a postgres database, and running a separate indexer process configured to use them. +The runner mode is well suited for runing the same set of tests consistently across many scenarios and for different releases. The runner mode automates this process by starting the **daemon** with many different configurations, managing a postgres database, and running a separate Conduit process configured to use them. -The results of the testing are written to the directory specified by the **--report-directory** option, and include many different metrics. In addition to the report, the indexer log is written to this directory. The files are named according to the scenario file, and end in "report" or "log". +The results of the testing are written to the directory specified by the **--report-directory** option, and include many different metrics. In addition to the report, the Conduit log is written to this directory. The files are named according to the scenario file, and end in "report" or "log". Here is an example report from running with a test duration of "1h": ``` @@ -92,20 +94,21 @@ final_overall_transactions_per_second:8493.40 final_uptime_seconds:3600.06 ``` -Here is the help output: +Here is the help output for **runner**: ```bash ~$ ./block-generator runner -h -Run test suite and collect results. +Run an automated test suite using the block-generator daemon and a provided conduit binary. Results are captured to a specified output directory. Usage: block-generator runner [flags] Flags: - --cpuprofile string Path where Indexer writes its CPU profile. + -i, --conduit-binary string Path to conduit binary. + --cpuprofile string Path where conduit writes its CPU profile. -h, --help help for runner - -i, --indexer-binary string Path to indexer binary. - -p, --indexer-port uint Port to start the server at. This is useful if you have a prometheus server for collecting additional data. (default 4010) - -l, --log-level string LogLevel to use when starting Indexer. [error, warn, info, debug, trace] (default "error") + -k, --keep-data-dir If set the validator will not delete the data directory after tests complete. + -l, --log-level string LogLevel to use when starting conduit. [panic, fatal, error, warn, info, debug, trace] (default "error") + -p, --metrics-port uint Port to start the metrics server at. (default 9999) -c, --postgres-connection-string string Postgres connection string. -r, --report-directory string Location to place test reports. --reset If set any existing report directory will be deleted before running tests. @@ -113,3 +116,37 @@ Flags: -d, --test-duration duration Duration to use for each scenario. (default 5m0s) --validate If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure. ``` + +## Example Scenario Run using Conduit and Postgres - `run_runner.sh` + +A typical **runner** scenario involves: +* a [scenario configuration](#scenario-configuration) file, e.g. [test_config.yml](./test_config.yml) +* access to a `conduit` binary to query the block generator's mock Algod endpoint and ingest the synthetic blocks +* a datastore -such as a postgres database- to collect `conduit`'s output +* a `conduit` config file to define its import/export behavior + +`run_runner.sh` makes the following choices for the previous bullet points: +* it can accept any scenario as its second argument, but defaults to [test_config.yml](./test_config.yml) when this isn't provided (this is a scenario with a lifetime of ~30 seconds) +* knows how to import through a mock Algod running on port 11112 (which is the port the runner avails) +* sets up a dockerized postgres database to receive conduit's output +* configures `conduit` for these specs using [this config template](./runner/template/conduit.yml.tmpl) + +### Sample Run + +First you'll need to get a `conduit` binary. For example you can follow the [developer portal's instructions](https://developer.algorand.org/docs/get-details/conduit/GettingStarted/#installation) or run `go build .` inside of the directory `cmd/conduit` after downloading the `conduit` repo. + +Assume you've navigated to the `tools/block-generator` directory of +the `go-algorand` repo, and: +* saved the conduit binary to `tools/block-generator/conduit` +* created a block generator scenario config at `tools/block-generator/scenario.yml` + +Then you can execute the following command to run the scenario: +```sh +./run_runner.sh ./conduit scenario.yml +``` + +### Scenario Report + +If all goes well, the run will generate a directory `tools/block-generator/OUTPUT_RUN_RUNNER_TEST` and in that directory you can see the statistics +of the run in `scenario.report`. + diff --git a/tools/block-generator/run_tests.sh b/tools/block-generator/run_tests.sh index 5d25d50af4..7916f45267 100755 --- a/tools/block-generator/run_tests.sh +++ b/tools/block-generator/run_tests.sh @@ -12,11 +12,11 @@ help() { echo " -v|--verbose enable verbose script output." echo " -c|--connection-string" echo " PostgreSQL connection string." - echo " -i|--indexer path to indexer binary." - echo " -s|--scenarios path to indexer test scenarios." + echo " -i|--conduit path to conduit binary." + echo " -s|--scenarios path to conduit test scenarios." echo " -r|--report-dir directory where the report should be written." echo " -d|--duration test duration." - echo " -l|--level log level to pass to Indexer." + echo " -l|--level log level to pass to conduit." echo " -g|--generator block-generator binary to run the generator." exit } @@ -33,7 +33,7 @@ while :; do GENERATOR_BINARY="${2-}" shift ;; - -i | --indexer) + -i | --conduit) CONDUIT_BINARY="${2-}" shift ;; @@ -67,12 +67,12 @@ if [ -z "$CONNECTION_STRING" ]; then fi if [ -z "$CONDUIT_BINARY" ]; then - echo "Missing required indexer binary parameter (-i / --indexer)." + echo "Missing required conduit binary parameter (-i / --conduit)." exit 1 fi if [ -z "$SCENARIOS" ]; then - echo "Missing required indexer test scenario parameter (-s / --scenarios)." + echo "Missing required conduit test scenario parameter (-s / --scenarios)." exit 1 fi diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 5e68cd49ae..fe1ac1f35d 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -19,6 +19,7 @@ package runner import ( "bytes" "context" + // embed conduit template config file _ "embed" "encoding/json" @@ -101,7 +102,7 @@ func (r *Args) run() error { baseName := filepath.Base(r.Path) baseNameNoExt := strings.TrimSuffix(baseName, filepath.Ext(baseName)) reportfile := path.Join(r.ReportDirectory, fmt.Sprintf("%s.report", baseNameNoExt)) - logfile := path.Join(r.ReportDirectory, fmt.Sprintf("%s.indexer-log", baseNameNoExt)) + logfile := path.Join(r.ReportDirectory, fmt.Sprintf("%s.conduit-log", baseNameNoExt)) dataDir := path.Join(r.ReportDirectory, fmt.Sprintf("%s_data", baseNameNoExt)) // create the data directory. if err := os.Mkdir(dataDir, os.ModeDir|os.ModePerm); err != nil { @@ -127,7 +128,7 @@ func (r *Args) run() error { defer func() { // Shutdown generator. if err := generatorShutdownFunc(); err != nil { - fmt.Printf("Failed to shutdown generator: %s\n", err) + fmt.Printf("failed to shutdown generator: %s\n", err) } }() @@ -140,7 +141,7 @@ func (r *Args) run() error { // create config file in the right data directory f, err := os.Create(path.Join(dataDir, "conduit.yml")) if err != nil { - return fmt.Errorf("creating conduit.yml: %v", err) + return fmt.Errorf("problem creating conduit.yml: %v", err) } defer f.Close() @@ -151,7 +152,7 @@ func (r *Args) run() error { err = t.Execute(f, conduitConfig) if err != nil { - return fmt.Errorf("execute template file: %v", err) + return fmt.Errorf("problem executing template file: %v", err) } // Start conduit @@ -162,7 +163,7 @@ func (r *Args) run() error { defer func() { // Shutdown conduit if err := conduitShutdownFunc(); err != nil { - fmt.Printf("Failed to shutdown Conduit: %s\n", err) + fmt.Printf("failed to shutdown Conduit: %s\n", err) } }() @@ -408,7 +409,7 @@ func startGenerator(configFile string, addr string, blockMiddleware func(http.Ha }, generator } -// startConduit starts the indexer. +// startConduit starts the conduit. func startConduit(dataDir string, conduitBinary string) (func() error, error) { cmd := exec.Command( conduitBinary, @@ -427,9 +428,9 @@ func startConduit(dataDir string, conduitBinary string) (func() error, error) { return func() error { if err := cmd.Process.Signal(os.Interrupt); err != nil { - fmt.Printf("failed to kill indexer process: %s\n", err) + fmt.Printf("failed to kill conduit process: %s\n", err) if err := cmd.Process.Kill(); err != nil { - return fmt.Errorf("failed to kill indexer process: %w", err) + return fmt.Errorf("failed to kill conduit process: %w", err) } } if err := cmd.Wait(); err != nil { diff --git a/tools/block-generator/runner/runner.go b/tools/block-generator/runner/runner.go index 6c653efaec..51149acc14 100644 --- a/tools/block-generator/runner/runner.go +++ b/tools/block-generator/runner/runner.go @@ -48,14 +48,14 @@ func init() { RunnerCmd.Flags().StringVarP(&runnerArgs.PostgresConnectionString, "postgres-connection-string", "c", "", "Postgres connection string.") RunnerCmd.Flags().DurationVarP(&runnerArgs.RunDuration, "test-duration", "d", 5*time.Minute, "Duration to use for each scenario.") RunnerCmd.Flags().StringVarP(&runnerArgs.ReportDirectory, "report-directory", "r", "", "Location to place test reports.") - RunnerCmd.Flags().StringVarP(&runnerArgs.LogLevel, "log-level", "l", "error", "LogLevel to use when starting Indexer. [panic, fatal, error, warn, info, debug, trace]") - RunnerCmd.Flags().StringVarP(&runnerArgs.CPUProfilePath, "cpuprofile", "", "", "Path where Indexer writes its CPU profile.") + RunnerCmd.Flags().StringVarP(&runnerArgs.LogLevel, "log-level", "l", "error", "LogLevel to use when starting conduit. [panic, fatal, error, warn, info, debug, trace]") + RunnerCmd.Flags().StringVarP(&runnerArgs.CPUProfilePath, "cpuprofile", "", "", "Path where conduit writes its CPU profile.") RunnerCmd.Flags().BoolVarP(&runnerArgs.ResetReportDir, "reset", "", false, "If set any existing report directory will be deleted before running tests.") RunnerCmd.Flags().BoolVarP(&runnerArgs.RunValidation, "validate", "", false, "If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure.") RunnerCmd.Flags().BoolVarP(&runnerArgs.KeepDataDir, "keep-data-dir", "k", false, "If set the validator will not delete the data directory after tests complete.") RunnerCmd.MarkFlagRequired("scenario") - RunnerCmd.MarkFlagRequired("indexer-binary") + RunnerCmd.MarkFlagRequired("conduit-binary") RunnerCmd.MarkFlagRequired("postgres-connection-string") RunnerCmd.MarkFlagRequired("report-directory") } From b2ae15aeb93eb4424590cf6a1f7acbfdda8a9f98 Mon Sep 17 00:00:00 2001 From: Zeph Grunschlag Date: Thu, 20 Apr 2023 16:16:34 -0500 Subject: [PATCH 09/14] update report output on README --- tools/block-generator/README.md | 48 +++++++++++++++------------------ 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/tools/block-generator/README.md b/tools/block-generator/README.md index 4b2fd9b5a6..82da184116 100644 --- a/tools/block-generator/README.md +++ b/tools/block-generator/README.md @@ -66,32 +66,28 @@ The results of the testing are written to the directory specified by the **--rep Here is an example report from running with a test duration of "1h": ``` -test_duration_seconds:3600 -test_duration_actual_seconds:3600.056457 -transaction_pay_total:30024226 -transaction_pay_create_total:614242 -early_average_import_time_sec:2.13 -early_cumulative_import_time_sec:1083.26 -early_average_imported_tx_per_block:19999.00 -early_cumulative_imported_tx_per_block:10179491 -early_average_block_upload_time_sec:NaN -early_cumulative_block_upload_time_sec:0.00 -early_average_postgres_eval_time_sec:0.33 -early_cumulative_postgres_eval_time_sec:167.41 -early_imported_round:509 -early_overall_transactions_per_second:9397.09 -early_uptime_seconds:3600.06 -final_average_import_time_sec:2.35 -final_cumulative_import_time_sec:3602.62 -final_average_imported_tx_per_block:19999.00 -final_cumulative_imported_tx_per_block:30598470 -final_average_block_upload_time_sec:NaN -final_cumulative_block_upload_time_sec:0.00 -final_average_postgres_eval_time_sec:0.33 -final_cumulative_postgres_eval_time_sec:507.38 -final_imported_round:1530 -final_overall_transactions_per_second:8493.40 -final_uptime_seconds:3600.06 +test_duration_seconds:30 +test_duration_actual_seconds:30.018076 +transaction_asset_close_total:472 +transaction_asset_create_total:711 +transaction_asset_optin_total:1230 +transaction_asset_xfer_total:468 +transaction_pay_total:1457 +transaction_pay_create_total:1472 +early_average_import_time_sec:0.05 +early_cumulative_import_time_sec:11.05 +early_average_imported_tx_per_block:10.00 +early_cumulative_imported_tx_per_block:2390 +early_imported_round:239 +early_overall_transactions_per_second:216.26 +early_uptime_seconds:30.02 +final_average_import_time_sec:0.05 +final_cumulative_import_time_sec:31.36 +final_average_imported_tx_per_block:10.00 +final_cumulative_imported_tx_per_block:5800 +final_imported_round:580 +final_overall_transactions_per_second:184.93 +final_uptime_seconds:30.02 ``` Here is the help output for **runner**: From 49e90e95178be77f68fc1292e5e5c6d63b6c3df8 Mon Sep 17 00:00:00 2001 From: Zeph Grunschlag Date: Thu, 20 Apr 2023 16:23:50 -0500 Subject: [PATCH 10/14] Update tools/block-generator/runner/runner.go --- tools/block-generator/runner/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/block-generator/runner/runner.go b/tools/block-generator/runner/runner.go index 51149acc14..89e4ac88c4 100644 --- a/tools/block-generator/runner/runner.go +++ b/tools/block-generator/runner/runner.go @@ -48,8 +48,8 @@ func init() { RunnerCmd.Flags().StringVarP(&runnerArgs.PostgresConnectionString, "postgres-connection-string", "c", "", "Postgres connection string.") RunnerCmd.Flags().DurationVarP(&runnerArgs.RunDuration, "test-duration", "d", 5*time.Minute, "Duration to use for each scenario.") RunnerCmd.Flags().StringVarP(&runnerArgs.ReportDirectory, "report-directory", "r", "", "Location to place test reports.") - RunnerCmd.Flags().StringVarP(&runnerArgs.LogLevel, "log-level", "l", "error", "LogLevel to use when starting conduit. [panic, fatal, error, warn, info, debug, trace]") - RunnerCmd.Flags().StringVarP(&runnerArgs.CPUProfilePath, "cpuprofile", "", "", "Path where conduit writes its CPU profile.") + RunnerCmd.Flags().StringVarP(&runnerArgs.LogLevel, "log-level", "l", "error", "LogLevel to use when starting Conduit. [panic, fatal, error, warn, info, debug, trace]") + RunnerCmd.Flags().StringVarP(&runnerArgs.CPUProfilePath, "cpuprofile", "", "", "Path where Conduit writes its CPU profile.") RunnerCmd.Flags().BoolVarP(&runnerArgs.ResetReportDir, "reset", "", false, "If set any existing report directory will be deleted before running tests.") RunnerCmd.Flags().BoolVarP(&runnerArgs.RunValidation, "validate", "", false, "If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure.") RunnerCmd.Flags().BoolVarP(&runnerArgs.KeepDataDir, "keep-data-dir", "k", false, "If set the validator will not delete the data directory after tests complete.") From 920878746e6f882fcc8a8fb4225d6898c5756d76 Mon Sep 17 00:00:00 2001 From: Zeph Grunschlag Date: Thu, 20 Apr 2023 16:38:12 -0500 Subject: [PATCH 11/14] Update tools/block-generator/runner/run.go --- tools/block-generator/runner/run.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 22102c4396..3631d3bbb1 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -19,7 +19,6 @@ package runner import ( "bytes" "context" - // embed conduit template config file _ "embed" "encoding/json" From ca72ea4a236ea91bea27d5f104ee3971fb83e434 Mon Sep 17 00:00:00 2001 From: Zeph Grunschlag Date: Thu, 20 Apr 2023 16:41:11 -0500 Subject: [PATCH 12/14] Update tools/block-generator/README.md --- tools/block-generator/README.md | 48 ++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/tools/block-generator/README.md b/tools/block-generator/README.md index 82da184116..4b2fd9b5a6 100644 --- a/tools/block-generator/README.md +++ b/tools/block-generator/README.md @@ -66,28 +66,32 @@ The results of the testing are written to the directory specified by the **--rep Here is an example report from running with a test duration of "1h": ``` -test_duration_seconds:30 -test_duration_actual_seconds:30.018076 -transaction_asset_close_total:472 -transaction_asset_create_total:711 -transaction_asset_optin_total:1230 -transaction_asset_xfer_total:468 -transaction_pay_total:1457 -transaction_pay_create_total:1472 -early_average_import_time_sec:0.05 -early_cumulative_import_time_sec:11.05 -early_average_imported_tx_per_block:10.00 -early_cumulative_imported_tx_per_block:2390 -early_imported_round:239 -early_overall_transactions_per_second:216.26 -early_uptime_seconds:30.02 -final_average_import_time_sec:0.05 -final_cumulative_import_time_sec:31.36 -final_average_imported_tx_per_block:10.00 -final_cumulative_imported_tx_per_block:5800 -final_imported_round:580 -final_overall_transactions_per_second:184.93 -final_uptime_seconds:30.02 +test_duration_seconds:3600 +test_duration_actual_seconds:3600.056457 +transaction_pay_total:30024226 +transaction_pay_create_total:614242 +early_average_import_time_sec:2.13 +early_cumulative_import_time_sec:1083.26 +early_average_imported_tx_per_block:19999.00 +early_cumulative_imported_tx_per_block:10179491 +early_average_block_upload_time_sec:NaN +early_cumulative_block_upload_time_sec:0.00 +early_average_postgres_eval_time_sec:0.33 +early_cumulative_postgres_eval_time_sec:167.41 +early_imported_round:509 +early_overall_transactions_per_second:9397.09 +early_uptime_seconds:3600.06 +final_average_import_time_sec:2.35 +final_cumulative_import_time_sec:3602.62 +final_average_imported_tx_per_block:19999.00 +final_cumulative_imported_tx_per_block:30598470 +final_average_block_upload_time_sec:NaN +final_cumulative_block_upload_time_sec:0.00 +final_average_postgres_eval_time_sec:0.33 +final_cumulative_postgres_eval_time_sec:507.38 +final_imported_round:1530 +final_overall_transactions_per_second:8493.40 +final_uptime_seconds:3600.06 ``` Here is the help output for **runner**: From 9dc16955cce7bed32a8b3e0d41f6331e02c77c1e Mon Sep 17 00:00:00 2001 From: Zeph Grunschlag Date: Fri, 21 Apr 2023 11:59:25 -0500 Subject: [PATCH 13/14] merge + %w instead of %v --- data/transactions/logic/assembler_test.go | 1504 ++++--------------- tools/block-generator/generator/generate.go | 2 +- tools/block-generator/generator/utils.go | 2 +- tools/block-generator/runner/run.go | 5 +- 4 files changed, 313 insertions(+), 1200 deletions(-) diff --git a/data/transactions/logic/assembler_test.go b/data/transactions/logic/assembler_test.go index 7c515c6931..2a63b75123 100644 --- a/data/transactions/logic/assembler_test.go +++ b/data/transactions/logic/assembler_test.go @@ -17,15 +17,11 @@ package logic import ( - "bytes" "encoding/hex" - "errors" "fmt" - "regexp" "strings" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/algorand/go-algorand/config" @@ -34,8 +30,7 @@ import ( ) // used by TestAssemble and others, see UPDATE PROCEDURE in TestAssemble() -const v1Nonsense = ` -err +const v1Nonsense = `err global MinTxnFee global MinBalance global MaxTxnLife @@ -121,8 +116,6 @@ intc 1 intc 1 ! % -| -& ^ ~ byte 0x4242 @@ -348,7 +341,6 @@ byte 0x0123456789abcd dup dup ecdsa_pk_recover Secp256k1 -itxn Sender itxna Logs 3 ` @@ -380,12 +372,6 @@ const boxNonsense = ` box_get ` -const randomnessNonsense = ` -pushint 0xffff -block BlkTimestamp -vrf_verify VrfAlgorand -` - const v7Nonsense = v6Nonsense + ` base64_decode URLEncoding json_ref JSONUint64 @@ -401,78 +387,38 @@ pushbytes 0x012345 dup dup ed25519verify_bare -` + randomnessNonsense + ` pushbytes 0x4321 pushbytes 0x77 replace2 2 pushbytes 0x88 pushint 1 replace3 -` +` + boxNonsense + pairingNonsense -const switchNonsense = ` -switch_label0: -pushint 1 -switch switch_label0 switch_label1 -switch_label1: -pushint 1 -` +const v6Compiled = "2004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f23102311231223132314181b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b53a03b6b7043cb8033a0c2349c42a9631007300810881088120978101c53a8101c6003a" -const matchNonsense = ` -match_label0: -pushints 1 2 1 -match match_label0 match_label1 -match_label1: -pushbytess "1" "2" "1" -` - -const v8Nonsense = v7Nonsense + switchNonsense + frameNonsense + matchNonsense + boxNonsense - -const v9Nonsense = v8Nonsense -const v10Nonsense = v9Nonsense + pairingNonsense - -const v6Compiled = "2004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f2310231123122313231418191a1b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b400b53a03b6b7043cb8033a0c2349c42a9631007300810881088120978101c53a8101c6003a" - -const randomnessCompiled = "81ffff03d101d000" - -const v7Compiled = v6Compiled + "5e005f018120af060180070123456789abcd49490501988003012345494984" + - randomnessCompiled + "800243218001775c0280018881015d" +const v7Compiled = v6Compiled + "5e005f018120af060180070123456789abcd49490501988003012345494984800243218001775c0280018881015d" + boxCompiled + pairingCompiled const boxCompiled = "b9babbbcbdbfbe" -const switchCompiled = "81018d02fff800008101" -const matchCompiled = "83030102018e02fff500008203013101320131" - -const v8Compiled = v7Compiled + switchCompiled + frameCompiled + matchCompiled + boxCompiled - -const v9Compiled = v8Compiled -const v10Compiled = v9Compiled + pairingCompiled - var nonsense = map[uint64]string{ - 1: v1Nonsense, - 2: v2Nonsense, - 3: v3Nonsense, - 4: v4Nonsense, - 5: v5Nonsense, - 6: v6Nonsense, - 7: v7Nonsense, - 8: v8Nonsense, - 9: v9Nonsense, - 10: v10Nonsense, + 1: v1Nonsense, + 2: v2Nonsense, + 3: v3Nonsense, + 4: v4Nonsense, + 5: v5Nonsense, + 6: v6Nonsense, + 7: v7Nonsense, } var compiled = map[uint64]string{ - 1: "012008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f2310231123122313231418191a1b1c2b1716154000032903494", - 2: "022008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f2310231123122313231418191a1b1c2b171615400003290349483403350222231d4a484848482a50512a63222352410003420000432105602105612105270463484821052b62482b642b65484821052b2106662b21056721072b682b692107210570004848210771004848361c0037001a0031183119311b311d311e311f3120210721051e312131223123312431253126312731283129312a312b312c312d312e312f", - 3: "032008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f2310231123122313231418191a1b1c2b171615400003290349483403350222231d4a484848482a50512a63222352410003420000432105602105612105270463484821052b62482b642b65484821052b2106662b21056721072b682b692107210570004848210771004848361c0037001a0031183119311b311d311e311f3120210721051e312131223123312431253126312731283129312a312b312c312d312e312f4478222105531421055427042106552105082106564c4d4b02210538212106391c0081e80780046a6f686e", - 4: "042004010200b7a60c26040242420c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f2310231123122313231418191a1b1c28171615400003290349483403350222231d4a484848482a50512a632223524100034200004322602261222b634848222862482864286548482228236628226724286828692422700048482471004848361c0037001a0031183119311b311d311e311f312024221e312131223123312431253126312731283129312a312b312c312d312e312f44782522531422542b2355220823564c4d4b0222382123391c0081e80780046a6f686e2281d00f24231f880003420001892223902291922394239593a0a1a2a3a4a5a6a7a8a9aaabacadae23af3a00003b003c003d8164", - 5: "052004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f2310231123122313231418191a1b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b400b53a03", + 1: "012008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f23102311231223132314181b1c2b1716154000032903494", + 2: "022008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f23102311231223132314181b1c2b171615400003290349483403350222231d4a484848482a50512a63222352410003420000432105602105612105270463484821052b62482b642b65484821052b2106662b21056721072b682b692107210570004848210771004848361c0037001a0031183119311b311d311e311f3120210721051e312131223123312431253126312731283129312a312b312c312d312e312f", + 3: "032008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f23102311231223132314181b1c2b171615400003290349483403350222231d4a484848482a50512a63222352410003420000432105602105612105270463484821052b62482b642b65484821052b2106662b21056721072b682b692107210570004848210771004848361c0037001a0031183119311b311d311e311f3120210721051e312131223123312431253126312731283129312a312b312c312d312e312f4478222105531421055427042106552105082106564c4d4b02210538212106391c0081e80780046a6f686e", + 4: "042004010200b7a60c26040242420c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f23102311231223132314181b1c28171615400003290349483403350222231d4a484848482a50512a632223524100034200004322602261222b634848222862482864286548482228236628226724286828692422700048482471004848361c0037001a0031183119311b311d311e311f312024221e312131223123312431253126312731283129312a312b312c312d312e312f44782522531422542b2355220823564c4d4b0222382123391c0081e80780046a6f686e2281d00f24231f880003420001892223902291922394239593a0a1a2a3a4a5a6a7a8a9aaabacadae23af3a00003b003c003d8164", + 5: "052004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f23102311231223132314181b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b53a03", 6: "06" + v6Compiled, 7: "07" + v7Compiled, - 8: "08" + v8Compiled, - 9: "09" + v9Compiled, - - 10: "10" + v10Compiled, } func pseudoOp(opcode string) bool { @@ -498,14 +444,12 @@ func TestAssemble(t *testing.T) { // This doesn't have to be a sensible program to run, it just has to compile. t.Parallel() - require.LessOrEqual(t, LogicVersion, len(nonsense)) // Allow nonsense for future versions + require.Equal(t, LogicVersion, len(nonsense)) for v := uint64(2); v <= AssemblerMaxVersion; v++ { t.Run(fmt.Sprintf("v=%d", v), func(t *testing.T) { for _, spec := range OpSpecs { - // Make sure our nonsense covers the ops. - hasOp, err := regexp.MatchString("\\s"+regexp.QuoteMeta(spec.Name)+"\\s", nonsense[v]) - require.NoError(t, err) - if !hasOp && + // Make sure our nonsense covers the ops + if !strings.Contains(nonsense[v], spec.Name) && !pseudoOp(spec.Name) && spec.Version <= v { t.Errorf("v%d nonsense test should contain op %v", v, spec.Name) } @@ -515,10 +459,7 @@ func TestAssemble(t *testing.T) { // check that compilation is stable over // time. we must assemble to the same bytes // this month that we did last month. - bytecode, ok := compiled[v] - require.True(t, ok, "Need v%d bytecode", v) - expectedBytes, _ := hex.DecodeString(bytecode) - require.NotEmpty(t, expectedBytes) + expectedBytes, _ := hex.DecodeString(compiled[v]) // the hex is for convenience if the program has been changed. the // hex string can be copy pasted back in as a new expected result. require.Equal(t, expectedBytes, ops.Program, hex.EncodeToString(ops.Program)) @@ -526,20 +467,16 @@ func TestAssemble(t *testing.T) { } } -var experiments = []uint64{pairingVersion} +var experiments = []uint64{fidoVersion, pairingVersion} // TestExperimental forces a conscious choice to promote "experimental" opcode // groups. This will fail when we increment vFuture's LogicSigVersion. If we had // intended to release the opcodes, they should have been removed from // `experiments`. func TestExperimental(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - futureV := config.Consensus[protocol.ConsensusFuture].LogicSigVersion for _, v := range experiments { - // Allows less, so we can push something out, even before vFuture has been updated. - require.LessOrEqual(t, futureV, v) + require.Equal(t, futureV, v) } } @@ -582,46 +519,20 @@ func testMatch(t testing.TB, actual, expected string) (ok bool) { } } -func assembleWithTrace(text string, ver uint64) (*OpStream, error) { +func assemblyTrace(text string, ver uint64) string { ops := newOpStream(ver) ops.Trace = &strings.Builder{} - err := ops.assemble(text) - return &ops, err -} - -func lines(s string, num int) (bool, string) { - if num < 1 { - return true, "" - } - found := 0 - for i := 0; i < len(s); i++ { - if s[i] == '\n' { - found++ - if found == num { - return true, s[0 : i+1] - } - } - } - return false, s -} - -func summarize(trace *strings.Builder) string { - truncated, msg := lines(trace.String(), 50) - if !truncated { - return msg - } - return msg + "(trace truncated)\n" + ops.assemble(text) + return ops.Trace.String() } func testProg(t testing.TB, source string, ver uint64, expected ...Expect) *OpStream { t.Helper() - ops, err := assembleWithTrace(source, ver) + program := strings.ReplaceAll(source, ";", "\n") + ops, err := AssembleStringWithVersion(program, ver) if len(expected) == 0 { if len(ops.Errors) > 0 || err != nil || ops == nil || ops.Program == nil { - t.Log(summarize(ops.Trace)) - } - if len(ops.Errors) > 10 { - ops.Errors = ops.Errors[:10] // Truncate to reasonable + t.Log(assemblyTrace(program, ver)) } require.Empty(t, ops.Errors) require.NoError(t, err) @@ -629,13 +540,13 @@ func testProg(t testing.TB, source string, ver uint64, expected ...Expect) *OpSt require.NotNil(t, ops.Program) // It should always be possible to Disassemble dis, err := Disassemble(ops.Program) - require.NoError(t, err, source) + require.NoError(t, err, program) // And, while the disassembly may not match input // exactly, the assembly of the disassembly should // give the same bytecode ops2, err := AssembleStringWithVersion(notrack(dis), ver) if len(ops2.Errors) > 0 || err != nil || ops2 == nil || ops2.Program == nil { - t.Log(source) + t.Log(program) t.Log(dis) } require.Empty(t, ops2.Errors) @@ -643,7 +554,7 @@ func testProg(t testing.TB, source string, ver uint64, expected ...Expect) *OpSt require.Equal(t, ops.Program, ops2.Program) } else { if err == nil { - t.Log(source) + t.Log(program) } require.Error(t, err) errors := ops.Errors @@ -659,7 +570,7 @@ func testProg(t testing.TB, source string, ver uint64, expected ...Expect) *OpSt } } if fail { - t.Log(summarize(ops.Trace)) + t.Log(assemblyTrace(program, ver)) t.FailNow() } } else { @@ -676,7 +587,7 @@ func testProg(t testing.TB, source string, ver uint64, expected ...Expect) *OpSt require.NotNil(t, found, "Error %s was not found on line %d", exp.s, exp.l) msg := found.Unwrap().Error() if !testMatch(t, msg, exp.s) { - t.Log(summarize(ops.Trace)) + t.Log(assemblyTrace(program, ver)) t.FailNow() } } @@ -710,47 +621,25 @@ func TestAssembleTxna(t *testing.T) { testLine(t, "gtxna 0 ApplicationArgs 256", AssemblerMaxVersion, "gtxna i beyond 255: 256") testLine(t, "gtxna 256 Accounts 0", AssemblerMaxVersion, "gtxna t beyond 255: 256") testLine(t, "gtxna 0 Sender 256", AssemblerMaxVersion, "gtxna unknown field: \"Sender\"") - testLine(t, "gtxna ApplicationArgs 0 255", AssemblerMaxVersion, "gtxna can only use \"ApplicationArgs\" as immediate 2") - testLine(t, "gtxna 0 255 ApplicationArgs", AssemblerMaxVersion, "gtxna can only use \"255\" as immediate 1 or 3") - - testLine(t, "txn Accounts 256", AssemblerMaxVersion, "txn i beyond 255: 256") - testLine(t, "txn ApplicationArgs 256", AssemblerMaxVersion, "txn i beyond 255: 256") - testLine(t, "txn 255 ApplicationArgs", AssemblerMaxVersion, "txn with 2 immediates can only use \"255\" as immediate 2") - testLine(t, "txn Sender 256", AssemblerMaxVersion, "\"Sender\" field of txn can only be used with 1 immediate") - testLine(t, "gtxn 0 Accounts 256", AssemblerMaxVersion, "gtxn i beyond 255: 256") - testLine(t, "gtxn 0 ApplicationArgs 256", AssemblerMaxVersion, "gtxn i beyond 255: 256") - testLine(t, "gtxn 256 Accounts 0", AssemblerMaxVersion, "gtxn t beyond 255: 256") - testLine(t, "gtxn 0 Sender 256", AssemblerMaxVersion, "\"Sender\" field of gtxn can only be used with 2 immediates") - testLine(t, "gtxn ApplicationArgs 0 255", AssemblerMaxVersion, "gtxn with 3 immediates can only use \"ApplicationArgs\" as immediate 2") - testLine(t, "gtxn 0 255 ApplicationArgs", AssemblerMaxVersion, "gtxn with 3 immediates can only use \"255\" as immediate 1 or 3") - - testLine(t, "txn Accounts 0", 1, "txn opcode with 2 immediates was introduced in v2") + testLine(t, "txn Accounts 0", 1, "txn expects 1 immediate argument") testLine(t, "txn Accounts 0 1", 2, "txn expects 1 or 2 immediate arguments") testLine(t, "txna Accounts 0 1", AssemblerMaxVersion, "txna expects 2 immediate arguments") - testLine(t, "txn Accounts 0 1", AssemblerMaxVersion, "txn expects 1 or 2 immediate arguments") testLine(t, "txnas Accounts 1", AssemblerMaxVersion, "txnas expects 1 immediate argument") testLine(t, "txna Accounts a", AssemblerMaxVersion, "txna unable to parse...") - testLine(t, "txn Accounts a", AssemblerMaxVersion, "txn unable to parse...") - testLine(t, "gtxn 0 Sender 0", 1, "gtxn opcode with 3 immediates was introduced in v2") + testLine(t, "gtxn 0 Sender 0", 1, "gtxn expects 2 immediate arguments") testLine(t, "gtxn 0 Sender 1 2", 2, "gtxn expects 2 or 3 immediate arguments") testLine(t, "gtxna 0 Accounts 1 2", AssemblerMaxVersion, "gtxna expects 3 immediate arguments") testLine(t, "gtxna a Accounts 0", AssemblerMaxVersion, "gtxna unable to parse...") testLine(t, "gtxna 0 Accounts a", AssemblerMaxVersion, "gtxna unable to parse...") - - testLine(t, "gtxn 0 Accounts 1 2", AssemblerMaxVersion, "gtxn expects 2 or 3 immediate arguments") - testLine(t, "gtxn a Accounts 0", AssemblerMaxVersion, "gtxn unable to parse...") - testLine(t, "gtxn 0 Accounts a", AssemblerMaxVersion, "gtxn unable to parse...") - testLine(t, "gtxnas Accounts 1 2", AssemblerMaxVersion, "gtxnas expects 2 immediate arguments") testLine(t, "txn ABC", 2, "txn unknown field: \"ABC\"") testLine(t, "gtxn 0 ABC", 2, "gtxn unknown field: \"ABC\"") testLine(t, "gtxn a ABC", 2, "gtxn unable to parse...") - // For now not going to additionally report version issue until version is only problem - testLine(t, "txn Accounts", 1, "\"Accounts\" field of txn can only be used with 2 immediates") - testLine(t, "txn Accounts", AssemblerMaxVersion, "\"Accounts\" field of txn can only be used with 2 immediates") + testLine(t, "txn Accounts", 1, "txn unknown field: \"Accounts\"") + testLine(t, "txn Accounts", AssemblerMaxVersion, "txn unknown field: \"Accounts\"") testLine(t, "txn Accounts 0", AssemblerMaxVersion, "") - testLine(t, "gtxn 0 Accounts", AssemblerMaxVersion, "\"Accounts\" field of gtxn can only be used with 3 immediates") - testLine(t, "gtxn 0 Accounts", 1, "\"Accounts\" field of gtxn can only be used with 3 immediates") + testLine(t, "gtxn 0 Accounts", AssemblerMaxVersion, "gtxn unknown field: \"Accounts\"...") + testLine(t, "gtxn 0 Accounts", 1, "gtxn unknown field: \"Accounts\"") testLine(t, "gtxn 0 Accounts 1", AssemblerMaxVersion, "") } @@ -763,9 +652,9 @@ func TestAssembleGlobal(t *testing.T) { testProg(t, "global MinTxnFee; int 2; +", AssemblerMaxVersion) testProg(t, "global ZeroAddress; byte 0x12; concat; len", AssemblerMaxVersion) testProg(t, "global MinTxnFee; byte 0x12; concat", AssemblerMaxVersion, - Expect{1, "concat arg 0 wanted type []byte..."}) + Expect{3, "concat arg 0 wanted type []byte..."}) testProg(t, "int 2; global ZeroAddress; +", AssemblerMaxVersion, - Expect{1, "+ arg 1 wanted type uint64..."}) + Expect{3, "+ arg 1 wanted type uint64..."}) } func TestAssembleDefault(t *testing.T) { @@ -1292,94 +1181,209 @@ func TestFieldsFromLine(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - check := func(line string, tokens ...string) { - t.Helper() - assert.Equal(t, tokensFromLine(line), tokens) - } - - check("op arg", "op", "arg") - check("op arg // test", "op", "arg") - check("op base64 ABC//==", "op", "base64", "ABC//==") - check("op base64 base64", "op", "base64", "base64") - check("op base64 base64 //comment", "op", "base64", "base64") - check("op base64 base64; op2 //done", "op", "base64", "base64", ";", "op2") - check("op base64 ABC/==", "op", "base64", "ABC/==") - check("op base64 ABC/== /", "op", "base64", "ABC/==", "/") - check("op base64 ABC/== //", "op", "base64", "ABC/==") - check("op base64 ABC//== //", "op", "base64", "ABC//==") - check("op b64 ABC//== //", "op", "b64", "ABC//==") - check("op b64(ABC//==) // comment", "op", "b64(ABC//==)") - check("op base64(ABC//==) // comment", "op", "base64(ABC//==)") - check("op b64(ABC/==) // comment", "op", "b64(ABC/==)") - check("op base64(ABC/==) // comment", "op", "base64(ABC/==)") - check("base64(ABC//==)", "base64(ABC//==)") - check("b(ABC//==)", "b(ABC") - check("b(ABC//==) //", "b(ABC") - check("b(ABC ==) //", "b(ABC", "==)") - check("op base64 ABC)", "op", "base64", "ABC)") - check("op base64 ABC) // comment", "op", "base64", "ABC)") - check("op base64 ABC//) // comment", "op", "base64", "ABC//)") - check(`op "test"`, "op", `"test"`) - check(`op "test1 test2"`, "op", `"test1 test2"`) - check(`op "test1 test2" // comment`, "op", `"test1 test2"`) - check(`op "test1 test2 // not a comment"`, "op", `"test1 test2 // not a comment"`) - check(`op "test1 test2 // not a comment" // comment`, "op", `"test1 test2 // not a comment"`) - check(`op "test1 test2" //`, "op", `"test1 test2"`) - check(`op "test1 test2"//`, "op", `"test1 test2"`) - check(`op "test1 test2`, "op", `"test1 test2`) // non-terminated string literal - check(`op "test1 test2\"`, "op", `"test1 test2\"`) // non-terminated string literal - check(`op \"test1 test2\"`, "op", `\"test1`, `test2\"`) // not a string literal - check(`"test1 test2"`, `"test1 test2"`) - check(`\"test1 test2"`, `\"test1`, `test2"`) - check(`"" // test`, `""`) - check("int 1; int 2", "int", "1", ";", "int", "2") - check("int 1;;;int 2", "int", "1", ";", ";", ";", "int", "2") - check("int 1; ;int 2;; ; ;; ", "int", "1", ";", ";", "int", "2", ";", ";", ";", ";", ";") - check(";", ";") - check("; ; ;;;;", ";", ";", ";", ";", ";", ";") - check(" ;", ";") - check(" ; ", ";") -} - -func TestNextStatement(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - // this test ensures nextStatement splits tokens on semicolons properly - // macro testing should be handled in TestMacros - ops := newOpStream(AssemblerMaxVersion) - check := func(tokens []string, left []string, right []string) { - t.Helper() - current, next := nextStatement(&ops, tokens) - assert.Equal(t, left, current) - assert.Equal(t, right, next) - } - - check([]string{"hey,", "how's", ";", ";", "it", "going", ";"}, - []string{"hey,", "how's"}, - []string{";", "it", "going", ";"}, - ) - - check([]string{";"}, - []string{}, - []string{}, - ) - - check([]string{";", "it", "going"}, - []string{}, - []string{"it", "going"}, - ) - - check([]string{"hey,", "how's"}, - []string{"hey,", "how's"}, - nil, - ) - - check([]string{`"hey in quotes;"`, "getting", `";"`, ";", "tricky"}, - []string{`"hey in quotes;"`, "getting", `";"`}, - []string{"tricky"}, - ) - + line := "op arg" + fields := fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "arg", fields[1]) + + line = "op arg // test" + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "arg", fields[1]) + + line = "op base64 ABC//==" + fields = fieldsFromLine(line) + require.Equal(t, 3, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "base64", fields[1]) + require.Equal(t, "ABC//==", fields[2]) + + line = "op base64 ABC/==" + fields = fieldsFromLine(line) + require.Equal(t, 3, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "base64", fields[1]) + require.Equal(t, "ABC/==", fields[2]) + + line = "op base64 ABC/== /" + fields = fieldsFromLine(line) + require.Equal(t, 4, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "base64", fields[1]) + require.Equal(t, "ABC/==", fields[2]) + require.Equal(t, "/", fields[3]) + + line = "op base64 ABC/== //" + fields = fieldsFromLine(line) + require.Equal(t, 3, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "base64", fields[1]) + require.Equal(t, "ABC/==", fields[2]) + + line = "op base64 ABC//== //" + fields = fieldsFromLine(line) + require.Equal(t, 3, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "base64", fields[1]) + require.Equal(t, "ABC//==", fields[2]) + + line = "op b64 ABC//== //" + fields = fieldsFromLine(line) + require.Equal(t, 3, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "b64", fields[1]) + require.Equal(t, "ABC//==", fields[2]) + + line = "op b64(ABC//==) // comment" + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "b64(ABC//==)", fields[1]) + + line = "op base64(ABC//==) // comment" + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "base64(ABC//==)", fields[1]) + + line = "op b64(ABC/==) // comment" + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "b64(ABC/==)", fields[1]) + + line = "op base64(ABC/==) // comment" + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "base64(ABC/==)", fields[1]) + + line = "base64(ABC//==)" + fields = fieldsFromLine(line) + require.Equal(t, 1, len(fields)) + require.Equal(t, "base64(ABC//==)", fields[0]) + + line = "b(ABC//==)" + fields = fieldsFromLine(line) + require.Equal(t, 1, len(fields)) + require.Equal(t, "b(ABC", fields[0]) + + line = "b(ABC//==) //" + fields = fieldsFromLine(line) + require.Equal(t, 1, len(fields)) + require.Equal(t, "b(ABC", fields[0]) + + line = "b(ABC ==) //" + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "b(ABC", fields[0]) + require.Equal(t, "==)", fields[1]) + + line = "op base64 ABC)" + fields = fieldsFromLine(line) + require.Equal(t, 3, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "base64", fields[1]) + require.Equal(t, "ABC)", fields[2]) + + line = "op base64 ABC) // comment" + fields = fieldsFromLine(line) + require.Equal(t, 3, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "base64", fields[1]) + require.Equal(t, "ABC)", fields[2]) + + line = "op base64 ABC//) // comment" + fields = fieldsFromLine(line) + require.Equal(t, 3, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, "base64", fields[1]) + require.Equal(t, "ABC//)", fields[2]) + + line = `op "test"` + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `"test"`, fields[1]) + + line = `op "test1 test2"` + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `"test1 test2"`, fields[1]) + + line = `op "test1 test2" // comment` + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `"test1 test2"`, fields[1]) + + line = `op "test1 test2 // not a comment"` + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `"test1 test2 // not a comment"`, fields[1]) + + line = `op "test1 test2 // not a comment" // comment` + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `"test1 test2 // not a comment"`, fields[1]) + + line = `op "test1 test2 // not a comment" // comment` + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `"test1 test2 // not a comment"`, fields[1]) + + line = `op "test1 test2" //` + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `"test1 test2"`, fields[1]) + + line = `op "test1 test2"//` + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `"test1 test2"`, fields[1]) + + line = `op "test1 test2` // non-terminated string literal + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `"test1 test2`, fields[1]) + + line = `op "test1 test2\"` // non-terminated string literal + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `"test1 test2\"`, fields[1]) + + line = `op \"test1 test2\"` // not a string literal + fields = fieldsFromLine(line) + require.Equal(t, 3, len(fields)) + require.Equal(t, "op", fields[0]) + require.Equal(t, `\"test1`, fields[1]) + require.Equal(t, `test2\"`, fields[2]) + + line = `"test1 test2"` + fields = fieldsFromLine(line) + require.Equal(t, 1, len(fields)) + require.Equal(t, `"test1 test2"`, fields[0]) + + line = `\"test1 test2"` + fields = fieldsFromLine(line) + require.Equal(t, 2, len(fields)) + require.Equal(t, `\"test1`, fields[0]) + require.Equal(t, `test2"`, fields[1]) + + line = `"" // test` + fields = fieldsFromLine(line) + require.Equal(t, 1, len(fields)) + require.Equal(t, `""`, fields[0]) } func TestAssembleRejectNegJump(t *testing.T) { @@ -1631,12 +1635,8 @@ func TestAssembleDisassembleCycle(t *testing.T) { // optimizations in later versions that change the bytecode // emitted. But currently it is, so we test it for now to // catch any suprises. - require.LessOrEqual(t, LogicVersion, len(nonsense)) // Allow nonsense for future versions + require.Equal(t, LogicVersion, len(nonsense)) for v, source := range nonsense { - v, source := v, source - if v > LogicVersion { - continue // We allow them to be set, but can't test assembly beyond LogicVersion - } t.Run(fmt.Sprintf("v=%d", v), func(t *testing.T) { t.Parallel() ops := testProg(t, source, v) @@ -1711,10 +1711,10 @@ func TestBranchArgs(t *testing.T) { for v := uint64(2); v <= AssemblerMaxVersion; v++ { testProg(t, "b", v, Expect{1, "b needs a single label argument"}) testProg(t, "b lab1 lab2", v, Expect{1, "b needs a single label argument"}) - testProg(t, "int 1; bz", v, Expect{1, "bz needs a single label argument"}) - testProg(t, "int 1; bz a b", v, Expect{1, "bz needs a single label argument"}) - testProg(t, "int 1; bnz", v, Expect{1, "bnz needs a single label argument"}) - testProg(t, "int 1; bnz c d", v, Expect{1, "bnz needs a single label argument"}) + testProg(t, "int 1; bz", v, Expect{2, "bz needs a single label argument"}) + testProg(t, "int 1; bz a b", v, Expect{2, "bz needs a single label argument"}) + testProg(t, "int 1; bnz", v, Expect{2, "bnz needs a single label argument"}) + testProg(t, "int 1; bnz c d", v, Expect{2, "bnz needs a single label argument"}) } for v := uint64(4); v <= AssemblerMaxVersion; v++ { @@ -1846,7 +1846,7 @@ func TestAssembleVersions(t *testing.T) { testLine(t, "txna Accounts 0", AssemblerMaxVersion, "") testLine(t, "txna Accounts 0", 2, "") - testLine(t, "txna Accounts 0", 1, "txna opcode was introduced in v2") + testLine(t, "txna Accounts 0", 1, "txna opcode was introduced in TEAL v2") } func TestAssembleBalance(t *testing.T) { @@ -1889,32 +1889,22 @@ func TestAssembleAsset(t *testing.T) { testProg(t, "asset_holding_get ABC 1", v, Expect{1, "asset_holding_get ABC 1 expects 2 stack arguments..."}) testProg(t, "int 1; asset_holding_get ABC 1", v, - Expect{1, "asset_holding_get ABC 1 expects 2 stack arguments..."}) - - if v < sharedResourcesVersion { - testProg(t, "int 1; int 1; asset_holding_get ABC 1", v, - Expect{1, "asset_holding_get expects 1 immediate argument"}) - testProg(t, "int 1; int 1; asset_holding_get ABC", v, - Expect{1, "asset_holding_get unknown field: \"ABC\""}) - } else { - testProg(t, "int 1; int 1; asset_holding_get ABC 1", v, - Expect{1, "asset_holding_get ABC 1 arg 0 wanted type..."}) - testProg(t, "txn Sender; int 1; asset_holding_get ABC 1", v, - Expect{1, "asset_holding_get expects 1 immediate argument"}) - testProg(t, "txn Sender; int 1; asset_holding_get ABC", v, - Expect{1, "asset_holding_get unknown field: \"ABC\""}) - } + Expect{2, "asset_holding_get ABC 1 expects 2 stack arguments..."}) + testProg(t, "int 1; int 1; asset_holding_get ABC 1", v, + Expect{3, "asset_holding_get expects 1 immediate argument"}) + testProg(t, "int 1; int 1; asset_holding_get ABC", v, + Expect{3, "asset_holding_get unknown field: \"ABC\""}) testProg(t, "byte 0x1234; asset_params_get ABC 1", v, - Expect{1, "asset_params_get ABC 1 arg 0 wanted type uint64..."}) + Expect{2, "asset_params_get ABC 1 arg 0 wanted type uint64..."}) // Test that AssetUnitName is known to return bytes testProg(t, "int 1; asset_params_get AssetUnitName; pop; int 1; +", v, - Expect{1, "+ arg 0 wanted type uint64..."}) + Expect{5, "+ arg 0 wanted type uint64..."}) // Test that AssetTotal is known to return uint64 testProg(t, "int 1; asset_params_get AssetTotal; pop; byte 0x12; concat", v, - Expect{1, "concat arg 0 wanted type []byte..."}) + Expect{5, "concat arg 0 wanted type []byte..."}) testLine(t, "asset_params_get ABC 1", v, "asset_params_get expects 1 immediate argument") testLine(t, "asset_params_get ABC", v, "asset_params_get unknown field: \"ABC\"") @@ -2047,7 +2037,7 @@ func TestDisassembleLastLabel(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - // starting from v2 branching to the last line are legal + // starting from TEAL v2 branching to the last line are legal for v := uint64(2); v <= AssemblerMaxVersion; v++ { t.Run(fmt.Sprintf("v=%d", v), func(t *testing.T) { source := fmt.Sprintf(`#pragma version %d @@ -2382,7 +2372,7 @@ int 1 ops = testProg(t, text, assemblerNoVersion) require.Equal(t, ops2.Program, ops.Program) - // check if no version it defaults to v1 + // check if no version it defaults to TEAL v1 text = `byte "test" len ` @@ -2512,92 +2502,69 @@ func TestSwapTypeCheck(t *testing.T) { t.Parallel() /* reconfirm that we detect this type error */ - testProg(t, "int 1; byte 0x1234; +", AssemblerMaxVersion, Expect{1, "+ arg 1..."}) + testProg(t, "int 1; byte 0x1234; +", AssemblerMaxVersion, Expect{3, "+ arg 1..."}) /* despite swap, we track types */ - testProg(t, "int 1; byte 0x1234; swap; +", AssemblerMaxVersion, Expect{1, "+ arg 0..."}) - testProg(t, "byte 0x1234; int 1; swap; +", AssemblerMaxVersion, Expect{1, "+ arg 1..."}) + testProg(t, "int 1; byte 0x1234; swap; +", AssemblerMaxVersion, Expect{4, "+ arg 0..."}) + testProg(t, "byte 0x1234; int 1; swap; +", AssemblerMaxVersion, Expect{4, "+ arg 1..."}) } func TestDigAsm(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "int 1; dig; +", AssemblerMaxVersion, Expect{1, "dig expects 1 immediate..."}) - testProg(t, "int 1; dig junk; +", AssemblerMaxVersion, Expect{1, "dig unable to parse..."}) + testProg(t, "int 1; dig; +", AssemblerMaxVersion, Expect{2, "dig expects 1 immediate..."}) + testProg(t, "int 1; dig junk; +", AssemblerMaxVersion, Expect{2, "dig unable to parse..."}) testProg(t, "int 1; byte 0x1234; int 2; dig 2; +", AssemblerMaxVersion) testProg(t, "byte 0x32; byte 0x1234; int 2; dig 2; +", AssemblerMaxVersion, - Expect{1, "+ arg 1..."}) + Expect{5, "+ arg 1..."}) testProg(t, "byte 0x32; byte 0x1234; int 2; dig 3; +", AssemblerMaxVersion, - Expect{1, "dig 3 expects 4..."}) + Expect{4, "dig 3 expects 4..."}) testProg(t, "int 1; byte 0x1234; int 2; dig 12; +", AssemblerMaxVersion, - Expect{1, "dig 12 expects 13..."}) + Expect{4, "dig 12 expects 13..."}) // Confirm that digging something out does not ruin our knowledge about the types in the middle testProg(t, "int 1; byte 0x1234; byte 0x1234; dig 2; dig 3; +; pop; +", AssemblerMaxVersion, - Expect{1, "+ arg 1..."}) + Expect{8, "+ arg 1..."}) testProg(t, "int 3; pushbytes \"123456\"; int 1; dig 2; substring3", AssemblerMaxVersion) } -func TestBuryAsm(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - testProg(t, "int 1; bury; +", AssemblerMaxVersion, Expect{1, "bury expects 1 immediate..."}) - testProg(t, "int 1; bury junk; +", AssemblerMaxVersion, Expect{1, "bury unable to parse..."}) - - testProg(t, "int 1; byte 0x1234; int 2; bury 1; +", AssemblerMaxVersion) // the 2 replaces the byte string - testProg(t, "int 2; int 2; byte 0x1234; bury 1; +", AssemblerMaxVersion, - Expect{1, "+ arg 1..."}) - testProg(t, "byte 0x32; byte 0x1234; int 2; bury 3; +", AssemblerMaxVersion, - Expect{1, "bury 3 expects 4..."}) - testProg(t, "int 1; byte 0x1234; int 2; bury 12; +", AssemblerMaxVersion, - Expect{1, "bury 12 expects 13..."}) - - // We do not lose track of the ints between ToS and bury index - testProg(t, "int 0; int 1; int 2; int 4; bury 3; concat", AssemblerMaxVersion, - Expect{1, "concat arg 1 wanted type []byte..."}) - - // Even when we are burying into unknown (seems repetitive, but is an easy bug) - testProg(t, "int 0; int 0; b LABEL; LABEL: int 1; int 2; int 4; bury 4; concat", AssemblerMaxVersion, - Expect{1, "concat arg 1 wanted type []byte..."}) -} - func TestEqualsTypeCheck(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "int 1; byte 0x1234; ==", AssemblerMaxVersion, Expect{1, "== arg 0..."}) - testProg(t, "int 1; byte 0x1234; !=", AssemblerMaxVersion, Expect{1, "!= arg 0..."}) - testProg(t, "byte 0x1234; int 1; ==", AssemblerMaxVersion, Expect{1, "== arg 0..."}) - testProg(t, "byte 0x1234; int 1; !=", AssemblerMaxVersion, Expect{1, "!= arg 0..."}) + testProg(t, "int 1; byte 0x1234; ==", AssemblerMaxVersion, Expect{3, "== arg 0..."}) + testProg(t, "int 1; byte 0x1234; !=", AssemblerMaxVersion, Expect{3, "!= arg 0..."}) + testProg(t, "byte 0x1234; int 1; ==", AssemblerMaxVersion, Expect{3, "== arg 0..."}) + testProg(t, "byte 0x1234; int 1; !=", AssemblerMaxVersion, Expect{3, "!= arg 0..."}) } func TestDupTypeCheck(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "byte 0x1234; dup; int 1; +", AssemblerMaxVersion, Expect{1, "+ arg 0..."}) + testProg(t, "byte 0x1234; dup; int 1; +", AssemblerMaxVersion, Expect{4, "+ arg 0..."}) testProg(t, "byte 0x1234; int 1; dup; +", AssemblerMaxVersion) - testProg(t, "byte 0x1234; int 1; dup2; +", AssemblerMaxVersion, Expect{1, "+ arg 0..."}) - testProg(t, "int 1; byte 0x1234; dup2; +", AssemblerMaxVersion, Expect{1, "+ arg 1..."}) + testProg(t, "byte 0x1234; int 1; dup2; +", AssemblerMaxVersion, Expect{4, "+ arg 0..."}) + testProg(t, "int 1; byte 0x1234; dup2; +", AssemblerMaxVersion, Expect{4, "+ arg 1..."}) - testProg(t, "byte 0x1234; int 1; dup; dig 1; len", AssemblerMaxVersion, Expect{1, "len arg 0..."}) - testProg(t, "int 1; byte 0x1234; dup; dig 1; !", AssemblerMaxVersion, Expect{1, "! arg 0..."}) + testProg(t, "byte 0x1234; int 1; dup; dig 1; len", AssemblerMaxVersion, Expect{5, "len arg 0..."}) + testProg(t, "int 1; byte 0x1234; dup; dig 1; !", AssemblerMaxVersion, Expect{5, "! arg 0..."}) - testProg(t, "byte 0x1234; int 1; dup2; dig 2; len", AssemblerMaxVersion, Expect{1, "len arg 0..."}) - testProg(t, "int 1; byte 0x1234; dup2; dig 2; !", AssemblerMaxVersion, Expect{1, "! arg 0..."}) + testProg(t, "byte 0x1234; int 1; dup2; dig 2; len", AssemblerMaxVersion, Expect{5, "len arg 0..."}) + testProg(t, "int 1; byte 0x1234; dup2; dig 2; !", AssemblerMaxVersion, Expect{5, "! arg 0..."}) } func TestSelectTypeCheck(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "int 1; int 2; int 3; select; len", AssemblerMaxVersion, Expect{1, "len arg 0..."}) - testProg(t, "byte 0x1234; byte 0x5678; int 3; select; !", AssemblerMaxVersion, Expect{1, "! arg 0..."}) + testProg(t, "int 1; int 2; int 3; select; len", AssemblerMaxVersion, Expect{5, "len arg 0..."}) + testProg(t, "byte 0x1234; byte 0x5678; int 3; select; !", AssemblerMaxVersion, Expect{5, "! arg 0..."}) } func TestSetBitTypeCheck(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "int 1; int 2; int 3; setbit; len", AssemblerMaxVersion, Expect{1, "len arg 0..."}) - testProg(t, "byte 0x1234; int 2; int 3; setbit; !", AssemblerMaxVersion, Expect{1, "! arg 0..."}) + testProg(t, "int 1; int 2; int 3; setbit; len", AssemblerMaxVersion, Expect{5, "len arg 0..."}) + testProg(t, "byte 0x1234; int 2; int 3; setbit; !", AssemblerMaxVersion, Expect{5, "! arg 0..."}) } func TestScratchTypeCheck(t *testing.T) { @@ -2606,13 +2573,13 @@ func TestScratchTypeCheck(t *testing.T) { // All scratch slots should start as uint64 testProg(t, "load 0; int 1; +", AssemblerMaxVersion) // Check load and store accurately using the scratch space - testProg(t, "byte 0x01; store 0; load 0; int 1; +", AssemblerMaxVersion, Expect{1, "+ arg 0..."}) + testProg(t, "byte 0x01; store 0; load 0; int 1; +", AssemblerMaxVersion, Expect{5, "+ arg 0..."}) // Loads should know the type it's loading if all the slots are the same type - testProg(t, "int 0; loads; btoi", AssemblerMaxVersion, Expect{1, "btoi arg 0..."}) + testProg(t, "int 0; loads; btoi", AssemblerMaxVersion, Expect{3, "btoi arg 0..."}) // Loads doesn't know the type when slot types vary testProg(t, "byte 0x01; store 0; int 1; loads; btoi", AssemblerMaxVersion) // Stores should only set slots to StackAny if they are not the same type as what is being stored - testProg(t, "byte 0x01; store 0; int 3; byte 0x01; stores; load 0; int 1; +", AssemblerMaxVersion, Expect{1, "+ arg 0..."}) + testProg(t, "byte 0x01; store 0; int 3; byte 0x01; stores; load 0; int 1; +", AssemblerMaxVersion, Expect{8, "+ arg 0..."}) // ScratchSpace should reset after hitting label in deadcode testProg(t, "byte 0x01; store 0; b label1; label1:; load 0; int 1; +", AssemblerMaxVersion) // But it should reset to StackAny not uint64 @@ -2620,32 +2587,7 @@ func TestScratchTypeCheck(t *testing.T) { // Callsubs should also reset the scratch space testProg(t, "callsub A; load 0; btoi; return; A: byte 0x01; store 0; retsub", AssemblerMaxVersion) // But the scratchspace should still be tracked after the callsub - testProg(t, "callsub A; int 1; store 0; load 0; btoi; return; A: retsub", AssemblerMaxVersion, Expect{1, "btoi arg 0..."}) -} - -// TestProtoAsm confirms that the assembler will yell at you if you are -// clearly dipping into the arguments when using `proto`. You should be using -// `frame_dig`. -func TestProtoAsm(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - testProg(t, "proto 0 0", AssemblerMaxVersion, Expect{1, "proto must be unreachable..."}) - testProg(t, notrack("proto 0 0"), AssemblerMaxVersion) - testProg(t, "b a; int 1; a: proto 0 0", AssemblerMaxVersion) // we could flag a `b` to `proto` - - testProg(t, ` - int 10 - int 20 - callsub main - int 1 - return -main: - proto 2 1 - + // This consumes the top arg. We complain. - dup; dup // Even though the dup;dup restores it, so it _evals_ fine. - retsub -`, AssemblerMaxVersion) - + testProg(t, "callsub A; int 1; store 0; load 0; btoi; return; A: retsub", AssemblerMaxVersion, Expect{5, "btoi arg 0..."}) } func TestCoverAsm(t *testing.T) { @@ -2653,10 +2595,9 @@ func TestCoverAsm(t *testing.T) { t.Parallel() testProg(t, `int 4; byte "john"; int 5; cover 2; pop; +`, AssemblerMaxVersion) testProg(t, `int 4; byte "ayush"; int 5; cover 1; pop; +`, AssemblerMaxVersion) - testProg(t, `int 4; byte "john"; int 5; cover 2; +`, AssemblerMaxVersion, Expect{1, "+ arg 1..."}) + testProg(t, `int 4; byte "john"; int 5; cover 2; +`, AssemblerMaxVersion, Expect{5, "+ arg 1..."}) - testProg(t, `int 4; cover junk`, AssemblerMaxVersion, Expect{1, "cover unable to parse n ..."}) - testProg(t, notrack(`int 4; int 5; cover 0`), AssemblerMaxVersion) + testProg(t, `int 4; cover junk`, AssemblerMaxVersion, Expect{2, "cover unable to parse n ..."}) } func TestUncoverAsm(t *testing.T) { @@ -2665,38 +2606,38 @@ func TestUncoverAsm(t *testing.T) { testProg(t, `int 4; byte "john"; int 5; uncover 2; +`, AssemblerMaxVersion) testProg(t, `int 4; byte "ayush"; int 5; uncover 1; pop; +`, AssemblerMaxVersion) testProg(t, `int 1; byte "jj"; byte "ayush"; byte "john"; int 5; uncover 4; +`, AssemblerMaxVersion) - testProg(t, `int 4; byte "ayush"; int 5; uncover 1; +`, AssemblerMaxVersion, Expect{1, "+ arg 1..."}) + testProg(t, `int 4; byte "ayush"; int 5; uncover 1; +`, AssemblerMaxVersion, Expect{5, "+ arg 1..."}) } func TestTxTypes(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "itxn_begin; itxn_field Sender", 5, Expect{1, "itxn_field Sender expects 1 stack argument..."}) - testProg(t, "itxn_begin; int 1; itxn_field Sender", 5, Expect{1, "...wanted type []byte got uint64"}) + testProg(t, "itxn_begin; itxn_field Sender", 5, Expect{2, "itxn_field Sender expects 1 stack argument..."}) + testProg(t, "itxn_begin; int 1; itxn_field Sender", 5, Expect{3, "...wanted type []byte got uint64"}) testProg(t, "itxn_begin; byte 0x56127823; itxn_field Sender", 5) - testProg(t, "itxn_begin; itxn_field Amount", 5, Expect{1, "itxn_field Amount expects 1 stack argument..."}) - testProg(t, "itxn_begin; byte 0x87123376; itxn_field Amount", 5, Expect{1, "...wanted type uint64 got []byte"}) + testProg(t, "itxn_begin; itxn_field Amount", 5, Expect{2, "itxn_field Amount expects 1 stack argument..."}) + testProg(t, "itxn_begin; byte 0x87123376; itxn_field Amount", 5, Expect{3, "...wanted type uint64 got []byte"}) testProg(t, "itxn_begin; int 1; itxn_field Amount", 5) } func TestBadInnerFields(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "itxn_begin; int 1000; itxn_field FirstValid", 5, Expect{1, "...is not allowed."}) - testProg(t, "itxn_begin; int 1000; itxn_field FirstValidTime", 5, Expect{1, "...is not allowed."}) - testProg(t, "itxn_begin; int 1000; itxn_field LastValid", 5, Expect{1, "...is not allowed."}) - testProg(t, "itxn_begin; int 32; bzero; itxn_field Lease", 5, Expect{1, "...is not allowed."}) - testProg(t, "itxn_begin; byte 0x7263; itxn_field Note", 5, Expect{1, "...Note field was introduced in v6..."}) - testProg(t, "itxn_begin; byte 0x7263; itxn_field VotePK", 5, Expect{1, "...VotePK field was introduced in v6..."}) - testProg(t, "itxn_begin; int 32; bzero; itxn_field TxID", 5, Expect{1, "...is not allowed."}) - - testProg(t, "itxn_begin; int 1000; itxn_field FirstValid", 6, Expect{1, "...is not allowed."}) - testProg(t, "itxn_begin; int 1000; itxn_field LastValid", 6, Expect{1, "...is not allowed."}) - testProg(t, "itxn_begin; int 32; bzero; itxn_field Lease", 6, Expect{1, "...is not allowed."}) + testProg(t, "itxn_begin; int 1000; itxn_field FirstValid", 5, Expect{3, "...is not allowed."}) + testProg(t, "itxn_begin; int 1000; itxn_field FirstValidTime", 5, Expect{3, "...is not allowed."}) + testProg(t, "itxn_begin; int 1000; itxn_field LastValid", 5, Expect{3, "...is not allowed."}) + testProg(t, "itxn_begin; int 32; bzero; itxn_field Lease", 5, Expect{4, "...is not allowed."}) + testProg(t, "itxn_begin; byte 0x7263; itxn_field Note", 5, Expect{3, "...Note field was introduced in TEAL v6..."}) + testProg(t, "itxn_begin; byte 0x7263; itxn_field VotePK", 5, Expect{3, "...VotePK field was introduced in TEAL v6..."}) + testProg(t, "itxn_begin; int 32; bzero; itxn_field TxID", 5, Expect{4, "...is not allowed."}) + + testProg(t, "itxn_begin; int 1000; itxn_field FirstValid", 6, Expect{3, "...is not allowed."}) + testProg(t, "itxn_begin; int 1000; itxn_field LastValid", 6, Expect{3, "...is not allowed."}) + testProg(t, "itxn_begin; int 32; bzero; itxn_field Lease", 6, Expect{4, "...is not allowed."}) testProg(t, "itxn_begin; byte 0x7263; itxn_field Note", 6) testProg(t, "itxn_begin; byte 0x7263; itxn_field VotePK", 6) - testProg(t, "itxn_begin; int 32; bzero; itxn_field TxID", 6, Expect{1, "...is not allowed."}) + testProg(t, "itxn_begin; int 32; bzero; itxn_field TxID", 6, Expect{4, "...is not allowed."}) } func TestTypeTracking(t *testing.T) { @@ -2712,7 +2653,7 @@ func TestTypeTracking(t *testing.T) { // but we do want to ensure we're not just treating the code after callsub as dead testProg(t, "callsub A; int 1; concat; return; A: int 1; int 2; retsub", LogicVersion, - Expect{1, "concat arg 1 wanted..."}) + Expect{3, "concat arg 1 wanted..."}) // retsub deadens code, like any unconditional branch testProg(t, "callsub A; +; return; A: int 1; int 2; retsub; concat", LogicVersion) @@ -2784,832 +2725,3 @@ done: concat `, LogicVersion, Expect{5, "concat arg 1 wanted type []byte..."}) } - -func TestMergeProtos(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - iVi := OpSpec{Proto: proto("i:i")} - bVb := OpSpec{Proto: proto("b:b")} - aaVa := OpSpec{Proto: proto("aa:a")} - aVaa := OpSpec{Proto: proto("a:aa")} - p, _, _ := mergeProtos(map[int]OpSpec{0: iVi, 1: bVb}) - require.Equal(t, proto("a:a"), p) - _, _, ok := mergeProtos(map[int]OpSpec{0: aaVa, 1: iVi}) - require.False(t, ok) - _, _, ok = mergeProtos(map[int]OpSpec{0: aVaa, 1: iVi}) - require.False(t, ok) - medley := OpSpec{Proto: proto("aibibabai:aibibabai")} - medley2 := OpSpec{Proto: proto("biabbaiia:biabbaiia")} - p, _, _ = mergeProtos(map[int]OpSpec{0: medley, 1: medley2}) - require.Equal(t, proto("aiaabaaaa:aiaabaaaa"), p) - v1 := OpSpec{Version: 1, Proto: proto(":")} - v2 := OpSpec{Version: 2, Proto: proto(":")} - _, v, _ := mergeProtos(map[int]OpSpec{0: v2, 1: v1}) - require.Equal(t, uint64(1), v) -} - -// Extra tests for features of getSpec that are currently not tested elsewhere -func TestGetSpec(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - ops := testProg(t, "int 1", AssemblerMaxVersion) - ops.versionedPseudoOps["dummyPseudo"] = make(map[int]OpSpec) - ops.versionedPseudoOps["dummyPseudo"][1] = OpSpec{Name: "b:", Version: AssemblerMaxVersion, Proto: proto("b:")} - ops.versionedPseudoOps["dummyPseudo"][2] = OpSpec{Name: ":", Version: AssemblerMaxVersion} - _, _, ok := getSpec(ops, "dummyPseudo", []string{}) - require.False(t, ok) - _, _, ok = getSpec(ops, "nonsense", []string{}) - require.False(t, ok) - require.Equal(t, 2, len(ops.Errors)) - require.Equal(t, "unknown opcode: nonsense", ops.Errors[1].Err.Error()) -} - -func TestAddPseudoDocTags(t *testing.T) { //nolint:paralleltest // Not parallel because it modifies pseudoOps and opDocByName which are global maps - partitiontest.PartitionTest(t) - defer func() { - delete(pseudoOps, "tests") - delete(opDocByName, "multiple") - delete(opDocByName, "single") - delete(opDocByName, "none") - delete(opDocByName, "any") - }() - - pseudoOps["tests"] = map[int]OpSpec{2: {Name: "multiple"}, 1: {Name: "single"}, 0: {Name: "none"}, anyImmediates: {Name: "any"}} - addPseudoDocTags() - require.Equal(t, "`multiple` can be called using `tests` with 2 immediates.", opDocByName["multiple"]) - require.Equal(t, "`single` can be called using `tests` with 1 immediate.", opDocByName["single"]) - require.Equal(t, "`none` can be called using `tests` with no immediates.", opDocByName["none"]) - require.Equal(t, "", opDocByName["any"]) -} -func TestReplacePseudo(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - replaceVersion := 7 - for v := uint64(replaceVersion); v <= AssemblerMaxVersion; v++ { - testProg(t, "byte 0x0000; byte 0x1234; replace 0", v) - testProg(t, "byte 0x0000; int 0; byte 0x1234; replace", v) - testProg(t, "byte 0x0000; byte 0x1234; replace", v, Expect{1, "replace without immediates expects 3 stack arguments but stack height is 2"}) - testProg(t, "byte 0x0000; int 0; byte 0x1234; replace 0", v, Expect{1, "replace 0 arg 0 wanted type []byte got uint64"}) - } -} - -func checkSame(t *testing.T, version uint64, first string, compares ...string) { - t.Helper() - if version == 0 { - version = assemblerNoVersion - } - ops := testProg(t, first, version) - for _, compare := range compares { - other := testProg(t, compare, version) - if bytes.Compare(other.Program, ops.Program) != 0 { - t.Log(Disassemble(ops.Program)) - t.Log(Disassemble(other.Program)) - } - assert.Equal(t, ops.Program, other.Program, "%s unlike %s", first, compare) - } -} - -func TestSemiColon(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - checkSame(t, AssemblerMaxVersion, - "pushint 0 ; pushint 1 ; +; int 3 ; *", - "pushint 0\npushint 1\n+\nint 3\n*", - "pushint 0; pushint 1; +; int 3; *; // comment; int 2", - "pushint 0; ; ; pushint 1 ; +; int 3 ; *//check", - ) - - checkSame(t, 0, - "#pragma version 7\nint 1", - "// junk;\n#pragma version 7\nint 1", - "// junk;\n #pragma version 7\nint 1", - ) - - checkSame(t, AssemblerMaxVersion, - `byte "test;this"; pop;`, - `byte "test;this"; ; pop;`, - `byte "test;this";;;pop;`, - ) -} - -func TestAssembleSwitch(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - // fail when target doesn't correspond to existing label - source := ` - pushint 1 - switch label1 label2 - label1: - ` - testProg(t, source, AssemblerMaxVersion, NewExpect(3, "reference to undefined label \"label2\"")) - - // fail when target index != uint64 - testProg(t, ` - byte "fail" - switch label1 - labe11: - `, AssemblerMaxVersion, Expect{3, "switch label1 arg 0 wanted type uint64..."}) - - // No labels is pretty degenerate, but ok, I suppose. It's just a no-op - testProg(t, ` -int 0 -switch -int 1 -`, AssemblerMaxVersion) - - // confirm arg limit - source = ` - pushint 1 - switch label1 label2 - label1: - label2: - ` - ops := testProg(t, source, AssemblerMaxVersion) - require.Len(t, ops.Program, 9) // ver (1) + pushint (2) + opcode (1) + length (1) + labels (2*2) - - var labels []string - for i := 0; i < 255; i++ { - labels = append(labels, fmt.Sprintf("label%d", i)) - } - - // test that 255 labels is ok - source = fmt.Sprintf(` - pushint 1 - switch %s - %s - `, strings.Join(labels, " "), strings.Join(labels, ":\n")+":\n") - ops = testProg(t, source, AssemblerMaxVersion) - require.Len(t, ops.Program, 515) // ver (1) + pushint (2) + opcode (1) + length (1) + labels (2*255) - - // 256 is too many - source = fmt.Sprintf(` - pushint 1 - switch %s extra - %s - `, strings.Join(labels, " "), strings.Join(labels, ":\n")+":\n") - testProg(t, source, AssemblerMaxVersion, Expect{3, "switch cannot take more than 255 labels"}) - - // allow duplicate label reference - source = ` - pushint 1 - switch label1 label1 - label1: - ` - testProg(t, source, AssemblerMaxVersion) -} - -func TestMacros(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - checkSame(t, AssemblerMaxVersion, ` - pushint 0; pushint 1; +`, ` - #define none 0 - #define one 1 - pushint none; pushint one; +`, - ) - - checkSame(t, AssemblerMaxVersion, ` - pushint 1 - pushint 2 - == - bnz label1 - err - label1: - pushint 1`, ` - #define ==? ==; bnz - pushint 1; pushint 2; ==? label1 - err - label1: - pushint 1`, - ) - - // Test redefining macros with macro chaining works - checkSame(t, AssemblerMaxVersion, ` - pushbytes 0x100000000000; substring 3 5; substring 0 1`, ` - #define rowSize 3 - #define columnSize 5 - #define tableDimensions rowSize columnSize - pushbytes 0x100000000000; substring tableDimensions - #define rowSize 0 - #define columnSize 1 - substring tableDimensions`, - ) - - // Test more complicated macros like multi-token - checkSame(t, AssemblerMaxVersion, ` - int 3 - store 0 - int 4 - store 1 - load 0 - load 1 - <`, ` - #define &x 0 - #define x load &x; - #define &y 1 - #define y load &y; - #define -> ; store - int 3 -> &x; int 4 -> &y - x y <`, - ) - - checkSame(t, AssemblerMaxVersion, ` - pushbytes 0xddf2554d - txna ApplicationArgs 0 - == - bnz kickstart - pushbytes 0x903f4535 - txna ApplicationArgs 0 - == - bnz portal_transfer - kickstart: - pushint 1 - portal_transfer: - pushint 1 - `, ` - #define abi-route txna ApplicationArgs 0; ==; bnz - method "kickstart(account)void"; abi-route kickstart - method "portal_transfer(byte[])byte[]"; abi-route portal_transfer - kickstart: - pushint 1 - portal_transfer: - pushint 1 - `) - - checkSame(t, AssemblerMaxVersion, ` -method "echo(string)string" -txn ApplicationArgs 0 -== -bnz echo - -echo: - int 1 - dup - txnas ApplicationArgs - extract 2 0 - stores - - int 1 - loads - dup - len - itob - extract 6 0 - swap - concat - - pushbytes 0x151f7c75 - swap - concat - log - int 1 - return - -method "add(uint32,uint32)uint32" -txn ApplicationArgs 0 -== -bnz add - -add: - int 1 - dup - txnas ApplicationArgs - int 0 - extract_uint32 - stores - - int 2 - dup - txnas ApplicationArgs - int 0 - extract_uint32 - stores - - load 1; load 2; + - store 255 - - int 255 - loads - itob - extract 4 0 - pushbytes 0x151f7c75 - swap - concat - - log - int 1 - return - `, ` -// Library Methods - -// codecs -#define abi-encode-uint16 ;itob; extract 6 0; -#define abi-decode-uint16 ;extract_uint16; - -#define abi-decode-uint32 ;int 0; extract_uint32; -#define abi-encode-uint32 ;itob;extract 4 0; - -#define abi-encode-bytes ;dup; len; abi-encode-uint16; swap; concat; -#define abi-decode-bytes ;extract 2 0; - -// abi method handling -#define abi-route ;txna ApplicationArgs 0; ==; bnz -#define abi-return ;pushbytes 0x151f7c75; swap; concat; log; int 1; return; - -// stanza: "set $var from-{type}" -#define parse ; int -#define read_arg ;dup; txnas ApplicationArgs; -#define from-string ;read_arg; abi-decode-bytes; stores; -#define from-uint16 ;read_arg; abi-decode-uint16; stores; -#define from-uint32 ;read_arg; abi-decode-uint32; stores; - -// stanza: "reply $var as-{type} -#define returns ; int -#define as-uint32; loads; abi-encode-uint32; abi-return; -#define as-string; loads; abi-encode-bytes; abi-return; - -// Contract - -// echo handler -method "echo(string)string"; abi-route echo -echo: - #define msg 1 - parse msg from-string - - // cool things happen ... - - returns msg as-string - - -// add handler -method "add(uint32,uint32)uint32"; abi-route add -add: - #define x 1 - parse x from-uint32 - - #define y 2 - parse y from-uint32 - - #define sum 255 - load x; load y; +; store sum - - returns sum as-uint32 - `) - - testProg(t, ` - #define x a d - #define d c a - #define hey wat's up x - #define c woah hey - int 1 - c`, - AssemblerMaxVersion, Expect{5, "Macro cycle discovered: c -> hey -> x -> d -> c"}, Expect{7, "unknown opcode: c"}, - ) - - testProg(t, ` - #define c + - #define x a c - #define d x - #define c d - int 1 - c`, - AssemblerMaxVersion, Expect{5, "Macro cycle discovered: c -> d -> x -> c"}, Expect{7, "+ expects..."}, - ) - - testProg(t, ` - #define X X - int 3`, - AssemblerMaxVersion, Expect{2, "Macro cycle discovered: X -> X"}, - ) - - // Check that macros names can't be things like named constants, opcodes, etc. - // If pragma is given, only macros that violate that version's stuff should be errored on - testProg(t, ` - #define return random - #define pay randomm - #define NoOp randommm - #define + randommmm - #pragma version 1 // now the versioned check should activate and check all previous macros - #define return hi // no error b/c return is after v1 - #define + hey // since versioned check is now online, we can error here - int 1`, - assemblerNoVersion, - Expect{3, "Named constants..."}, - Expect{4, "Named constants..."}, - Expect{6, "Macro names cannot be opcodes: +"}, - Expect{8, "Macro names cannot be opcodes: +"}, - ) - - // Same check, but this time since no version is given, the versioned check - // uses AssemblerDefaultVersion and activates on first instruction (int 1) - testProg(t, ` - #define return random - #define pay randomm - #define NoOp randommm - #define + randommmm - int 1 // versioned check activates here - #define return hi - #define + hey`, - assemblerNoVersion, - Expect{3, "Named constants..."}, - Expect{4, "Named constants..."}, - Expect{6, "Macro names cannot be opcodes: +"}, - Expect{8, "Macro names cannot be opcodes: +"}, - ) - - testProg(t, ` - #define Sender hello - #define ApplicationArgs hiya - #pragma version 1 - #define Sender helllooooo - #define ApplicationArgs heyyyyy // no error b/c ApplicationArgs is after v1 - int 1`, - assemblerNoVersion, - Expect{4, "Macro names cannot be field names: Sender"}, // error happens once version is known - ) - - // Same check but defaults to AssemblerDefaultVersion instead of pragma - testProg(t, ` - #define Sender hello - #define ApplicationArgs hiya - int 1 - #define Sender helllooooo - #define ApplicationArgs heyyyyy`, - assemblerNoVersion, - Expect{4, "Macro names cannot be field names: Sender"}, // error happens once version is auto-set - Expect{5, "Macro names cannot be field names: Sender"}, // and on following line - ) - // define needs name and body - testLine(t, "#define", AssemblerMaxVersion, "define directive requires a name and body") - testLine(t, "#define hello", AssemblerMaxVersion, "define directive requires a name and body") - // macro names cannot be directives - testLine(t, "#define #define 1", AssemblerMaxVersion, "# character not allowed in macro name") - testLine(t, "#define #pragma 1", AssemblerMaxVersion, "# character not allowed in macro name") - // macro names cannot begin with digits (including negative ones) - testLine(t, "#define 1hello one", AssemblerMaxVersion, "Cannot begin macro name with number: 1hello") - testLine(t, "#define -1hello negativeOne", AssemblerMaxVersion, "Cannot begin macro name with number: -1hello") - // macro names can't use base64/32 notation - testLine(t, "#define b64 AA", AssemblerMaxVersion, "Cannot use b64 as macro name") - testLine(t, "#define base64 AA", AssemblerMaxVersion, "Cannot use base64 as macro name") - testLine(t, "#define b32 AA", AssemblerMaxVersion, "Cannot use b32 as macro name") - testLine(t, "#define base32 AA", AssemblerMaxVersion, "Cannot use base32 as macro name") - // macro names can't use non-alphanumeric characters that aren't specifically allowed - testLine(t, "#define wh@t 1", AssemblerMaxVersion, "@ character not allowed in macro name") - // check both kinds of pseudo-ops to make sure they can't be used as macro names - testLine(t, "#define int 3", AssemblerMaxVersion, "Macro names cannot be pseudo-ops: int") - testLine(t, "#define extract 3", AssemblerMaxVersion, "Macro names cannot be pseudo-ops: extract") - // check labels to make sure they can't be used as macro names - testProg(t, ` - coolLabel: - int 1 - #define coolLabel 1`, - AssemblerMaxVersion, - Expect{4, "Labels cannot be used as macro names: coolLabel"}, - ) - testProg(t, ` - #define coolLabel 1 - coolLabel: - int 1`, - AssemblerMaxVersion, - Expect{3, "Cannot create label with same name as macro: coolLabel"}, - ) - // Admittedly these two tests are just for coverage - ops := newOpStream(AssemblerMaxVersion) - err := define(&ops, []string{"not#define"}) - require.EqualError(t, err, "0: invalid syntax: not#define") - err = pragma(&ops, []string{"not#pragma"}) - require.EqualError(t, err, "0: invalid syntax: not#pragma") -} - -func TestAssembleImmediateRanges(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - /* Perhaps all of these "unable to parse" errors could be improved to - discuss limits rather than bailout when the immediate is, in fact, an - integer. */ - - testProg(t, "int 1; store 0;", AssemblerMaxVersion) - testProg(t, "load 255;", AssemblerMaxVersion) - - testProg(t, "int 1; store -1000;", AssemblerMaxVersion, - Expect{1, "store unable to parse..."}) - testProg(t, "load -100;", AssemblerMaxVersion, - Expect{1, "load unable to parse..."}) - testProg(t, "int 1; store 256;", AssemblerMaxVersion, - Expect{1, "store i beyond 255: 256"}) - - testProg(t, "frame_dig -1;", AssemblerMaxVersion) - testProg(t, "frame_dig 127;", AssemblerMaxVersion) - testProg(t, "int 1; frame_bury -128;", AssemblerMaxVersion) - - testProg(t, "frame_dig 128;", AssemblerMaxVersion, - Expect{1, "frame_dig unable to parse..."}) - testProg(t, "int 1; frame_bury -129;", AssemblerMaxVersion, - Expect{1, "frame_bury unable to parse..."}) -} - -func TestAssembleMatch(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - // fail when target doesn't correspond to existing label - source := ` - pushints 1 1 1 - match label1 label2 - label1: - ` - testProg(t, source, AssemblerMaxVersion, NewExpect(3, "reference to undefined label \"label2\"")) - - // No labels is pretty degenerate, but ok, I suppose. It's just a no-op - testProg(t, ` -int 0 -match -int 1 -`, AssemblerMaxVersion) - - // confirm arg limit - source = ` - pushints 1 2 1 - match label1 label2 - label1: - label2: - ` - ops := testProg(t, source, AssemblerMaxVersion) - require.Len(t, ops.Program, 12) // ver (1) + pushints (5) + opcode (1) + length (1) + labels (2*2) - - // confirm byte array args are assembled successfully - source = ` - pushbytess "1" "2" "1" - match label1 label2 - label1: - label2: - ` - testProg(t, source, AssemblerMaxVersion) - - var labels []string - for i := 0; i < 255; i++ { - labels = append(labels, fmt.Sprintf("label%d", i)) - } - - // test that 255 labels is ok - source = fmt.Sprintf(` - pushint 1 - match %s - %s - `, strings.Join(labels, " "), strings.Join(labels, ":\n")+":\n") - ops = testProg(t, source, AssemblerMaxVersion) - require.Len(t, ops.Program, 515) // ver (1) + pushint (2) + opcode (1) + length (1) + labels (2*255) - - // 256 is too many - source = fmt.Sprintf(` - pushint 1 - match %s extra - %s - `, strings.Join(labels, " "), strings.Join(labels, ":\n")+":\n") - testProg(t, source, AssemblerMaxVersion, Expect{3, "match cannot take more than 255 labels"}) - - // allow duplicate label reference - source = ` - pushint 1 - match label1 label1 - label1: - ` - testProg(t, source, AssemblerMaxVersion) -} - -func TestAssemblePushConsts(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - // allow empty const int list - source := `pushints` - testProg(t, source, AssemblerMaxVersion) - - // allow empty const bytes list - source = `pushbytess` - testProg(t, source, AssemblerMaxVersion) - - // basic test - source = `pushints 1 2 3` - ops := testProg(t, source, AssemblerMaxVersion) - require.Len(t, ops.Program, 6) // ver (1) + pushints (5) - source = `pushbytess "1" "2" "33"` - ops = testProg(t, source, AssemblerMaxVersion) - require.Len(t, ops.Program, 10) // ver (1) + pushbytess (9) - - // 256 increases size of encoded length to two bytes - valsStr := make([]string, 256) - for i := range valsStr { - valsStr[i] = fmt.Sprintf("%d", 1) - } - source = fmt.Sprintf(`pushints %s`, strings.Join(valsStr, " ")) - ops = testProg(t, source, AssemblerMaxVersion) - require.Len(t, ops.Program, 260) // ver (1) + opcode (1) + len (2) + ints (256) - - for i := range valsStr { - valsStr[i] = fmt.Sprintf("\"%d\"", 1) - } - source = fmt.Sprintf(`pushbytess %s`, strings.Join(valsStr, " ")) - ops = testProg(t, source, AssemblerMaxVersion) - require.Len(t, ops.Program, 516) // ver (1) + opcode (1) + len (2) + bytess (512) - - // enforce correct types - source = `pushints "1" "2" "3"` - testProg(t, source, AssemblerMaxVersion, Expect{1, `strconv.ParseUint: parsing "\"1\"": invalid syntax`}) - source = `pushbytess 1 2 3` - testProg(t, source, AssemblerMaxVersion, Expect{1, "byte arg did not parse: 1"}) - source = `pushints 6 4; concat` - testProg(t, source, AssemblerMaxVersion, Expect{1, "concat arg 1 wanted type []byte got uint64"}) - source = `pushbytess "x" "y"; +` - testProg(t, source, AssemblerMaxVersion, Expect{1, "+ arg 1 wanted type uint64 got []byte"}) -} - -func TestAssembleEmpty(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - emptyExpect := Expect{0, "Cannot assemble empty program text"} - emptyPrograms := []string{ - "", - " ", - " \n\t\t\t\n\n ", - " \n \t \t \t \n \n \n\n", - } - - nonEmpty := " \n \t \t \t int 1 \n \n \t \t \n\n" - - for version := uint64(1); version <= AssemblerMaxVersion; version++ { - for _, prog := range emptyPrograms { - testProg(t, prog, version, emptyExpect) - } - testProg(t, nonEmpty, version) - } -} - -func TestReportMultipleErrors(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - assertWithMsg := func(t *testing.T, expectedOutput string, b bytes.Buffer) { - if b.String() != expectedOutput { - t.Errorf("Unexpected output: got %q, want %q", b.String(), expectedOutput) - } - } - - ops := &OpStream{ - Errors: []lineError{ - {Line: 1, Err: errors.New("error 1")}, - {Err: errors.New("error 2")}, - {Line: 3, Err: errors.New("error 3")}, - }, - Warnings: []error{ - errors.New("warning 1"), - errors.New("warning 2"), - }, - } - - // Test the case where fname is not empty - var b bytes.Buffer - ops.ReportMultipleErrors("test.txt", &b) - expected := `test.txt: 1: error 1 -test.txt: 0: error 2 -test.txt: 3: error 3 -test.txt: warning 1 -test.txt: warning 2 -` - assertWithMsg(t, expected, b) - - // Test the case where fname is empty - b.Reset() - ops.ReportMultipleErrors("", &b) - expected = `1: error 1 -0: error 2 -3: error 3 -warning 1 -warning 2 -` - assertWithMsg(t, expected, b) - - // no errors or warnings at all - ops = &OpStream{} - b.Reset() - ops.ReportMultipleErrors("blah blah", &b) - expected = "" - assertWithMsg(t, expected, b) - - // more than 10 errors: - file := "great-file.go" - les := []lineError{} - expectedStrs := []string{} - for i := 1; i <= 11; i++ { - errS := fmt.Errorf("error %d", i) - les = append(les, lineError{i, errS}) - if i <= 10 { - expectedStrs = append(expectedStrs, fmt.Sprintf("%s: %d: %s", file, i, errS)) - } - } - expected = strings.Join(expectedStrs, "\n") + "\n" - ops = &OpStream{Errors: les} - b.Reset() - ops.ReportMultipleErrors(file, &b) - assertWithMsg(t, expected, b) - - // exactly 1 error + filename - ops = &OpStream{Errors: []lineError{{42, errors.New("super annoying error")}}} - b.Reset() - ops.ReportMultipleErrors("galaxy.py", &b) - expected = "galaxy.py: 1 error: 42: super annoying error\n" - assertWithMsg(t, expected, b) - - // exactly 1 error w/o filename - ops = &OpStream{Errors: []lineError{{42, errors.New("super annoying error")}}} - b.Reset() - ops.ReportMultipleErrors("", &b) - expected = "1 error: 42: super annoying error\n" - assertWithMsg(t, expected, b) -} - -// TestDisassembleBadBranch ensures a clean error when a branch has no target. -func TestDisassembleBadBranch(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - for _, br := range []byte{0x40, 0x41, 0x42} { - dis, err := Disassemble([]byte{2, br}) - require.Error(t, err, dis) - dis, err = Disassemble([]byte{2, br, 0x01}) - require.Error(t, err, dis) - - // It would be reasonable to error here, since it's a jump past the end. - dis, err = Disassemble([]byte{2, br, 0x00, 0x05}) - require.NoError(t, err, dis) - - // It would be reasonable to error here, since it's a back jump in v2. - dis, err = Disassemble([]byte{2, br, 0xff, 0x02}) - require.NoError(t, err, dis) - - dis, err = Disassemble([]byte{2, br, 0x00, 0x01, 0x00}) - require.NoError(t, err) - } -} - -// TestDisassembleBadSwitch ensures a clean error when a switch ends early -func TestDisassembleBadSwitch(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - source := ` - int 1 - switch label1 label2 - label1: - label2: - ` - ops, err := AssembleStringWithVersion(source, AssemblerMaxVersion) - require.NoError(t, err) - - dis, err := Disassemble(ops.Program) - require.NoError(t, err, dis) - - // chop off all the labels, but keep the label count - dis, err = Disassemble(ops.Program[:len(ops.Program)-4]) - require.ErrorContains(t, err, "could not decode labels for switch", dis) - - // chop off before the label count - dis, err = Disassemble(ops.Program[:len(ops.Program)-5]) - require.ErrorContains(t, err, "could not decode label count for switch", dis) - - // chop off half of a label - dis, err = Disassemble(ops.Program[:len(ops.Program)-1]) - require.ErrorContains(t, err, "could not decode labels for switch", dis) -} - -// TestDisassembleBadMatch ensures a clean error when a match ends early -func TestDisassembleBadMatch(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - source := ` - int 40 - int 45 - int 40 - match label1 label2 - label1: - label2: - ` - ops, err := AssembleStringWithVersion(source, AssemblerMaxVersion) - require.NoError(t, err) - - dis, err := Disassemble(ops.Program) - require.NoError(t, err, dis) - - // return the label count, but chop off the labels themselves - dis, err = Disassemble(ops.Program[:len(ops.Program)-5]) - require.ErrorContains(t, err, "could not decode label count for match", dis) - - dis, err = Disassemble(ops.Program[:len(ops.Program)-1]) - require.ErrorContains(t, err, "could not decode labels for match", dis) -} diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index a8c245a488..d1e5b2bda6 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -453,7 +453,7 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { func (g *generator) WriteDeltas(output io.Writer, round uint64) error { delta, err := g.ledger.GetStateDeltaForRound(basics.Round(round)) if err != nil { - return fmt.Errorf("err getting state delta for round %d, %v", round, err) + return fmt.Errorf("err getting state delta for round %d: %w", round, err) } // msgp encode deltas data, err := encode(protocol.CodecHandle, delta) diff --git a/tools/block-generator/generator/utils.go b/tools/block-generator/generator/utils.go index 59ff7f2e06..5f048a0253 100644 --- a/tools/block-generator/generator/utils.go +++ b/tools/block-generator/generator/utils.go @@ -75,7 +75,7 @@ func encode(handle codec.Handle, obj interface{}) ([]byte, error) { err := enc.Encode(obj) if err != nil { - return nil, fmt.Errorf("failed to encode object: %v", err) + return nil, fmt.Errorf("failed to encode object: %w", err) } return output, nil } diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index 3631d3bbb1..2792c94be7 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -19,6 +19,7 @@ package runner import ( "bytes" "context" + // embed conduit template config file _ "embed" "encoding/json" @@ -140,7 +141,7 @@ func (r *Args) run() error { // create config file in the right data directory f, err := os.Create(path.Join(dataDir, "conduit.yml")) if err != nil { - return fmt.Errorf("problem creating conduit.yml: %v", err) + return fmt.Errorf("problem creating conduit.yml: %w", err) } defer f.Close() @@ -151,7 +152,7 @@ func (r *Args) run() error { err = t.Execute(f, conduitConfig) if err != nil { - return fmt.Errorf("problem executing template file: %v", err) + return fmt.Errorf("problem executing template file: %w", err) } // Start conduit From 3ac682e9ab1511dd48fa04378f1a4fe02b8c54a2 Mon Sep 17 00:00:00 2001 From: Zeph Grunschlag Date: Fri, 21 Apr 2023 12:01:44 -0500 Subject: [PATCH 14/14] fix bad merge --- data/transactions/logic/assembler_test.go | 1504 ++++++++++++++++----- 1 file changed, 1196 insertions(+), 308 deletions(-) diff --git a/data/transactions/logic/assembler_test.go b/data/transactions/logic/assembler_test.go index 2a63b75123..7c515c6931 100644 --- a/data/transactions/logic/assembler_test.go +++ b/data/transactions/logic/assembler_test.go @@ -17,11 +17,15 @@ package logic import ( + "bytes" "encoding/hex" + "errors" "fmt" + "regexp" "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/algorand/go-algorand/config" @@ -30,7 +34,8 @@ import ( ) // used by TestAssemble and others, see UPDATE PROCEDURE in TestAssemble() -const v1Nonsense = `err +const v1Nonsense = ` +err global MinTxnFee global MinBalance global MaxTxnLife @@ -116,6 +121,8 @@ intc 1 intc 1 ! % +| +& ^ ~ byte 0x4242 @@ -341,6 +348,7 @@ byte 0x0123456789abcd dup dup ecdsa_pk_recover Secp256k1 +itxn Sender itxna Logs 3 ` @@ -372,6 +380,12 @@ const boxNonsense = ` box_get ` +const randomnessNonsense = ` +pushint 0xffff +block BlkTimestamp +vrf_verify VrfAlgorand +` + const v7Nonsense = v6Nonsense + ` base64_decode URLEncoding json_ref JSONUint64 @@ -387,38 +401,78 @@ pushbytes 0x012345 dup dup ed25519verify_bare +` + randomnessNonsense + ` pushbytes 0x4321 pushbytes 0x77 replace2 2 pushbytes 0x88 pushint 1 replace3 -` + boxNonsense + pairingNonsense +` -const v6Compiled = "2004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f23102311231223132314181b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b53a03b6b7043cb8033a0c2349c42a9631007300810881088120978101c53a8101c6003a" +const switchNonsense = ` +switch_label0: +pushint 1 +switch switch_label0 switch_label1 +switch_label1: +pushint 1 +` -const v7Compiled = v6Compiled + "5e005f018120af060180070123456789abcd49490501988003012345494984800243218001775c0280018881015d" + boxCompiled + pairingCompiled +const matchNonsense = ` +match_label0: +pushints 1 2 1 +match match_label0 match_label1 +match_label1: +pushbytess "1" "2" "1" +` + +const v8Nonsense = v7Nonsense + switchNonsense + frameNonsense + matchNonsense + boxNonsense + +const v9Nonsense = v8Nonsense +const v10Nonsense = v9Nonsense + pairingNonsense + +const v6Compiled = "2004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f2310231123122313231418191a1b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b400b53a03b6b7043cb8033a0c2349c42a9631007300810881088120978101c53a8101c6003a" + +const randomnessCompiled = "81ffff03d101d000" + +const v7Compiled = v6Compiled + "5e005f018120af060180070123456789abcd49490501988003012345494984" + + randomnessCompiled + "800243218001775c0280018881015d" const boxCompiled = "b9babbbcbdbfbe" +const switchCompiled = "81018d02fff800008101" +const matchCompiled = "83030102018e02fff500008203013101320131" + +const v8Compiled = v7Compiled + switchCompiled + frameCompiled + matchCompiled + boxCompiled + +const v9Compiled = v8Compiled +const v10Compiled = v9Compiled + pairingCompiled + var nonsense = map[uint64]string{ - 1: v1Nonsense, - 2: v2Nonsense, - 3: v3Nonsense, - 4: v4Nonsense, - 5: v5Nonsense, - 6: v6Nonsense, - 7: v7Nonsense, + 1: v1Nonsense, + 2: v2Nonsense, + 3: v3Nonsense, + 4: v4Nonsense, + 5: v5Nonsense, + 6: v6Nonsense, + 7: v7Nonsense, + 8: v8Nonsense, + 9: v9Nonsense, + 10: v10Nonsense, } var compiled = map[uint64]string{ - 1: "012008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f23102311231223132314181b1c2b1716154000032903494", - 2: "022008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f23102311231223132314181b1c2b171615400003290349483403350222231d4a484848482a50512a63222352410003420000432105602105612105270463484821052b62482b642b65484821052b2106662b21056721072b682b692107210570004848210771004848361c0037001a0031183119311b311d311e311f3120210721051e312131223123312431253126312731283129312a312b312c312d312e312f", - 3: "032008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f23102311231223132314181b1c2b171615400003290349483403350222231d4a484848482a50512a63222352410003420000432105602105612105270463484821052b62482b642b65484821052b2106662b21056721072b682b692107210570004848210771004848361c0037001a0031183119311b311d311e311f3120210721051e312131223123312431253126312731283129312a312b312c312d312e312f4478222105531421055427042106552105082106564c4d4b02210538212106391c0081e80780046a6f686e", - 4: "042004010200b7a60c26040242420c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f23102311231223132314181b1c28171615400003290349483403350222231d4a484848482a50512a632223524100034200004322602261222b634848222862482864286548482228236628226724286828692422700048482471004848361c0037001a0031183119311b311d311e311f312024221e312131223123312431253126312731283129312a312b312c312d312e312f44782522531422542b2355220823564c4d4b0222382123391c0081e80780046a6f686e2281d00f24231f880003420001892223902291922394239593a0a1a2a3a4a5a6a7a8a9aaabacadae23af3a00003b003c003d8164", - 5: "052004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f23102311231223132314181b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b53a03", + 1: "012008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f2310231123122313231418191a1b1c2b1716154000032903494", + 2: "022008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f2310231123122313231418191a1b1c2b171615400003290349483403350222231d4a484848482a50512a63222352410003420000432105602105612105270463484821052b62482b642b65484821052b2106662b21056721072b682b692107210570004848210771004848361c0037001a0031183119311b311d311e311f3120210721051e312131223123312431253126312731283129312a312b312c312d312e312f", + 3: "032008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f2310231123122313231418191a1b1c2b171615400003290349483403350222231d4a484848482a50512a63222352410003420000432105602105612105270463484821052b62482b642b65484821052b2106662b21056721072b682b692107210570004848210771004848361c0037001a0031183119311b311d311e311f3120210721051e312131223123312431253126312731283129312a312b312c312d312e312f4478222105531421055427042106552105082106564c4d4b02210538212106391c0081e80780046a6f686e", + 4: "042004010200b7a60c26040242420c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f2310231123122313231418191a1b1c28171615400003290349483403350222231d4a484848482a50512a632223524100034200004322602261222b634848222862482864286548482228236628226724286828692422700048482471004848361c0037001a0031183119311b311d311e311f312024221e312131223123312431253126312731283129312a312b312c312d312e312f44782522531422542b2355220823564c4d4b0222382123391c0081e80780046a6f686e2281d00f24231f880003420001892223902291922394239593a0a1a2a3a4a5a6a7a8a9aaabacadae23af3a00003b003c003d8164", + 5: "052004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f2310231123122313231418191a1b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b400b53a03", 6: "06" + v6Compiled, 7: "07" + v7Compiled, + 8: "08" + v8Compiled, + 9: "09" + v9Compiled, + + 10: "10" + v10Compiled, } func pseudoOp(opcode string) bool { @@ -444,12 +498,14 @@ func TestAssemble(t *testing.T) { // This doesn't have to be a sensible program to run, it just has to compile. t.Parallel() - require.Equal(t, LogicVersion, len(nonsense)) + require.LessOrEqual(t, LogicVersion, len(nonsense)) // Allow nonsense for future versions for v := uint64(2); v <= AssemblerMaxVersion; v++ { t.Run(fmt.Sprintf("v=%d", v), func(t *testing.T) { for _, spec := range OpSpecs { - // Make sure our nonsense covers the ops - if !strings.Contains(nonsense[v], spec.Name) && + // Make sure our nonsense covers the ops. + hasOp, err := regexp.MatchString("\\s"+regexp.QuoteMeta(spec.Name)+"\\s", nonsense[v]) + require.NoError(t, err) + if !hasOp && !pseudoOp(spec.Name) && spec.Version <= v { t.Errorf("v%d nonsense test should contain op %v", v, spec.Name) } @@ -459,7 +515,10 @@ func TestAssemble(t *testing.T) { // check that compilation is stable over // time. we must assemble to the same bytes // this month that we did last month. - expectedBytes, _ := hex.DecodeString(compiled[v]) + bytecode, ok := compiled[v] + require.True(t, ok, "Need v%d bytecode", v) + expectedBytes, _ := hex.DecodeString(bytecode) + require.NotEmpty(t, expectedBytes) // the hex is for convenience if the program has been changed. the // hex string can be copy pasted back in as a new expected result. require.Equal(t, expectedBytes, ops.Program, hex.EncodeToString(ops.Program)) @@ -467,16 +526,20 @@ func TestAssemble(t *testing.T) { } } -var experiments = []uint64{fidoVersion, pairingVersion} +var experiments = []uint64{pairingVersion} // TestExperimental forces a conscious choice to promote "experimental" opcode // groups. This will fail when we increment vFuture's LogicSigVersion. If we had // intended to release the opcodes, they should have been removed from // `experiments`. func TestExperimental(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + futureV := config.Consensus[protocol.ConsensusFuture].LogicSigVersion for _, v := range experiments { - require.Equal(t, futureV, v) + // Allows less, so we can push something out, even before vFuture has been updated. + require.LessOrEqual(t, futureV, v) } } @@ -519,20 +582,46 @@ func testMatch(t testing.TB, actual, expected string) (ok bool) { } } -func assemblyTrace(text string, ver uint64) string { +func assembleWithTrace(text string, ver uint64) (*OpStream, error) { ops := newOpStream(ver) ops.Trace = &strings.Builder{} - ops.assemble(text) - return ops.Trace.String() + err := ops.assemble(text) + return &ops, err +} + +func lines(s string, num int) (bool, string) { + if num < 1 { + return true, "" + } + found := 0 + for i := 0; i < len(s); i++ { + if s[i] == '\n' { + found++ + if found == num { + return true, s[0 : i+1] + } + } + } + return false, s +} + +func summarize(trace *strings.Builder) string { + truncated, msg := lines(trace.String(), 50) + if !truncated { + return msg + } + return msg + "(trace truncated)\n" } func testProg(t testing.TB, source string, ver uint64, expected ...Expect) *OpStream { t.Helper() - program := strings.ReplaceAll(source, ";", "\n") - ops, err := AssembleStringWithVersion(program, ver) + ops, err := assembleWithTrace(source, ver) if len(expected) == 0 { if len(ops.Errors) > 0 || err != nil || ops == nil || ops.Program == nil { - t.Log(assemblyTrace(program, ver)) + t.Log(summarize(ops.Trace)) + } + if len(ops.Errors) > 10 { + ops.Errors = ops.Errors[:10] // Truncate to reasonable } require.Empty(t, ops.Errors) require.NoError(t, err) @@ -540,13 +629,13 @@ func testProg(t testing.TB, source string, ver uint64, expected ...Expect) *OpSt require.NotNil(t, ops.Program) // It should always be possible to Disassemble dis, err := Disassemble(ops.Program) - require.NoError(t, err, program) + require.NoError(t, err, source) // And, while the disassembly may not match input // exactly, the assembly of the disassembly should // give the same bytecode ops2, err := AssembleStringWithVersion(notrack(dis), ver) if len(ops2.Errors) > 0 || err != nil || ops2 == nil || ops2.Program == nil { - t.Log(program) + t.Log(source) t.Log(dis) } require.Empty(t, ops2.Errors) @@ -554,7 +643,7 @@ func testProg(t testing.TB, source string, ver uint64, expected ...Expect) *OpSt require.Equal(t, ops.Program, ops2.Program) } else { if err == nil { - t.Log(program) + t.Log(source) } require.Error(t, err) errors := ops.Errors @@ -570,7 +659,7 @@ func testProg(t testing.TB, source string, ver uint64, expected ...Expect) *OpSt } } if fail { - t.Log(assemblyTrace(program, ver)) + t.Log(summarize(ops.Trace)) t.FailNow() } } else { @@ -587,7 +676,7 @@ func testProg(t testing.TB, source string, ver uint64, expected ...Expect) *OpSt require.NotNil(t, found, "Error %s was not found on line %d", exp.s, exp.l) msg := found.Unwrap().Error() if !testMatch(t, msg, exp.s) { - t.Log(assemblyTrace(program, ver)) + t.Log(summarize(ops.Trace)) t.FailNow() } } @@ -621,25 +710,47 @@ func TestAssembleTxna(t *testing.T) { testLine(t, "gtxna 0 ApplicationArgs 256", AssemblerMaxVersion, "gtxna i beyond 255: 256") testLine(t, "gtxna 256 Accounts 0", AssemblerMaxVersion, "gtxna t beyond 255: 256") testLine(t, "gtxna 0 Sender 256", AssemblerMaxVersion, "gtxna unknown field: \"Sender\"") - testLine(t, "txn Accounts 0", 1, "txn expects 1 immediate argument") + testLine(t, "gtxna ApplicationArgs 0 255", AssemblerMaxVersion, "gtxna can only use \"ApplicationArgs\" as immediate 2") + testLine(t, "gtxna 0 255 ApplicationArgs", AssemblerMaxVersion, "gtxna can only use \"255\" as immediate 1 or 3") + + testLine(t, "txn Accounts 256", AssemblerMaxVersion, "txn i beyond 255: 256") + testLine(t, "txn ApplicationArgs 256", AssemblerMaxVersion, "txn i beyond 255: 256") + testLine(t, "txn 255 ApplicationArgs", AssemblerMaxVersion, "txn with 2 immediates can only use \"255\" as immediate 2") + testLine(t, "txn Sender 256", AssemblerMaxVersion, "\"Sender\" field of txn can only be used with 1 immediate") + testLine(t, "gtxn 0 Accounts 256", AssemblerMaxVersion, "gtxn i beyond 255: 256") + testLine(t, "gtxn 0 ApplicationArgs 256", AssemblerMaxVersion, "gtxn i beyond 255: 256") + testLine(t, "gtxn 256 Accounts 0", AssemblerMaxVersion, "gtxn t beyond 255: 256") + testLine(t, "gtxn 0 Sender 256", AssemblerMaxVersion, "\"Sender\" field of gtxn can only be used with 2 immediates") + testLine(t, "gtxn ApplicationArgs 0 255", AssemblerMaxVersion, "gtxn with 3 immediates can only use \"ApplicationArgs\" as immediate 2") + testLine(t, "gtxn 0 255 ApplicationArgs", AssemblerMaxVersion, "gtxn with 3 immediates can only use \"255\" as immediate 1 or 3") + + testLine(t, "txn Accounts 0", 1, "txn opcode with 2 immediates was introduced in v2") testLine(t, "txn Accounts 0 1", 2, "txn expects 1 or 2 immediate arguments") testLine(t, "txna Accounts 0 1", AssemblerMaxVersion, "txna expects 2 immediate arguments") + testLine(t, "txn Accounts 0 1", AssemblerMaxVersion, "txn expects 1 or 2 immediate arguments") testLine(t, "txnas Accounts 1", AssemblerMaxVersion, "txnas expects 1 immediate argument") testLine(t, "txna Accounts a", AssemblerMaxVersion, "txna unable to parse...") - testLine(t, "gtxn 0 Sender 0", 1, "gtxn expects 2 immediate arguments") + testLine(t, "txn Accounts a", AssemblerMaxVersion, "txn unable to parse...") + testLine(t, "gtxn 0 Sender 0", 1, "gtxn opcode with 3 immediates was introduced in v2") testLine(t, "gtxn 0 Sender 1 2", 2, "gtxn expects 2 or 3 immediate arguments") testLine(t, "gtxna 0 Accounts 1 2", AssemblerMaxVersion, "gtxna expects 3 immediate arguments") testLine(t, "gtxna a Accounts 0", AssemblerMaxVersion, "gtxna unable to parse...") testLine(t, "gtxna 0 Accounts a", AssemblerMaxVersion, "gtxna unable to parse...") + + testLine(t, "gtxn 0 Accounts 1 2", AssemblerMaxVersion, "gtxn expects 2 or 3 immediate arguments") + testLine(t, "gtxn a Accounts 0", AssemblerMaxVersion, "gtxn unable to parse...") + testLine(t, "gtxn 0 Accounts a", AssemblerMaxVersion, "gtxn unable to parse...") + testLine(t, "gtxnas Accounts 1 2", AssemblerMaxVersion, "gtxnas expects 2 immediate arguments") testLine(t, "txn ABC", 2, "txn unknown field: \"ABC\"") testLine(t, "gtxn 0 ABC", 2, "gtxn unknown field: \"ABC\"") testLine(t, "gtxn a ABC", 2, "gtxn unable to parse...") - testLine(t, "txn Accounts", 1, "txn unknown field: \"Accounts\"") - testLine(t, "txn Accounts", AssemblerMaxVersion, "txn unknown field: \"Accounts\"") + // For now not going to additionally report version issue until version is only problem + testLine(t, "txn Accounts", 1, "\"Accounts\" field of txn can only be used with 2 immediates") + testLine(t, "txn Accounts", AssemblerMaxVersion, "\"Accounts\" field of txn can only be used with 2 immediates") testLine(t, "txn Accounts 0", AssemblerMaxVersion, "") - testLine(t, "gtxn 0 Accounts", AssemblerMaxVersion, "gtxn unknown field: \"Accounts\"...") - testLine(t, "gtxn 0 Accounts", 1, "gtxn unknown field: \"Accounts\"") + testLine(t, "gtxn 0 Accounts", AssemblerMaxVersion, "\"Accounts\" field of gtxn can only be used with 3 immediates") + testLine(t, "gtxn 0 Accounts", 1, "\"Accounts\" field of gtxn can only be used with 3 immediates") testLine(t, "gtxn 0 Accounts 1", AssemblerMaxVersion, "") } @@ -652,9 +763,9 @@ func TestAssembleGlobal(t *testing.T) { testProg(t, "global MinTxnFee; int 2; +", AssemblerMaxVersion) testProg(t, "global ZeroAddress; byte 0x12; concat; len", AssemblerMaxVersion) testProg(t, "global MinTxnFee; byte 0x12; concat", AssemblerMaxVersion, - Expect{3, "concat arg 0 wanted type []byte..."}) + Expect{1, "concat arg 0 wanted type []byte..."}) testProg(t, "int 2; global ZeroAddress; +", AssemblerMaxVersion, - Expect{3, "+ arg 1 wanted type uint64..."}) + Expect{1, "+ arg 1 wanted type uint64..."}) } func TestAssembleDefault(t *testing.T) { @@ -1181,209 +1292,94 @@ func TestFieldsFromLine(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - line := "op arg" - fields := fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "arg", fields[1]) - - line = "op arg // test" - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "arg", fields[1]) - - line = "op base64 ABC//==" - fields = fieldsFromLine(line) - require.Equal(t, 3, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "base64", fields[1]) - require.Equal(t, "ABC//==", fields[2]) - - line = "op base64 ABC/==" - fields = fieldsFromLine(line) - require.Equal(t, 3, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "base64", fields[1]) - require.Equal(t, "ABC/==", fields[2]) - - line = "op base64 ABC/== /" - fields = fieldsFromLine(line) - require.Equal(t, 4, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "base64", fields[1]) - require.Equal(t, "ABC/==", fields[2]) - require.Equal(t, "/", fields[3]) - - line = "op base64 ABC/== //" - fields = fieldsFromLine(line) - require.Equal(t, 3, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "base64", fields[1]) - require.Equal(t, "ABC/==", fields[2]) - - line = "op base64 ABC//== //" - fields = fieldsFromLine(line) - require.Equal(t, 3, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "base64", fields[1]) - require.Equal(t, "ABC//==", fields[2]) - - line = "op b64 ABC//== //" - fields = fieldsFromLine(line) - require.Equal(t, 3, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "b64", fields[1]) - require.Equal(t, "ABC//==", fields[2]) - - line = "op b64(ABC//==) // comment" - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "b64(ABC//==)", fields[1]) - - line = "op base64(ABC//==) // comment" - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "base64(ABC//==)", fields[1]) - - line = "op b64(ABC/==) // comment" - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "b64(ABC/==)", fields[1]) - - line = "op base64(ABC/==) // comment" - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "base64(ABC/==)", fields[1]) - - line = "base64(ABC//==)" - fields = fieldsFromLine(line) - require.Equal(t, 1, len(fields)) - require.Equal(t, "base64(ABC//==)", fields[0]) - - line = "b(ABC//==)" - fields = fieldsFromLine(line) - require.Equal(t, 1, len(fields)) - require.Equal(t, "b(ABC", fields[0]) - - line = "b(ABC//==) //" - fields = fieldsFromLine(line) - require.Equal(t, 1, len(fields)) - require.Equal(t, "b(ABC", fields[0]) - - line = "b(ABC ==) //" - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "b(ABC", fields[0]) - require.Equal(t, "==)", fields[1]) - - line = "op base64 ABC)" - fields = fieldsFromLine(line) - require.Equal(t, 3, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "base64", fields[1]) - require.Equal(t, "ABC)", fields[2]) - - line = "op base64 ABC) // comment" - fields = fieldsFromLine(line) - require.Equal(t, 3, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "base64", fields[1]) - require.Equal(t, "ABC)", fields[2]) - - line = "op base64 ABC//) // comment" - fields = fieldsFromLine(line) - require.Equal(t, 3, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, "base64", fields[1]) - require.Equal(t, "ABC//)", fields[2]) - - line = `op "test"` - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `"test"`, fields[1]) - - line = `op "test1 test2"` - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `"test1 test2"`, fields[1]) - - line = `op "test1 test2" // comment` - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `"test1 test2"`, fields[1]) - - line = `op "test1 test2 // not a comment"` - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `"test1 test2 // not a comment"`, fields[1]) - - line = `op "test1 test2 // not a comment" // comment` - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `"test1 test2 // not a comment"`, fields[1]) - - line = `op "test1 test2 // not a comment" // comment` - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `"test1 test2 // not a comment"`, fields[1]) - - line = `op "test1 test2" //` - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `"test1 test2"`, fields[1]) - - line = `op "test1 test2"//` - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `"test1 test2"`, fields[1]) - - line = `op "test1 test2` // non-terminated string literal - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `"test1 test2`, fields[1]) - - line = `op "test1 test2\"` // non-terminated string literal - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `"test1 test2\"`, fields[1]) - - line = `op \"test1 test2\"` // not a string literal - fields = fieldsFromLine(line) - require.Equal(t, 3, len(fields)) - require.Equal(t, "op", fields[0]) - require.Equal(t, `\"test1`, fields[1]) - require.Equal(t, `test2\"`, fields[2]) - - line = `"test1 test2"` - fields = fieldsFromLine(line) - require.Equal(t, 1, len(fields)) - require.Equal(t, `"test1 test2"`, fields[0]) - - line = `\"test1 test2"` - fields = fieldsFromLine(line) - require.Equal(t, 2, len(fields)) - require.Equal(t, `\"test1`, fields[0]) - require.Equal(t, `test2"`, fields[1]) - - line = `"" // test` - fields = fieldsFromLine(line) - require.Equal(t, 1, len(fields)) - require.Equal(t, `""`, fields[0]) + check := func(line string, tokens ...string) { + t.Helper() + assert.Equal(t, tokensFromLine(line), tokens) + } + + check("op arg", "op", "arg") + check("op arg // test", "op", "arg") + check("op base64 ABC//==", "op", "base64", "ABC//==") + check("op base64 base64", "op", "base64", "base64") + check("op base64 base64 //comment", "op", "base64", "base64") + check("op base64 base64; op2 //done", "op", "base64", "base64", ";", "op2") + check("op base64 ABC/==", "op", "base64", "ABC/==") + check("op base64 ABC/== /", "op", "base64", "ABC/==", "/") + check("op base64 ABC/== //", "op", "base64", "ABC/==") + check("op base64 ABC//== //", "op", "base64", "ABC//==") + check("op b64 ABC//== //", "op", "b64", "ABC//==") + check("op b64(ABC//==) // comment", "op", "b64(ABC//==)") + check("op base64(ABC//==) // comment", "op", "base64(ABC//==)") + check("op b64(ABC/==) // comment", "op", "b64(ABC/==)") + check("op base64(ABC/==) // comment", "op", "base64(ABC/==)") + check("base64(ABC//==)", "base64(ABC//==)") + check("b(ABC//==)", "b(ABC") + check("b(ABC//==) //", "b(ABC") + check("b(ABC ==) //", "b(ABC", "==)") + check("op base64 ABC)", "op", "base64", "ABC)") + check("op base64 ABC) // comment", "op", "base64", "ABC)") + check("op base64 ABC//) // comment", "op", "base64", "ABC//)") + check(`op "test"`, "op", `"test"`) + check(`op "test1 test2"`, "op", `"test1 test2"`) + check(`op "test1 test2" // comment`, "op", `"test1 test2"`) + check(`op "test1 test2 // not a comment"`, "op", `"test1 test2 // not a comment"`) + check(`op "test1 test2 // not a comment" // comment`, "op", `"test1 test2 // not a comment"`) + check(`op "test1 test2" //`, "op", `"test1 test2"`) + check(`op "test1 test2"//`, "op", `"test1 test2"`) + check(`op "test1 test2`, "op", `"test1 test2`) // non-terminated string literal + check(`op "test1 test2\"`, "op", `"test1 test2\"`) // non-terminated string literal + check(`op \"test1 test2\"`, "op", `\"test1`, `test2\"`) // not a string literal + check(`"test1 test2"`, `"test1 test2"`) + check(`\"test1 test2"`, `\"test1`, `test2"`) + check(`"" // test`, `""`) + check("int 1; int 2", "int", "1", ";", "int", "2") + check("int 1;;;int 2", "int", "1", ";", ";", ";", "int", "2") + check("int 1; ;int 2;; ; ;; ", "int", "1", ";", ";", "int", "2", ";", ";", ";", ";", ";") + check(";", ";") + check("; ; ;;;;", ";", ";", ";", ";", ";", ";") + check(" ;", ";") + check(" ; ", ";") +} + +func TestNextStatement(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + // this test ensures nextStatement splits tokens on semicolons properly + // macro testing should be handled in TestMacros + ops := newOpStream(AssemblerMaxVersion) + check := func(tokens []string, left []string, right []string) { + t.Helper() + current, next := nextStatement(&ops, tokens) + assert.Equal(t, left, current) + assert.Equal(t, right, next) + } + + check([]string{"hey,", "how's", ";", ";", "it", "going", ";"}, + []string{"hey,", "how's"}, + []string{";", "it", "going", ";"}, + ) + + check([]string{";"}, + []string{}, + []string{}, + ) + + check([]string{";", "it", "going"}, + []string{}, + []string{"it", "going"}, + ) + + check([]string{"hey,", "how's"}, + []string{"hey,", "how's"}, + nil, + ) + + check([]string{`"hey in quotes;"`, "getting", `";"`, ";", "tricky"}, + []string{`"hey in quotes;"`, "getting", `";"`}, + []string{"tricky"}, + ) + } func TestAssembleRejectNegJump(t *testing.T) { @@ -1635,8 +1631,12 @@ func TestAssembleDisassembleCycle(t *testing.T) { // optimizations in later versions that change the bytecode // emitted. But currently it is, so we test it for now to // catch any suprises. - require.Equal(t, LogicVersion, len(nonsense)) + require.LessOrEqual(t, LogicVersion, len(nonsense)) // Allow nonsense for future versions for v, source := range nonsense { + v, source := v, source + if v > LogicVersion { + continue // We allow them to be set, but can't test assembly beyond LogicVersion + } t.Run(fmt.Sprintf("v=%d", v), func(t *testing.T) { t.Parallel() ops := testProg(t, source, v) @@ -1711,10 +1711,10 @@ func TestBranchArgs(t *testing.T) { for v := uint64(2); v <= AssemblerMaxVersion; v++ { testProg(t, "b", v, Expect{1, "b needs a single label argument"}) testProg(t, "b lab1 lab2", v, Expect{1, "b needs a single label argument"}) - testProg(t, "int 1; bz", v, Expect{2, "bz needs a single label argument"}) - testProg(t, "int 1; bz a b", v, Expect{2, "bz needs a single label argument"}) - testProg(t, "int 1; bnz", v, Expect{2, "bnz needs a single label argument"}) - testProg(t, "int 1; bnz c d", v, Expect{2, "bnz needs a single label argument"}) + testProg(t, "int 1; bz", v, Expect{1, "bz needs a single label argument"}) + testProg(t, "int 1; bz a b", v, Expect{1, "bz needs a single label argument"}) + testProg(t, "int 1; bnz", v, Expect{1, "bnz needs a single label argument"}) + testProg(t, "int 1; bnz c d", v, Expect{1, "bnz needs a single label argument"}) } for v := uint64(4); v <= AssemblerMaxVersion; v++ { @@ -1846,7 +1846,7 @@ func TestAssembleVersions(t *testing.T) { testLine(t, "txna Accounts 0", AssemblerMaxVersion, "") testLine(t, "txna Accounts 0", 2, "") - testLine(t, "txna Accounts 0", 1, "txna opcode was introduced in TEAL v2") + testLine(t, "txna Accounts 0", 1, "txna opcode was introduced in v2") } func TestAssembleBalance(t *testing.T) { @@ -1889,22 +1889,32 @@ func TestAssembleAsset(t *testing.T) { testProg(t, "asset_holding_get ABC 1", v, Expect{1, "asset_holding_get ABC 1 expects 2 stack arguments..."}) testProg(t, "int 1; asset_holding_get ABC 1", v, - Expect{2, "asset_holding_get ABC 1 expects 2 stack arguments..."}) - testProg(t, "int 1; int 1; asset_holding_get ABC 1", v, - Expect{3, "asset_holding_get expects 1 immediate argument"}) - testProg(t, "int 1; int 1; asset_holding_get ABC", v, - Expect{3, "asset_holding_get unknown field: \"ABC\""}) + Expect{1, "asset_holding_get ABC 1 expects 2 stack arguments..."}) + + if v < sharedResourcesVersion { + testProg(t, "int 1; int 1; asset_holding_get ABC 1", v, + Expect{1, "asset_holding_get expects 1 immediate argument"}) + testProg(t, "int 1; int 1; asset_holding_get ABC", v, + Expect{1, "asset_holding_get unknown field: \"ABC\""}) + } else { + testProg(t, "int 1; int 1; asset_holding_get ABC 1", v, + Expect{1, "asset_holding_get ABC 1 arg 0 wanted type..."}) + testProg(t, "txn Sender; int 1; asset_holding_get ABC 1", v, + Expect{1, "asset_holding_get expects 1 immediate argument"}) + testProg(t, "txn Sender; int 1; asset_holding_get ABC", v, + Expect{1, "asset_holding_get unknown field: \"ABC\""}) + } testProg(t, "byte 0x1234; asset_params_get ABC 1", v, - Expect{2, "asset_params_get ABC 1 arg 0 wanted type uint64..."}) + Expect{1, "asset_params_get ABC 1 arg 0 wanted type uint64..."}) // Test that AssetUnitName is known to return bytes testProg(t, "int 1; asset_params_get AssetUnitName; pop; int 1; +", v, - Expect{5, "+ arg 0 wanted type uint64..."}) + Expect{1, "+ arg 0 wanted type uint64..."}) // Test that AssetTotal is known to return uint64 testProg(t, "int 1; asset_params_get AssetTotal; pop; byte 0x12; concat", v, - Expect{5, "concat arg 0 wanted type []byte..."}) + Expect{1, "concat arg 0 wanted type []byte..."}) testLine(t, "asset_params_get ABC 1", v, "asset_params_get expects 1 immediate argument") testLine(t, "asset_params_get ABC", v, "asset_params_get unknown field: \"ABC\"") @@ -2037,7 +2047,7 @@ func TestDisassembleLastLabel(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - // starting from TEAL v2 branching to the last line are legal + // starting from v2 branching to the last line are legal for v := uint64(2); v <= AssemblerMaxVersion; v++ { t.Run(fmt.Sprintf("v=%d", v), func(t *testing.T) { source := fmt.Sprintf(`#pragma version %d @@ -2372,7 +2382,7 @@ int 1 ops = testProg(t, text, assemblerNoVersion) require.Equal(t, ops2.Program, ops.Program) - // check if no version it defaults to TEAL v1 + // check if no version it defaults to v1 text = `byte "test" len ` @@ -2502,69 +2512,92 @@ func TestSwapTypeCheck(t *testing.T) { t.Parallel() /* reconfirm that we detect this type error */ - testProg(t, "int 1; byte 0x1234; +", AssemblerMaxVersion, Expect{3, "+ arg 1..."}) + testProg(t, "int 1; byte 0x1234; +", AssemblerMaxVersion, Expect{1, "+ arg 1..."}) /* despite swap, we track types */ - testProg(t, "int 1; byte 0x1234; swap; +", AssemblerMaxVersion, Expect{4, "+ arg 0..."}) - testProg(t, "byte 0x1234; int 1; swap; +", AssemblerMaxVersion, Expect{4, "+ arg 1..."}) + testProg(t, "int 1; byte 0x1234; swap; +", AssemblerMaxVersion, Expect{1, "+ arg 0..."}) + testProg(t, "byte 0x1234; int 1; swap; +", AssemblerMaxVersion, Expect{1, "+ arg 1..."}) } func TestDigAsm(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "int 1; dig; +", AssemblerMaxVersion, Expect{2, "dig expects 1 immediate..."}) - testProg(t, "int 1; dig junk; +", AssemblerMaxVersion, Expect{2, "dig unable to parse..."}) + testProg(t, "int 1; dig; +", AssemblerMaxVersion, Expect{1, "dig expects 1 immediate..."}) + testProg(t, "int 1; dig junk; +", AssemblerMaxVersion, Expect{1, "dig unable to parse..."}) testProg(t, "int 1; byte 0x1234; int 2; dig 2; +", AssemblerMaxVersion) testProg(t, "byte 0x32; byte 0x1234; int 2; dig 2; +", AssemblerMaxVersion, - Expect{5, "+ arg 1..."}) + Expect{1, "+ arg 1..."}) testProg(t, "byte 0x32; byte 0x1234; int 2; dig 3; +", AssemblerMaxVersion, - Expect{4, "dig 3 expects 4..."}) + Expect{1, "dig 3 expects 4..."}) testProg(t, "int 1; byte 0x1234; int 2; dig 12; +", AssemblerMaxVersion, - Expect{4, "dig 12 expects 13..."}) + Expect{1, "dig 12 expects 13..."}) // Confirm that digging something out does not ruin our knowledge about the types in the middle testProg(t, "int 1; byte 0x1234; byte 0x1234; dig 2; dig 3; +; pop; +", AssemblerMaxVersion, - Expect{8, "+ arg 1..."}) + Expect{1, "+ arg 1..."}) testProg(t, "int 3; pushbytes \"123456\"; int 1; dig 2; substring3", AssemblerMaxVersion) } +func TestBuryAsm(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + testProg(t, "int 1; bury; +", AssemblerMaxVersion, Expect{1, "bury expects 1 immediate..."}) + testProg(t, "int 1; bury junk; +", AssemblerMaxVersion, Expect{1, "bury unable to parse..."}) + + testProg(t, "int 1; byte 0x1234; int 2; bury 1; +", AssemblerMaxVersion) // the 2 replaces the byte string + testProg(t, "int 2; int 2; byte 0x1234; bury 1; +", AssemblerMaxVersion, + Expect{1, "+ arg 1..."}) + testProg(t, "byte 0x32; byte 0x1234; int 2; bury 3; +", AssemblerMaxVersion, + Expect{1, "bury 3 expects 4..."}) + testProg(t, "int 1; byte 0x1234; int 2; bury 12; +", AssemblerMaxVersion, + Expect{1, "bury 12 expects 13..."}) + + // We do not lose track of the ints between ToS and bury index + testProg(t, "int 0; int 1; int 2; int 4; bury 3; concat", AssemblerMaxVersion, + Expect{1, "concat arg 1 wanted type []byte..."}) + + // Even when we are burying into unknown (seems repetitive, but is an easy bug) + testProg(t, "int 0; int 0; b LABEL; LABEL: int 1; int 2; int 4; bury 4; concat", AssemblerMaxVersion, + Expect{1, "concat arg 1 wanted type []byte..."}) +} + func TestEqualsTypeCheck(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "int 1; byte 0x1234; ==", AssemblerMaxVersion, Expect{3, "== arg 0..."}) - testProg(t, "int 1; byte 0x1234; !=", AssemblerMaxVersion, Expect{3, "!= arg 0..."}) - testProg(t, "byte 0x1234; int 1; ==", AssemblerMaxVersion, Expect{3, "== arg 0..."}) - testProg(t, "byte 0x1234; int 1; !=", AssemblerMaxVersion, Expect{3, "!= arg 0..."}) + testProg(t, "int 1; byte 0x1234; ==", AssemblerMaxVersion, Expect{1, "== arg 0..."}) + testProg(t, "int 1; byte 0x1234; !=", AssemblerMaxVersion, Expect{1, "!= arg 0..."}) + testProg(t, "byte 0x1234; int 1; ==", AssemblerMaxVersion, Expect{1, "== arg 0..."}) + testProg(t, "byte 0x1234; int 1; !=", AssemblerMaxVersion, Expect{1, "!= arg 0..."}) } func TestDupTypeCheck(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "byte 0x1234; dup; int 1; +", AssemblerMaxVersion, Expect{4, "+ arg 0..."}) + testProg(t, "byte 0x1234; dup; int 1; +", AssemblerMaxVersion, Expect{1, "+ arg 0..."}) testProg(t, "byte 0x1234; int 1; dup; +", AssemblerMaxVersion) - testProg(t, "byte 0x1234; int 1; dup2; +", AssemblerMaxVersion, Expect{4, "+ arg 0..."}) - testProg(t, "int 1; byte 0x1234; dup2; +", AssemblerMaxVersion, Expect{4, "+ arg 1..."}) + testProg(t, "byte 0x1234; int 1; dup2; +", AssemblerMaxVersion, Expect{1, "+ arg 0..."}) + testProg(t, "int 1; byte 0x1234; dup2; +", AssemblerMaxVersion, Expect{1, "+ arg 1..."}) - testProg(t, "byte 0x1234; int 1; dup; dig 1; len", AssemblerMaxVersion, Expect{5, "len arg 0..."}) - testProg(t, "int 1; byte 0x1234; dup; dig 1; !", AssemblerMaxVersion, Expect{5, "! arg 0..."}) + testProg(t, "byte 0x1234; int 1; dup; dig 1; len", AssemblerMaxVersion, Expect{1, "len arg 0..."}) + testProg(t, "int 1; byte 0x1234; dup; dig 1; !", AssemblerMaxVersion, Expect{1, "! arg 0..."}) - testProg(t, "byte 0x1234; int 1; dup2; dig 2; len", AssemblerMaxVersion, Expect{5, "len arg 0..."}) - testProg(t, "int 1; byte 0x1234; dup2; dig 2; !", AssemblerMaxVersion, Expect{5, "! arg 0..."}) + testProg(t, "byte 0x1234; int 1; dup2; dig 2; len", AssemblerMaxVersion, Expect{1, "len arg 0..."}) + testProg(t, "int 1; byte 0x1234; dup2; dig 2; !", AssemblerMaxVersion, Expect{1, "! arg 0..."}) } func TestSelectTypeCheck(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "int 1; int 2; int 3; select; len", AssemblerMaxVersion, Expect{5, "len arg 0..."}) - testProg(t, "byte 0x1234; byte 0x5678; int 3; select; !", AssemblerMaxVersion, Expect{5, "! arg 0..."}) + testProg(t, "int 1; int 2; int 3; select; len", AssemblerMaxVersion, Expect{1, "len arg 0..."}) + testProg(t, "byte 0x1234; byte 0x5678; int 3; select; !", AssemblerMaxVersion, Expect{1, "! arg 0..."}) } func TestSetBitTypeCheck(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "int 1; int 2; int 3; setbit; len", AssemblerMaxVersion, Expect{5, "len arg 0..."}) - testProg(t, "byte 0x1234; int 2; int 3; setbit; !", AssemblerMaxVersion, Expect{5, "! arg 0..."}) + testProg(t, "int 1; int 2; int 3; setbit; len", AssemblerMaxVersion, Expect{1, "len arg 0..."}) + testProg(t, "byte 0x1234; int 2; int 3; setbit; !", AssemblerMaxVersion, Expect{1, "! arg 0..."}) } func TestScratchTypeCheck(t *testing.T) { @@ -2573,13 +2606,13 @@ func TestScratchTypeCheck(t *testing.T) { // All scratch slots should start as uint64 testProg(t, "load 0; int 1; +", AssemblerMaxVersion) // Check load and store accurately using the scratch space - testProg(t, "byte 0x01; store 0; load 0; int 1; +", AssemblerMaxVersion, Expect{5, "+ arg 0..."}) + testProg(t, "byte 0x01; store 0; load 0; int 1; +", AssemblerMaxVersion, Expect{1, "+ arg 0..."}) // Loads should know the type it's loading if all the slots are the same type - testProg(t, "int 0; loads; btoi", AssemblerMaxVersion, Expect{3, "btoi arg 0..."}) + testProg(t, "int 0; loads; btoi", AssemblerMaxVersion, Expect{1, "btoi arg 0..."}) // Loads doesn't know the type when slot types vary testProg(t, "byte 0x01; store 0; int 1; loads; btoi", AssemblerMaxVersion) // Stores should only set slots to StackAny if they are not the same type as what is being stored - testProg(t, "byte 0x01; store 0; int 3; byte 0x01; stores; load 0; int 1; +", AssemblerMaxVersion, Expect{8, "+ arg 0..."}) + testProg(t, "byte 0x01; store 0; int 3; byte 0x01; stores; load 0; int 1; +", AssemblerMaxVersion, Expect{1, "+ arg 0..."}) // ScratchSpace should reset after hitting label in deadcode testProg(t, "byte 0x01; store 0; b label1; label1:; load 0; int 1; +", AssemblerMaxVersion) // But it should reset to StackAny not uint64 @@ -2587,7 +2620,32 @@ func TestScratchTypeCheck(t *testing.T) { // Callsubs should also reset the scratch space testProg(t, "callsub A; load 0; btoi; return; A: byte 0x01; store 0; retsub", AssemblerMaxVersion) // But the scratchspace should still be tracked after the callsub - testProg(t, "callsub A; int 1; store 0; load 0; btoi; return; A: retsub", AssemblerMaxVersion, Expect{5, "btoi arg 0..."}) + testProg(t, "callsub A; int 1; store 0; load 0; btoi; return; A: retsub", AssemblerMaxVersion, Expect{1, "btoi arg 0..."}) +} + +// TestProtoAsm confirms that the assembler will yell at you if you are +// clearly dipping into the arguments when using `proto`. You should be using +// `frame_dig`. +func TestProtoAsm(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + testProg(t, "proto 0 0", AssemblerMaxVersion, Expect{1, "proto must be unreachable..."}) + testProg(t, notrack("proto 0 0"), AssemblerMaxVersion) + testProg(t, "b a; int 1; a: proto 0 0", AssemblerMaxVersion) // we could flag a `b` to `proto` + + testProg(t, ` + int 10 + int 20 + callsub main + int 1 + return +main: + proto 2 1 + + // This consumes the top arg. We complain. + dup; dup // Even though the dup;dup restores it, so it _evals_ fine. + retsub +`, AssemblerMaxVersion) + } func TestCoverAsm(t *testing.T) { @@ -2595,9 +2653,10 @@ func TestCoverAsm(t *testing.T) { t.Parallel() testProg(t, `int 4; byte "john"; int 5; cover 2; pop; +`, AssemblerMaxVersion) testProg(t, `int 4; byte "ayush"; int 5; cover 1; pop; +`, AssemblerMaxVersion) - testProg(t, `int 4; byte "john"; int 5; cover 2; +`, AssemblerMaxVersion, Expect{5, "+ arg 1..."}) + testProg(t, `int 4; byte "john"; int 5; cover 2; +`, AssemblerMaxVersion, Expect{1, "+ arg 1..."}) - testProg(t, `int 4; cover junk`, AssemblerMaxVersion, Expect{2, "cover unable to parse n ..."}) + testProg(t, `int 4; cover junk`, AssemblerMaxVersion, Expect{1, "cover unable to parse n ..."}) + testProg(t, notrack(`int 4; int 5; cover 0`), AssemblerMaxVersion) } func TestUncoverAsm(t *testing.T) { @@ -2606,38 +2665,38 @@ func TestUncoverAsm(t *testing.T) { testProg(t, `int 4; byte "john"; int 5; uncover 2; +`, AssemblerMaxVersion) testProg(t, `int 4; byte "ayush"; int 5; uncover 1; pop; +`, AssemblerMaxVersion) testProg(t, `int 1; byte "jj"; byte "ayush"; byte "john"; int 5; uncover 4; +`, AssemblerMaxVersion) - testProg(t, `int 4; byte "ayush"; int 5; uncover 1; +`, AssemblerMaxVersion, Expect{5, "+ arg 1..."}) + testProg(t, `int 4; byte "ayush"; int 5; uncover 1; +`, AssemblerMaxVersion, Expect{1, "+ arg 1..."}) } func TestTxTypes(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "itxn_begin; itxn_field Sender", 5, Expect{2, "itxn_field Sender expects 1 stack argument..."}) - testProg(t, "itxn_begin; int 1; itxn_field Sender", 5, Expect{3, "...wanted type []byte got uint64"}) + testProg(t, "itxn_begin; itxn_field Sender", 5, Expect{1, "itxn_field Sender expects 1 stack argument..."}) + testProg(t, "itxn_begin; int 1; itxn_field Sender", 5, Expect{1, "...wanted type []byte got uint64"}) testProg(t, "itxn_begin; byte 0x56127823; itxn_field Sender", 5) - testProg(t, "itxn_begin; itxn_field Amount", 5, Expect{2, "itxn_field Amount expects 1 stack argument..."}) - testProg(t, "itxn_begin; byte 0x87123376; itxn_field Amount", 5, Expect{3, "...wanted type uint64 got []byte"}) + testProg(t, "itxn_begin; itxn_field Amount", 5, Expect{1, "itxn_field Amount expects 1 stack argument..."}) + testProg(t, "itxn_begin; byte 0x87123376; itxn_field Amount", 5, Expect{1, "...wanted type uint64 got []byte"}) testProg(t, "itxn_begin; int 1; itxn_field Amount", 5) } func TestBadInnerFields(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - testProg(t, "itxn_begin; int 1000; itxn_field FirstValid", 5, Expect{3, "...is not allowed."}) - testProg(t, "itxn_begin; int 1000; itxn_field FirstValidTime", 5, Expect{3, "...is not allowed."}) - testProg(t, "itxn_begin; int 1000; itxn_field LastValid", 5, Expect{3, "...is not allowed."}) - testProg(t, "itxn_begin; int 32; bzero; itxn_field Lease", 5, Expect{4, "...is not allowed."}) - testProg(t, "itxn_begin; byte 0x7263; itxn_field Note", 5, Expect{3, "...Note field was introduced in TEAL v6..."}) - testProg(t, "itxn_begin; byte 0x7263; itxn_field VotePK", 5, Expect{3, "...VotePK field was introduced in TEAL v6..."}) - testProg(t, "itxn_begin; int 32; bzero; itxn_field TxID", 5, Expect{4, "...is not allowed."}) - - testProg(t, "itxn_begin; int 1000; itxn_field FirstValid", 6, Expect{3, "...is not allowed."}) - testProg(t, "itxn_begin; int 1000; itxn_field LastValid", 6, Expect{3, "...is not allowed."}) - testProg(t, "itxn_begin; int 32; bzero; itxn_field Lease", 6, Expect{4, "...is not allowed."}) + testProg(t, "itxn_begin; int 1000; itxn_field FirstValid", 5, Expect{1, "...is not allowed."}) + testProg(t, "itxn_begin; int 1000; itxn_field FirstValidTime", 5, Expect{1, "...is not allowed."}) + testProg(t, "itxn_begin; int 1000; itxn_field LastValid", 5, Expect{1, "...is not allowed."}) + testProg(t, "itxn_begin; int 32; bzero; itxn_field Lease", 5, Expect{1, "...is not allowed."}) + testProg(t, "itxn_begin; byte 0x7263; itxn_field Note", 5, Expect{1, "...Note field was introduced in v6..."}) + testProg(t, "itxn_begin; byte 0x7263; itxn_field VotePK", 5, Expect{1, "...VotePK field was introduced in v6..."}) + testProg(t, "itxn_begin; int 32; bzero; itxn_field TxID", 5, Expect{1, "...is not allowed."}) + + testProg(t, "itxn_begin; int 1000; itxn_field FirstValid", 6, Expect{1, "...is not allowed."}) + testProg(t, "itxn_begin; int 1000; itxn_field LastValid", 6, Expect{1, "...is not allowed."}) + testProg(t, "itxn_begin; int 32; bzero; itxn_field Lease", 6, Expect{1, "...is not allowed."}) testProg(t, "itxn_begin; byte 0x7263; itxn_field Note", 6) testProg(t, "itxn_begin; byte 0x7263; itxn_field VotePK", 6) - testProg(t, "itxn_begin; int 32; bzero; itxn_field TxID", 6, Expect{4, "...is not allowed."}) + testProg(t, "itxn_begin; int 32; bzero; itxn_field TxID", 6, Expect{1, "...is not allowed."}) } func TestTypeTracking(t *testing.T) { @@ -2653,7 +2712,7 @@ func TestTypeTracking(t *testing.T) { // but we do want to ensure we're not just treating the code after callsub as dead testProg(t, "callsub A; int 1; concat; return; A: int 1; int 2; retsub", LogicVersion, - Expect{3, "concat arg 1 wanted..."}) + Expect{1, "concat arg 1 wanted..."}) // retsub deadens code, like any unconditional branch testProg(t, "callsub A; +; return; A: int 1; int 2; retsub; concat", LogicVersion) @@ -2725,3 +2784,832 @@ done: concat `, LogicVersion, Expect{5, "concat arg 1 wanted type []byte..."}) } + +func TestMergeProtos(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + iVi := OpSpec{Proto: proto("i:i")} + bVb := OpSpec{Proto: proto("b:b")} + aaVa := OpSpec{Proto: proto("aa:a")} + aVaa := OpSpec{Proto: proto("a:aa")} + p, _, _ := mergeProtos(map[int]OpSpec{0: iVi, 1: bVb}) + require.Equal(t, proto("a:a"), p) + _, _, ok := mergeProtos(map[int]OpSpec{0: aaVa, 1: iVi}) + require.False(t, ok) + _, _, ok = mergeProtos(map[int]OpSpec{0: aVaa, 1: iVi}) + require.False(t, ok) + medley := OpSpec{Proto: proto("aibibabai:aibibabai")} + medley2 := OpSpec{Proto: proto("biabbaiia:biabbaiia")} + p, _, _ = mergeProtos(map[int]OpSpec{0: medley, 1: medley2}) + require.Equal(t, proto("aiaabaaaa:aiaabaaaa"), p) + v1 := OpSpec{Version: 1, Proto: proto(":")} + v2 := OpSpec{Version: 2, Proto: proto(":")} + _, v, _ := mergeProtos(map[int]OpSpec{0: v2, 1: v1}) + require.Equal(t, uint64(1), v) +} + +// Extra tests for features of getSpec that are currently not tested elsewhere +func TestGetSpec(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + ops := testProg(t, "int 1", AssemblerMaxVersion) + ops.versionedPseudoOps["dummyPseudo"] = make(map[int]OpSpec) + ops.versionedPseudoOps["dummyPseudo"][1] = OpSpec{Name: "b:", Version: AssemblerMaxVersion, Proto: proto("b:")} + ops.versionedPseudoOps["dummyPseudo"][2] = OpSpec{Name: ":", Version: AssemblerMaxVersion} + _, _, ok := getSpec(ops, "dummyPseudo", []string{}) + require.False(t, ok) + _, _, ok = getSpec(ops, "nonsense", []string{}) + require.False(t, ok) + require.Equal(t, 2, len(ops.Errors)) + require.Equal(t, "unknown opcode: nonsense", ops.Errors[1].Err.Error()) +} + +func TestAddPseudoDocTags(t *testing.T) { //nolint:paralleltest // Not parallel because it modifies pseudoOps and opDocByName which are global maps + partitiontest.PartitionTest(t) + defer func() { + delete(pseudoOps, "tests") + delete(opDocByName, "multiple") + delete(opDocByName, "single") + delete(opDocByName, "none") + delete(opDocByName, "any") + }() + + pseudoOps["tests"] = map[int]OpSpec{2: {Name: "multiple"}, 1: {Name: "single"}, 0: {Name: "none"}, anyImmediates: {Name: "any"}} + addPseudoDocTags() + require.Equal(t, "`multiple` can be called using `tests` with 2 immediates.", opDocByName["multiple"]) + require.Equal(t, "`single` can be called using `tests` with 1 immediate.", opDocByName["single"]) + require.Equal(t, "`none` can be called using `tests` with no immediates.", opDocByName["none"]) + require.Equal(t, "", opDocByName["any"]) +} +func TestReplacePseudo(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + replaceVersion := 7 + for v := uint64(replaceVersion); v <= AssemblerMaxVersion; v++ { + testProg(t, "byte 0x0000; byte 0x1234; replace 0", v) + testProg(t, "byte 0x0000; int 0; byte 0x1234; replace", v) + testProg(t, "byte 0x0000; byte 0x1234; replace", v, Expect{1, "replace without immediates expects 3 stack arguments but stack height is 2"}) + testProg(t, "byte 0x0000; int 0; byte 0x1234; replace 0", v, Expect{1, "replace 0 arg 0 wanted type []byte got uint64"}) + } +} + +func checkSame(t *testing.T, version uint64, first string, compares ...string) { + t.Helper() + if version == 0 { + version = assemblerNoVersion + } + ops := testProg(t, first, version) + for _, compare := range compares { + other := testProg(t, compare, version) + if bytes.Compare(other.Program, ops.Program) != 0 { + t.Log(Disassemble(ops.Program)) + t.Log(Disassemble(other.Program)) + } + assert.Equal(t, ops.Program, other.Program, "%s unlike %s", first, compare) + } +} + +func TestSemiColon(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + checkSame(t, AssemblerMaxVersion, + "pushint 0 ; pushint 1 ; +; int 3 ; *", + "pushint 0\npushint 1\n+\nint 3\n*", + "pushint 0; pushint 1; +; int 3; *; // comment; int 2", + "pushint 0; ; ; pushint 1 ; +; int 3 ; *//check", + ) + + checkSame(t, 0, + "#pragma version 7\nint 1", + "// junk;\n#pragma version 7\nint 1", + "// junk;\n #pragma version 7\nint 1", + ) + + checkSame(t, AssemblerMaxVersion, + `byte "test;this"; pop;`, + `byte "test;this"; ; pop;`, + `byte "test;this";;;pop;`, + ) +} + +func TestAssembleSwitch(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + // fail when target doesn't correspond to existing label + source := ` + pushint 1 + switch label1 label2 + label1: + ` + testProg(t, source, AssemblerMaxVersion, NewExpect(3, "reference to undefined label \"label2\"")) + + // fail when target index != uint64 + testProg(t, ` + byte "fail" + switch label1 + labe11: + `, AssemblerMaxVersion, Expect{3, "switch label1 arg 0 wanted type uint64..."}) + + // No labels is pretty degenerate, but ok, I suppose. It's just a no-op + testProg(t, ` +int 0 +switch +int 1 +`, AssemblerMaxVersion) + + // confirm arg limit + source = ` + pushint 1 + switch label1 label2 + label1: + label2: + ` + ops := testProg(t, source, AssemblerMaxVersion) + require.Len(t, ops.Program, 9) // ver (1) + pushint (2) + opcode (1) + length (1) + labels (2*2) + + var labels []string + for i := 0; i < 255; i++ { + labels = append(labels, fmt.Sprintf("label%d", i)) + } + + // test that 255 labels is ok + source = fmt.Sprintf(` + pushint 1 + switch %s + %s + `, strings.Join(labels, " "), strings.Join(labels, ":\n")+":\n") + ops = testProg(t, source, AssemblerMaxVersion) + require.Len(t, ops.Program, 515) // ver (1) + pushint (2) + opcode (1) + length (1) + labels (2*255) + + // 256 is too many + source = fmt.Sprintf(` + pushint 1 + switch %s extra + %s + `, strings.Join(labels, " "), strings.Join(labels, ":\n")+":\n") + testProg(t, source, AssemblerMaxVersion, Expect{3, "switch cannot take more than 255 labels"}) + + // allow duplicate label reference + source = ` + pushint 1 + switch label1 label1 + label1: + ` + testProg(t, source, AssemblerMaxVersion) +} + +func TestMacros(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + checkSame(t, AssemblerMaxVersion, ` + pushint 0; pushint 1; +`, ` + #define none 0 + #define one 1 + pushint none; pushint one; +`, + ) + + checkSame(t, AssemblerMaxVersion, ` + pushint 1 + pushint 2 + == + bnz label1 + err + label1: + pushint 1`, ` + #define ==? ==; bnz + pushint 1; pushint 2; ==? label1 + err + label1: + pushint 1`, + ) + + // Test redefining macros with macro chaining works + checkSame(t, AssemblerMaxVersion, ` + pushbytes 0x100000000000; substring 3 5; substring 0 1`, ` + #define rowSize 3 + #define columnSize 5 + #define tableDimensions rowSize columnSize + pushbytes 0x100000000000; substring tableDimensions + #define rowSize 0 + #define columnSize 1 + substring tableDimensions`, + ) + + // Test more complicated macros like multi-token + checkSame(t, AssemblerMaxVersion, ` + int 3 + store 0 + int 4 + store 1 + load 0 + load 1 + <`, ` + #define &x 0 + #define x load &x; + #define &y 1 + #define y load &y; + #define -> ; store + int 3 -> &x; int 4 -> &y + x y <`, + ) + + checkSame(t, AssemblerMaxVersion, ` + pushbytes 0xddf2554d + txna ApplicationArgs 0 + == + bnz kickstart + pushbytes 0x903f4535 + txna ApplicationArgs 0 + == + bnz portal_transfer + kickstart: + pushint 1 + portal_transfer: + pushint 1 + `, ` + #define abi-route txna ApplicationArgs 0; ==; bnz + method "kickstart(account)void"; abi-route kickstart + method "portal_transfer(byte[])byte[]"; abi-route portal_transfer + kickstart: + pushint 1 + portal_transfer: + pushint 1 + `) + + checkSame(t, AssemblerMaxVersion, ` +method "echo(string)string" +txn ApplicationArgs 0 +== +bnz echo + +echo: + int 1 + dup + txnas ApplicationArgs + extract 2 0 + stores + + int 1 + loads + dup + len + itob + extract 6 0 + swap + concat + + pushbytes 0x151f7c75 + swap + concat + log + int 1 + return + +method "add(uint32,uint32)uint32" +txn ApplicationArgs 0 +== +bnz add + +add: + int 1 + dup + txnas ApplicationArgs + int 0 + extract_uint32 + stores + + int 2 + dup + txnas ApplicationArgs + int 0 + extract_uint32 + stores + + load 1; load 2; + + store 255 + + int 255 + loads + itob + extract 4 0 + pushbytes 0x151f7c75 + swap + concat + + log + int 1 + return + `, ` +// Library Methods + +// codecs +#define abi-encode-uint16 ;itob; extract 6 0; +#define abi-decode-uint16 ;extract_uint16; + +#define abi-decode-uint32 ;int 0; extract_uint32; +#define abi-encode-uint32 ;itob;extract 4 0; + +#define abi-encode-bytes ;dup; len; abi-encode-uint16; swap; concat; +#define abi-decode-bytes ;extract 2 0; + +// abi method handling +#define abi-route ;txna ApplicationArgs 0; ==; bnz +#define abi-return ;pushbytes 0x151f7c75; swap; concat; log; int 1; return; + +// stanza: "set $var from-{type}" +#define parse ; int +#define read_arg ;dup; txnas ApplicationArgs; +#define from-string ;read_arg; abi-decode-bytes; stores; +#define from-uint16 ;read_arg; abi-decode-uint16; stores; +#define from-uint32 ;read_arg; abi-decode-uint32; stores; + +// stanza: "reply $var as-{type} +#define returns ; int +#define as-uint32; loads; abi-encode-uint32; abi-return; +#define as-string; loads; abi-encode-bytes; abi-return; + +// Contract + +// echo handler +method "echo(string)string"; abi-route echo +echo: + #define msg 1 + parse msg from-string + + // cool things happen ... + + returns msg as-string + + +// add handler +method "add(uint32,uint32)uint32"; abi-route add +add: + #define x 1 + parse x from-uint32 + + #define y 2 + parse y from-uint32 + + #define sum 255 + load x; load y; +; store sum + + returns sum as-uint32 + `) + + testProg(t, ` + #define x a d + #define d c a + #define hey wat's up x + #define c woah hey + int 1 + c`, + AssemblerMaxVersion, Expect{5, "Macro cycle discovered: c -> hey -> x -> d -> c"}, Expect{7, "unknown opcode: c"}, + ) + + testProg(t, ` + #define c + + #define x a c + #define d x + #define c d + int 1 + c`, + AssemblerMaxVersion, Expect{5, "Macro cycle discovered: c -> d -> x -> c"}, Expect{7, "+ expects..."}, + ) + + testProg(t, ` + #define X X + int 3`, + AssemblerMaxVersion, Expect{2, "Macro cycle discovered: X -> X"}, + ) + + // Check that macros names can't be things like named constants, opcodes, etc. + // If pragma is given, only macros that violate that version's stuff should be errored on + testProg(t, ` + #define return random + #define pay randomm + #define NoOp randommm + #define + randommmm + #pragma version 1 // now the versioned check should activate and check all previous macros + #define return hi // no error b/c return is after v1 + #define + hey // since versioned check is now online, we can error here + int 1`, + assemblerNoVersion, + Expect{3, "Named constants..."}, + Expect{4, "Named constants..."}, + Expect{6, "Macro names cannot be opcodes: +"}, + Expect{8, "Macro names cannot be opcodes: +"}, + ) + + // Same check, but this time since no version is given, the versioned check + // uses AssemblerDefaultVersion and activates on first instruction (int 1) + testProg(t, ` + #define return random + #define pay randomm + #define NoOp randommm + #define + randommmm + int 1 // versioned check activates here + #define return hi + #define + hey`, + assemblerNoVersion, + Expect{3, "Named constants..."}, + Expect{4, "Named constants..."}, + Expect{6, "Macro names cannot be opcodes: +"}, + Expect{8, "Macro names cannot be opcodes: +"}, + ) + + testProg(t, ` + #define Sender hello + #define ApplicationArgs hiya + #pragma version 1 + #define Sender helllooooo + #define ApplicationArgs heyyyyy // no error b/c ApplicationArgs is after v1 + int 1`, + assemblerNoVersion, + Expect{4, "Macro names cannot be field names: Sender"}, // error happens once version is known + ) + + // Same check but defaults to AssemblerDefaultVersion instead of pragma + testProg(t, ` + #define Sender hello + #define ApplicationArgs hiya + int 1 + #define Sender helllooooo + #define ApplicationArgs heyyyyy`, + assemblerNoVersion, + Expect{4, "Macro names cannot be field names: Sender"}, // error happens once version is auto-set + Expect{5, "Macro names cannot be field names: Sender"}, // and on following line + ) + // define needs name and body + testLine(t, "#define", AssemblerMaxVersion, "define directive requires a name and body") + testLine(t, "#define hello", AssemblerMaxVersion, "define directive requires a name and body") + // macro names cannot be directives + testLine(t, "#define #define 1", AssemblerMaxVersion, "# character not allowed in macro name") + testLine(t, "#define #pragma 1", AssemblerMaxVersion, "# character not allowed in macro name") + // macro names cannot begin with digits (including negative ones) + testLine(t, "#define 1hello one", AssemblerMaxVersion, "Cannot begin macro name with number: 1hello") + testLine(t, "#define -1hello negativeOne", AssemblerMaxVersion, "Cannot begin macro name with number: -1hello") + // macro names can't use base64/32 notation + testLine(t, "#define b64 AA", AssemblerMaxVersion, "Cannot use b64 as macro name") + testLine(t, "#define base64 AA", AssemblerMaxVersion, "Cannot use base64 as macro name") + testLine(t, "#define b32 AA", AssemblerMaxVersion, "Cannot use b32 as macro name") + testLine(t, "#define base32 AA", AssemblerMaxVersion, "Cannot use base32 as macro name") + // macro names can't use non-alphanumeric characters that aren't specifically allowed + testLine(t, "#define wh@t 1", AssemblerMaxVersion, "@ character not allowed in macro name") + // check both kinds of pseudo-ops to make sure they can't be used as macro names + testLine(t, "#define int 3", AssemblerMaxVersion, "Macro names cannot be pseudo-ops: int") + testLine(t, "#define extract 3", AssemblerMaxVersion, "Macro names cannot be pseudo-ops: extract") + // check labels to make sure they can't be used as macro names + testProg(t, ` + coolLabel: + int 1 + #define coolLabel 1`, + AssemblerMaxVersion, + Expect{4, "Labels cannot be used as macro names: coolLabel"}, + ) + testProg(t, ` + #define coolLabel 1 + coolLabel: + int 1`, + AssemblerMaxVersion, + Expect{3, "Cannot create label with same name as macro: coolLabel"}, + ) + // Admittedly these two tests are just for coverage + ops := newOpStream(AssemblerMaxVersion) + err := define(&ops, []string{"not#define"}) + require.EqualError(t, err, "0: invalid syntax: not#define") + err = pragma(&ops, []string{"not#pragma"}) + require.EqualError(t, err, "0: invalid syntax: not#pragma") +} + +func TestAssembleImmediateRanges(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + /* Perhaps all of these "unable to parse" errors could be improved to + discuss limits rather than bailout when the immediate is, in fact, an + integer. */ + + testProg(t, "int 1; store 0;", AssemblerMaxVersion) + testProg(t, "load 255;", AssemblerMaxVersion) + + testProg(t, "int 1; store -1000;", AssemblerMaxVersion, + Expect{1, "store unable to parse..."}) + testProg(t, "load -100;", AssemblerMaxVersion, + Expect{1, "load unable to parse..."}) + testProg(t, "int 1; store 256;", AssemblerMaxVersion, + Expect{1, "store i beyond 255: 256"}) + + testProg(t, "frame_dig -1;", AssemblerMaxVersion) + testProg(t, "frame_dig 127;", AssemblerMaxVersion) + testProg(t, "int 1; frame_bury -128;", AssemblerMaxVersion) + + testProg(t, "frame_dig 128;", AssemblerMaxVersion, + Expect{1, "frame_dig unable to parse..."}) + testProg(t, "int 1; frame_bury -129;", AssemblerMaxVersion, + Expect{1, "frame_bury unable to parse..."}) +} + +func TestAssembleMatch(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + // fail when target doesn't correspond to existing label + source := ` + pushints 1 1 1 + match label1 label2 + label1: + ` + testProg(t, source, AssemblerMaxVersion, NewExpect(3, "reference to undefined label \"label2\"")) + + // No labels is pretty degenerate, but ok, I suppose. It's just a no-op + testProg(t, ` +int 0 +match +int 1 +`, AssemblerMaxVersion) + + // confirm arg limit + source = ` + pushints 1 2 1 + match label1 label2 + label1: + label2: + ` + ops := testProg(t, source, AssemblerMaxVersion) + require.Len(t, ops.Program, 12) // ver (1) + pushints (5) + opcode (1) + length (1) + labels (2*2) + + // confirm byte array args are assembled successfully + source = ` + pushbytess "1" "2" "1" + match label1 label2 + label1: + label2: + ` + testProg(t, source, AssemblerMaxVersion) + + var labels []string + for i := 0; i < 255; i++ { + labels = append(labels, fmt.Sprintf("label%d", i)) + } + + // test that 255 labels is ok + source = fmt.Sprintf(` + pushint 1 + match %s + %s + `, strings.Join(labels, " "), strings.Join(labels, ":\n")+":\n") + ops = testProg(t, source, AssemblerMaxVersion) + require.Len(t, ops.Program, 515) // ver (1) + pushint (2) + opcode (1) + length (1) + labels (2*255) + + // 256 is too many + source = fmt.Sprintf(` + pushint 1 + match %s extra + %s + `, strings.Join(labels, " "), strings.Join(labels, ":\n")+":\n") + testProg(t, source, AssemblerMaxVersion, Expect{3, "match cannot take more than 255 labels"}) + + // allow duplicate label reference + source = ` + pushint 1 + match label1 label1 + label1: + ` + testProg(t, source, AssemblerMaxVersion) +} + +func TestAssemblePushConsts(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + // allow empty const int list + source := `pushints` + testProg(t, source, AssemblerMaxVersion) + + // allow empty const bytes list + source = `pushbytess` + testProg(t, source, AssemblerMaxVersion) + + // basic test + source = `pushints 1 2 3` + ops := testProg(t, source, AssemblerMaxVersion) + require.Len(t, ops.Program, 6) // ver (1) + pushints (5) + source = `pushbytess "1" "2" "33"` + ops = testProg(t, source, AssemblerMaxVersion) + require.Len(t, ops.Program, 10) // ver (1) + pushbytess (9) + + // 256 increases size of encoded length to two bytes + valsStr := make([]string, 256) + for i := range valsStr { + valsStr[i] = fmt.Sprintf("%d", 1) + } + source = fmt.Sprintf(`pushints %s`, strings.Join(valsStr, " ")) + ops = testProg(t, source, AssemblerMaxVersion) + require.Len(t, ops.Program, 260) // ver (1) + opcode (1) + len (2) + ints (256) + + for i := range valsStr { + valsStr[i] = fmt.Sprintf("\"%d\"", 1) + } + source = fmt.Sprintf(`pushbytess %s`, strings.Join(valsStr, " ")) + ops = testProg(t, source, AssemblerMaxVersion) + require.Len(t, ops.Program, 516) // ver (1) + opcode (1) + len (2) + bytess (512) + + // enforce correct types + source = `pushints "1" "2" "3"` + testProg(t, source, AssemblerMaxVersion, Expect{1, `strconv.ParseUint: parsing "\"1\"": invalid syntax`}) + source = `pushbytess 1 2 3` + testProg(t, source, AssemblerMaxVersion, Expect{1, "byte arg did not parse: 1"}) + source = `pushints 6 4; concat` + testProg(t, source, AssemblerMaxVersion, Expect{1, "concat arg 1 wanted type []byte got uint64"}) + source = `pushbytess "x" "y"; +` + testProg(t, source, AssemblerMaxVersion, Expect{1, "+ arg 1 wanted type uint64 got []byte"}) +} + +func TestAssembleEmpty(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + emptyExpect := Expect{0, "Cannot assemble empty program text"} + emptyPrograms := []string{ + "", + " ", + " \n\t\t\t\n\n ", + " \n \t \t \t \n \n \n\n", + } + + nonEmpty := " \n \t \t \t int 1 \n \n \t \t \n\n" + + for version := uint64(1); version <= AssemblerMaxVersion; version++ { + for _, prog := range emptyPrograms { + testProg(t, prog, version, emptyExpect) + } + testProg(t, nonEmpty, version) + } +} + +func TestReportMultipleErrors(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + assertWithMsg := func(t *testing.T, expectedOutput string, b bytes.Buffer) { + if b.String() != expectedOutput { + t.Errorf("Unexpected output: got %q, want %q", b.String(), expectedOutput) + } + } + + ops := &OpStream{ + Errors: []lineError{ + {Line: 1, Err: errors.New("error 1")}, + {Err: errors.New("error 2")}, + {Line: 3, Err: errors.New("error 3")}, + }, + Warnings: []error{ + errors.New("warning 1"), + errors.New("warning 2"), + }, + } + + // Test the case where fname is not empty + var b bytes.Buffer + ops.ReportMultipleErrors("test.txt", &b) + expected := `test.txt: 1: error 1 +test.txt: 0: error 2 +test.txt: 3: error 3 +test.txt: warning 1 +test.txt: warning 2 +` + assertWithMsg(t, expected, b) + + // Test the case where fname is empty + b.Reset() + ops.ReportMultipleErrors("", &b) + expected = `1: error 1 +0: error 2 +3: error 3 +warning 1 +warning 2 +` + assertWithMsg(t, expected, b) + + // no errors or warnings at all + ops = &OpStream{} + b.Reset() + ops.ReportMultipleErrors("blah blah", &b) + expected = "" + assertWithMsg(t, expected, b) + + // more than 10 errors: + file := "great-file.go" + les := []lineError{} + expectedStrs := []string{} + for i := 1; i <= 11; i++ { + errS := fmt.Errorf("error %d", i) + les = append(les, lineError{i, errS}) + if i <= 10 { + expectedStrs = append(expectedStrs, fmt.Sprintf("%s: %d: %s", file, i, errS)) + } + } + expected = strings.Join(expectedStrs, "\n") + "\n" + ops = &OpStream{Errors: les} + b.Reset() + ops.ReportMultipleErrors(file, &b) + assertWithMsg(t, expected, b) + + // exactly 1 error + filename + ops = &OpStream{Errors: []lineError{{42, errors.New("super annoying error")}}} + b.Reset() + ops.ReportMultipleErrors("galaxy.py", &b) + expected = "galaxy.py: 1 error: 42: super annoying error\n" + assertWithMsg(t, expected, b) + + // exactly 1 error w/o filename + ops = &OpStream{Errors: []lineError{{42, errors.New("super annoying error")}}} + b.Reset() + ops.ReportMultipleErrors("", &b) + expected = "1 error: 42: super annoying error\n" + assertWithMsg(t, expected, b) +} + +// TestDisassembleBadBranch ensures a clean error when a branch has no target. +func TestDisassembleBadBranch(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + for _, br := range []byte{0x40, 0x41, 0x42} { + dis, err := Disassemble([]byte{2, br}) + require.Error(t, err, dis) + dis, err = Disassemble([]byte{2, br, 0x01}) + require.Error(t, err, dis) + + // It would be reasonable to error here, since it's a jump past the end. + dis, err = Disassemble([]byte{2, br, 0x00, 0x05}) + require.NoError(t, err, dis) + + // It would be reasonable to error here, since it's a back jump in v2. + dis, err = Disassemble([]byte{2, br, 0xff, 0x02}) + require.NoError(t, err, dis) + + dis, err = Disassemble([]byte{2, br, 0x00, 0x01, 0x00}) + require.NoError(t, err) + } +} + +// TestDisassembleBadSwitch ensures a clean error when a switch ends early +func TestDisassembleBadSwitch(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + source := ` + int 1 + switch label1 label2 + label1: + label2: + ` + ops, err := AssembleStringWithVersion(source, AssemblerMaxVersion) + require.NoError(t, err) + + dis, err := Disassemble(ops.Program) + require.NoError(t, err, dis) + + // chop off all the labels, but keep the label count + dis, err = Disassemble(ops.Program[:len(ops.Program)-4]) + require.ErrorContains(t, err, "could not decode labels for switch", dis) + + // chop off before the label count + dis, err = Disassemble(ops.Program[:len(ops.Program)-5]) + require.ErrorContains(t, err, "could not decode label count for switch", dis) + + // chop off half of a label + dis, err = Disassemble(ops.Program[:len(ops.Program)-1]) + require.ErrorContains(t, err, "could not decode labels for switch", dis) +} + +// TestDisassembleBadMatch ensures a clean error when a match ends early +func TestDisassembleBadMatch(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + source := ` + int 40 + int 45 + int 40 + match label1 label2 + label1: + label2: + ` + ops, err := AssembleStringWithVersion(source, AssemblerMaxVersion) + require.NoError(t, err) + + dis, err := Disassemble(ops.Program) + require.NoError(t, err, dis) + + // return the label count, but chop off the labels themselves + dis, err = Disassemble(ops.Program[:len(ops.Program)-5]) + require.ErrorContains(t, err, "could not decode label count for match", dis) + + dis, err = Disassemble(ops.Program[:len(ops.Program)-1]) + require.ErrorContains(t, err, "could not decode labels for match", dis) +}