Merge remote-tracking branch 'origin/master' into folower-node-test-f…

…lakiness

.circleci/config.yml

@@ -22,6 +22,11 @@ parameters:
   valid_nightly_branch:
     type: string
     default: /hotfix\/.*/
+  # The following is intentional - hardcoding a token for public repos
+  # is recommended here to allow fork access
+  codecov:
+    type: string
+    default: "8b4a1f91-f154-4c26-b84c-c9aaa90159c6"
 
 executors:
   amd64_medium:
@@ -239,6 +244,8 @@ jobs:
     executor: << parameters.platform >>_medium
     working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 32
+    environment:
+      CODECOV_TOKEN: << pipeline.parameters.codecov >>
     steps:
       - generic_build
       - generic_test:
@@ -254,6 +261,8 @@ jobs:
     executor: << parameters.platform >>_large
     working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
+    environment:
+      CODECOV_TOKEN: << pipeline.parameters.codecov >>
     steps:
       - generic_build
       - generic_test:

catchup/service.go

@@ -157,25 +157,31 @@ func (s *Service) IsSynchronizing() (synchronizing bool, initialSync bool) {
 	return
 }
 
+// triggerSync attempts to wake up the sync loop.
+func (s *Service) triggerSync() {
+	if syncing, initial := s.IsSynchronizing(); !syncing && !initial {
+		select {
+		case s.syncNow <- struct{}{}:
+		default:
+		}
+	}
+}
+
 // SetDisableSyncRound attempts to set the first round we _do_not_ want to fetch from the network
 // Blocks from disableSyncRound or any round after disableSyncRound will not be fetched while this is set
 func (s *Service) SetDisableSyncRound(rnd uint64) error {
 	if basics.Round(rnd) < s.ledger.LastRound() {
 		return ErrSyncRoundInvalid
 	}
 	atomic.StoreUint64(&s.disableSyncRound, rnd)
-	if syncing, initial := s.IsSynchronizing(); !syncing && !initial {
-		s.syncNow <- struct{}{}
-	}
+	s.triggerSync()
 	return nil
 }
 
 // UnsetDisableSyncRound removes any previously set disabled sync round
 func (s *Service) UnsetDisableSyncRound() {
 	atomic.StoreUint64(&s.disableSyncRound, 0)
-	if syncing, initial := s.IsSynchronizing(); !syncing && !initial {
-		s.syncNow <- struct{}{}
-	}
+	s.triggerSync()
 }
 
 // GetDisableSyncRound returns the disabled sync round

cmd/goal/README.md

@@ -68,7 +68,7 @@ TEALDIR=cmd/goal/examples
 echo $TEALDIR
 
 # create the app and TAKE NOTE of its "app index"
-goal app create --creator ${ACCOUNT} --approval-prog ${TEALDIR}/boxes.teal --clear-prog ${TEALDIR}/clear.teal --global-byteslices 0 --global-ints 0 --local-byteslices 0 --local-ints 0
+goal app create --creator ${ACCOUNT} --approval-prog ${TEALDIR}/boxes.teal --clear-prog ${TEALDIR}/clear.teal
 ```
 
 For the following questions, you'll need to use the app index. That will be shown in the last line printed. EG:

cmd/goal/application.go

@@ -163,10 +163,6 @@ func init() {
 	readStateAppCmd.Flags().BoolVar(&guessFormat, "guess-format", false, "Format application state using heuristics to guess data encoding.")
 
 	createAppCmd.MarkFlagRequired("creator")
-	createAppCmd.MarkFlagRequired("global-ints")
-	createAppCmd.MarkFlagRequired("global-byteslices")
-	createAppCmd.MarkFlagRequired("local-ints")
-	createAppCmd.MarkFlagRequired("local-byteslices")
 
 	optInAppCmd.MarkFlagRequired("app-id")
 	optInAppCmd.MarkFlagRequired("from")

config/consensus.go

@@ -498,6 +498,9 @@ type ConsensusParams struct {
 	// EnableCatchpointsWithSPContexts specifies when to re-enable version 7 catchpoints.
 	// Version 7 includes state proof verification contexts
 	EnableCatchpointsWithSPContexts bool
+
+	// EnableBoxRefNameError specifies that box ref names should be validated early
+	EnableBoxRefNameError bool
 }
 
 // PaysetCommitType enumerates possible ways for the block header to commit to
@@ -1266,6 +1269,7 @@ func initConsensusProtocols() {
 	vFuture.LogicSigVersion = 9 // When moving this to a release, put a new higher LogicSigVersion here
 	vFuture.EnablePrecheckECDSACurve = true
 	vFuture.EnableBareBudgetError = true
+	vFuture.EnableBoxRefNameError = true
 
 	vFuture.StateProofUseTrackerVerification = true
 	vFuture.EnableCatchpointsWithSPContexts = true

daemon/algod/api/server/v2/handlers.go

@@ -1348,8 +1348,6 @@ func (v2 *Handlers) startCatchup(ctx echo.Context, catchpoint string) error {
 		code = http.StatusOK
 	case *node.CatchpointUnableToStartError:
 		return badRequest(ctx, err, err.Error(), v2.Log)
-	case *node.CatchpointSyncRoundFailure:
-		return badRequest(ctx, err, fmt.Sprintf(errFailedToStartCatchup, err), v2.Log)
 	default:
 		return internalError(ctx, err, fmt.Sprintf(errFailedToStartCatchup, err), v2.Log)
 	}

daemon/algod/api/server/v2/test/handlers_test.go

@@ -1222,10 +1222,6 @@ func TestStartCatchup(t *testing.T) {
 
 	badCatchPoint := "bad catchpoint"
 	startCatchupTest(t, badCatchPoint, nil, 400)
-
-	// Test that a catchup fails w/ 400 when the catchpoint round is > syncRound (while syncRound is set)
-	syncRoundError := node.MakeCatchpointSyncRoundFailure(goodCatchPoint, 1)
-	startCatchupTest(t, goodCatchPoint, syncRoundError, 400)
 }
 
 func abortCatchupTest(t *testing.T, catchpoint string, expectedCode int) {

daemon/algod/server.go

@@ -83,7 +83,6 @@ func (s *Server) Initialize(cfg config.Local, phonebookAddresses []string, genes
 
 	liveLog := filepath.Join(s.RootPath, "node.log")
 	archive := filepath.Join(s.RootPath, cfg.LogArchiveName)
-	fmt.Println("Logging to: ", liveLog)
 	var maxLogAge time.Duration
 	var err error
 	if cfg.LogArchiveMaxAge != "" {
@@ -96,8 +95,10 @@ func (s *Server) Initialize(cfg config.Local, phonebookAddresses []string, genes
 
 	var logWriter io.Writer
 	if cfg.LogSizeLimit > 0 {
+		fmt.Println("Logging to: ", liveLog)
 		logWriter = logging.MakeCyclicFileWriter(liveLog, archive, cfg.LogSizeLimit, maxLogAge)
 	} else {
+		fmt.Println("Logging to: stdout")
 		logWriter = os.Stdout
 	}
 	s.log.SetOutput(logWriter)

data/transactions/transaction.go

@@ -477,6 +477,9 @@ func (tx Transaction) WellFormed(spec SpecialAddresses, proto config.ConsensusPa
 			if br.Index > uint64(len(tx.ForeignApps)) {
 				return fmt.Errorf("tx.Boxes[%d].Index is %d. Exceeds len(tx.ForeignApps)", i, br.Index)
 			}
+			if proto.EnableBoxRefNameError && len(br.Name) > proto.MaxAppKeyLen {
+				return fmt.Errorf("tx.Boxes[%d].Name too long, max len %d bytes", i, proto.MaxAppKeyLen)
+			}
 		}
 
 		if tx.LocalStateSchema.NumEntries() > proto.MaxLocalSchemaEntries {

data/transactions/transaction_test.go

@@ -274,6 +274,7 @@ func TestWellFormedErrors(t *testing.T) {
 	protoV27 := config.Consensus[protocol.ConsensusV27]
 	protoV28 := config.Consensus[protocol.ConsensusV28]
 	protoV32 := config.Consensus[protocol.ConsensusV32]
+	protoV36 := config.Consensus[protocol.ConsensusV36]
 	addr1, err := basics.UnmarshalChecksumAddress("NDQCJNNY5WWWFLP4GFZ7MEF2QJSMZYK6OWIV2AQ7OMAVLEFCGGRHFPKJJA")
 	require.NoError(t, err)
 	v5 := []byte{0x05}
@@ -566,6 +567,32 @@ func TestWellFormedErrors(t *testing.T) {
 			proto:         protoV32,
 			expectedError: fmt.Errorf("tx.Boxes too long, max number of box references is 0"),
 		},
+		{
+			tx: Transaction{
+				Type:   protocol.ApplicationCallTx,
+				Header: okHeader,
+				ApplicationCallTxnFields: ApplicationCallTxnFields{
+					ApplicationID: 1,
+					Boxes:         []BoxRef{{Index: 1, Name: make([]byte, 65)}},
+					ForeignApps:   []basics.AppIndex{1},
+				},
+			},
+			proto:         futureProto,
+			expectedError: fmt.Errorf("tx.Boxes[0].Name too long, max len 64 bytes"),
+		},
+		{
+			tx: Transaction{
+				Type:   protocol.ApplicationCallTx,
+				Header: okHeader,
+				ApplicationCallTxnFields: ApplicationCallTxnFields{
+					ApplicationID: 1,
+					Boxes:         []BoxRef{{Index: 1, Name: make([]byte, 65)}},
+					ForeignApps:   []basics.AppIndex{1},
+				},
+			},
+			proto:         protoV36,
+			expectedError: nil,
+		},
 	}
 	for _, usecase := range usecases {
 		err := usecase.tx.WellFormed(specialAddr, usecase.proto)

node/error.go

@@ -62,24 +62,3 @@ func (e *CatchpointUnableToStartError) Error() string {
 		e.catchpointRequested,
 		e.catchpointRunning)
 }
-
-// CatchpointSyncRoundFailure indicates that the requested catchpoint is beyond the currently set sync round
-type CatchpointSyncRoundFailure struct {
-	catchpoint string
-	syncRound  uint64
-}
-
-// MakeCatchpointSyncRoundFailure creates the error type
-func MakeCatchpointSyncRoundFailure(catchpoint string, syncRound uint64) *CatchpointSyncRoundFailure {
-	return &CatchpointSyncRoundFailure{
-		catchpoint: catchpoint,
-		syncRound:  syncRound,
-	}
-}
-
-// Error satisfies the builtin `error` interface
-func (e *CatchpointSyncRoundFailure) Error() string {
-	return fmt.Sprintf(
-		"unable to start catchpoint catchup for '%s' - resulting round is beyond current sync round '%v'",
-		e.catchpoint, e.syncRound)
-}

node/follower_node.go

@@ -342,14 +342,7 @@ func (node *AlgorandFollowerNode) StartCatchup(catchpoint string) error {
 		}
 		return MakeCatchpointUnableToStartError(stats.CatchpointLabel, catchpoint)
 	}
-	cpRound, _, err := ledgercore.ParseCatchpointLabel(catchpoint)
-	if err != nil {
-		return err
-	}
-	sRound := node.GetSyncRound()
-	if sRound > 0 && uint64(cpRound) > sRound {
-		return MakeCatchpointSyncRoundFailure(catchpoint, sRound)
-	}
+	var err error
 	accessor := ledger.MakeCatchpointCatchupAccessor(node.ledger.Ledger, node.log)
 	node.catchpointCatchupService, err = catchup.MakeNewCatchpointCatchupService(catchpoint, node, node.log, node.net, accessor, node.config)
 	if err != nil {
@@ -414,7 +407,15 @@ func (node *AlgorandFollowerNode) SetCatchpointCatchupMode(catchpointCatchupMode
 			prevNodeCancelFunc()
 			return
 		}
+
+		// Catchup finished, resume.
 		defer node.mu.Unlock()
+
+		// update sync round before starting services
+		if err := node.SetSyncRound(uint64(node.ledger.LastRound())); err != nil {
+			node.log.Warnf("unable to set sync round while resuming fast catchup: %v", err)
+		}
+
 		// start
 		node.catchupService.Start()
 		node.blockService.Start()

node/follower_node_test.go

@@ -17,11 +17,13 @@
 package node
 
 import (
+	"context"
 	"testing"
 	"time"
 
 	"github.com/sirupsen/logrus"
 	"github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/algorand/go-algorand/agreement"
@@ -227,3 +229,22 @@ func TestSyncRoundWithRemake(t *testing.T) {
 	syncRound := followNode.GetSyncRound()
 	require.Equal(t, uint64(maxAcctLookback+1), syncRound)
 }
+
+func TestFastCatchupResume(t *testing.T) {
+	partitiontest.PartitionTest(t)
+	t.Parallel()
+	node := setupFollowNode(t)
+	node.ctx = context.Background()
+
+	// Initialize sync round to a future round.
+	syncRound := uint64(10000)
+	node.SetSyncRound(syncRound)
+	require.Equal(t, syncRound, node.GetSyncRound())
+
+	// Force catchpoint catchup mode to end, this should set the sync round to the current ledger round (0).
+	out := node.SetCatchpointCatchupMode(false)
+	<-out
+
+	// Verify the sync was reset.
+	assert.Equal(t, uint64(0), node.GetSyncRound())
+}

scripts/travis/codecov

@@ -5,7 +5,7 @@
 
 set -e +o pipefail
 
-VERSION="1.0.3"
+VERSION="1.0.6"
 
 codecov_flags=( )
 url="https://codecov.io"
@@ -865,14 +865,17 @@ then
   if [  "$GITHUB_HEAD_REF" != "" ];
   then
     # PR refs are in the format: refs/pull/7/merge
-    pr="${GITHUB_REF#refs/pull/}"
-    pr="${pr%/merge}"
+    if [[ "$GITHUB_REF" =~ ^refs\/pull\/[0-9]+\/merge$ ]];
+    then
+      pr="${GITHUB_REF#refs/pull/}"
+      pr="${pr%/merge}"
+    fi
     branch="${GITHUB_HEAD_REF}"
   fi
   commit="${GITHUB_SHA}"
   slug="${GITHUB_REPOSITORY}"
   build="${GITHUB_RUN_ID}"
-  build_url=$(urlencode "http://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}")
+  build_url=$(urlencode "${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}")
   job="$(urlencode "${GITHUB_WORKFLOW}")"
 
   # actions/checkout runs in detached HEAD
@@ -987,6 +990,7 @@ else
 
 fi
 
+say "    ${e}current dir: ${x} $PWD"
 say "    ${e}project root:${x} $git_root"
 
 # find branch, commit, repo from git command
@@ -1618,7 +1622,7 @@ then
   # [ or ]
   syntax_list='^[[:space:]]*[][][[:space:]]*(//.*)?$'
   # func ... {
-  syntax_go_func='^[[:space:]]*[func].*[\{][[:space:]]*$'
+  syntax_go_func='^[[:space:]]*func[[:space:]]*[\{][[:space:]]*$'
 
   # shellcheck disable=SC2089
   skip_dirs="-not -path '*/$bower_components/*' \
@@ -1783,7 +1787,7 @@ if [ "$dump" != "0" ];
 then
   # trim whitespace from query
   say "    ${e}->${x} Dumping upload file (no upload)"
-  echo "$url/upload/v4?$(echo "package=$package-$VERSION&token=$token&$query" | tr -d ' ')"
+  echo "$url/upload/v4?$(echo "package=$package-$VERSION&$query" | tr -d ' ')"
   cat "$upload_file"
 else
   if [ "$save_to" != "" ];
@@ -1802,10 +1806,9 @@ else
   say "    ${e}url:${x} $url"
   say "    ${e}query:${x} $query"
 
-  # Full query without token (to display on terminal output)
-  queryNoToken=$(echo "package=$package-$VERSION&token=secret&$query" | tr -d ' ')
-  # now add token to query
+  # Full query (to display on terminal output)
   query=$(echo "package=$package-$VERSION&token=$token&$query" | tr -d ' ')
+  queryNoToken=$(echo "package=$package-$VERSION&token=<hidden>&$query" | tr -d ' ')
 
   if [ "$ft_s3" = "1" ];
   then
@@ -1817,6 +1820,7 @@ else
           -H 'X-Reduced-Redundancy: false' \
           -H 'X-Content-Type: application/x-gzip' \
           -H 'Content-Length: 0' \
+          -H "X-Upload-Token: ${token}" \
           --write-out "\n%{response_code}\n" \
           $curlargs \
           "$url/upload/v4?$query" || true)
@@ -1863,6 +1867,7 @@ else
         -H 'Content-Type: text/plain' \
         -H 'Content-Encoding: gzip' \
         -H 'X-Content-Encoding: gzip' \
+        -H "X-Upload-Token: ${token}" \
         -H 'Accept: text/plain' \
         $curlargs \
         "$url/upload/v2?$query&attempt=$i" || echo 'HTTP 500')

scripts/travis/upload_coverage.sh

@@ -1,15 +1,9 @@
 #!/usr/bin/env bash
 
-# Print a warning when there is a new version notification before uploading the
-# coverage report to codecov.
 set -eo pipefail
 
-# Check if there is a new version.
-curl -fLso codecov https://codecov.io/bash
-UPSTREAM_VERSION=$(grep -o 'VERSION=\"[0-9\.]*\"' codecov | cut -d'"' -f2)
-LOCAL_VERSION=$(grep -o 'VERSION=\"[0-9\.]*\"' scripts/travis/codecov | cut -d'"' -f2)
-if [[ "${UPSTREAM_VERSION}" != "${LOCAL_VERSION}" ]]; then
-  echo "WARN: version ${UPSTREAM_VERSION} of the codecov upload script is available."
+if [[ -z "$CODECOV_TOKEN" ]]; then
+  /usr/bin/env bash scripts/travis/codecov
+else
+  /usr/bin/env bash scripts/travis/codecov -t $CODECOV_TOKEN
 fi
-
-/usr/bin/env bash scripts/travis/codecov