Merge pull request #4885 from Algo-devops-service/relbeta3.13.0

Dockerfile

@@ -0,0 +1,75 @@
+ARG GO_VERSION=1.17.5
+FROM golang:$GO_VERSION-bullseye as builder
+
+ARG CHANNEL=nightly
+ARG URL=
+ARG BRANCH=
+ARG SHA=
+
+# Basic dependencies.
+ENV HOME /node
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && \
+    apt-get install -y \
+    apt-utils \
+    bsdmainutils \
+    curl \
+    git \
+    git-core \
+    python3
+
+COPY ./docker/files/ /node/files
+COPY ./installer/genesis /node/files/run/genesis
+COPY ./cmd/updater/update.sh /node/files/build/update.sh
+COPY ./installer/config.json.example /node/files/build/config.json
+
+RUN find /node/files
+
+# Install algod binaries.
+RUN /node/files/build/install.sh \
+    -p "/node/bin" \
+    -d "/node/data" \
+    -c "${CHANNEL}" \
+    -u "${URL}" \
+    -b "${BRANCH}" \
+    -s "${SHA}"
+
+# Copy binaries into a clean image
+# TODO: We don't need most of the binaries.
+#       Should we delete everything except goal/algod/algocfg/tealdbg?
+FROM debian:bullseye-slim as final
+COPY --from=builder "/node/bin/" "/node/bin"
+COPY --from=builder "/node/data/" "/node/dataTemplate"
+COPY --from=builder "/node/files/run" "/node/run"
+
+ENV BIN_DIR="/node/bin"
+ENV PATH="$BIN_DIR:${PATH}"
+ENV ALGOD_PORT=8080
+ENV ALGORAND_DATA="/algod/data"
+RUN mkdir -p "$ALGORAND_DATA"
+WORKDIR /node/data
+
+# curl is needed to lookup the fast catchup url
+RUN apt-get update && apt-get install -y \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# TODO: This works fine, but causes problems when mounting a volume
+# Use algorand user instead of root
+#RUN groupadd -r algorand && \
+#  useradd --no-log-init -r -g algorand algorand && \
+#  chown -R algorand.algorand /node && \
+#  chown -R algorand.algorand /algod
+#USER algorand
+
+# Algod REST API
+EXPOSE $ALGOD_PORT
+
+# Algod Gossip Port
+EXPOSE 4160
+
+# Prometheus Metrics
+EXPOSE 9100
+
+CMD ["/node/run/run.sh"]
+#CMD ["/bin/bash"]

Makefile

@@ -85,7 +85,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
 UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ ))
 ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... ))
 
-MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./stateproof ./data/account ./daemon/algod/api/spec/v2
+MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./ledger/store ./stateproof ./data/account ./daemon/algod/api/spec/v2
 
 default: build
 
@@ -99,7 +99,7 @@ fix: build
 	$(GOPATH1)/bin/algofix */
 
 lint: deps
-	$(GOPATH1)/bin/golangci-lint run -c .golangci.yml 
+	$(GOPATH1)/bin/golangci-lint run -c .golangci.yml
 
 check_shell:
 	find . -type f -name "*.sh" -exec shellcheck {} +
@@ -147,21 +147,6 @@ deps:
 
 # artifacts
 
-# Regenerate algod swagger spec files
-ALGOD_API_SWAGGER_SPEC := daemon/algod/api/swagger.json
-ALGOD_API_FILES := $(shell find daemon/algod/api/server/common daemon/algod/api/server/v1 daemon/algod/api/spec/v1 -type f) \
-	daemon/algod/api/server/router.go
-ALGOD_API_SWAGGER_INJECT := daemon/algod/api/server/lib/bundledSpecInject.go
-
-# Note that swagger.json requires the go-swagger dep.
-$(ALGOD_API_SWAGGER_SPEC): $(ALGOD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a
-	cd daemon/algod/api && \
-		PATH=$(GOPATH1)/bin:$$PATH \
-		go generate ./...
-
-$(ALGOD_API_SWAGGER_INJECT): deps $(ALGOD_API_SWAGGER_SPEC) $(ALGOD_API_SWAGGER_SPEC).validated
-	./daemon/algod/api/server/lib/bundle_swagger_json.sh
-
 # Regenerate kmd swagger spec files
 KMD_API_SWAGGER_SPEC := daemon/kmd/api/swagger.json
 KMD_API_FILES := $(shell find daemon/kmd/api/ -type f | grep -v $(KMD_API_SWAGGER_SPEC))
@@ -191,15 +176,13 @@ $(KMD_API_SWAGGER_INJECT): deps $(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).
 
 # generated files we should make sure we clean
 GENERATED_FILES := \
-	$(ALGOD_API_SWAGGER_INJECT) \
 	$(KMD_API_SWAGGER_INJECT) \
-	$(ALGOD_API_SWAGGER_SPEC) $(ALGOD_API_SWAGGER_SPEC).validated \
 	$(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
 
-rebuild_swagger: deps
+rebuild_kmd_swagger: deps
 	rm -f $(GENERATED_FILES)
 	# we need to invoke the make here since we want to ensure that the deletion and re-creating are sequential
-	make $(KMD_API_SWAGGER_INJECT) $(ALGOD_API_SWAGGER_INJECT)
+	make $(KMD_API_SWAGGER_INJECT)
 
 # develop
 
@@ -327,7 +310,7 @@ dump: $(addprefix gen/,$(addsuffix /genesis.dump, $(NETWORKS)))
 install: build
 	scripts/dev_install.sh -p $(GOPATH1)/bin
 
-.PHONY: default fmt lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_swagger
+.PHONY: default fmt lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_kmd_swagger
 
 ###### TARGETS FOR CICD PROCESS ######
 include ./scripts/release/mule/Makefile.mule

buildnumber.dat

@@ -1 +1 @@
-2
+0

catchup/service.go

@@ -43,6 +43,9 @@ const blockQueryPeerLimit = 10
 // this should be at least the number of relays
 const catchupRetryLimit = 500
 
+// ErrSyncRoundInvalid is returned when the sync round requested is behind the current ledger round
+var ErrSyncRoundInvalid = errors.New("requested sync round cannot be less than the latest round")
+
 // PendingUnmatchedCertificate is a single certificate that is being waited upon to have its corresponding block fetched.
 type PendingUnmatchedCertificate struct {
 	Cert         agreement.Certificate
@@ -64,7 +67,10 @@ type Ledger interface {
 
 // Service represents the catchup service. Once started and until it is stopped, it ensures that the ledger is up to date with network.
 type Service struct {
-	syncStartNS         int64 // at top of struct to keep 64 bit aligned for atomic.* ops
+	syncStartNS int64 // at top of struct to keep 64 bit aligned for atomic.* ops
+	// disableSyncRound, provided externally, is the first round we will _not_ fetch from the network
+	// any round >= disableSyncRound will not be fetched. If set to 0, it will be disregarded.
+	disableSyncRound    uint64
 	cfg                 config.Local
 	ledger              Ledger
 	ctx                 context.Context
@@ -147,6 +153,26 @@ func (s *Service) IsSynchronizing() (synchronizing bool, initialSync bool) {
 	return
 }
 
+// SetDisableSyncRound attempts to set the first round we _do_not_ want to fetch from the network
+// Blocks from disableSyncRound or any round after disableSyncRound will not be fetched while this is set
+func (s *Service) SetDisableSyncRound(rnd uint64) error {
+	if basics.Round(rnd) < s.ledger.LastRound() {
+		return ErrSyncRoundInvalid
+	}
+	atomic.StoreUint64(&s.disableSyncRound, rnd)
+	return nil
+}
+
+// UnsetDisableSyncRound removes any previously set disabled sync round
+func (s *Service) UnsetDisableSyncRound() {
+	atomic.StoreUint64(&s.disableSyncRound, 0)
+}
+
+// GetDisableSyncRound returns the disabled sync round
+func (s *Service) GetDisableSyncRound() uint64 {
+	return atomic.LoadUint64(&s.disableSyncRound)
+}
+
 // SynchronizingTime returns the time we've been performing a catchup operation (0 if not currently catching up)
 func (s *Service) SynchronizingTime() time.Duration {
 	startNS := atomic.LoadInt64(&s.syncStartNS)
@@ -202,6 +228,10 @@ func (s *Service) innerFetch(r basics.Round, peer network.Peer) (blk *bookkeepin
 //  - If the block is already in the ledger (e.g. if agreement service has already written it)
 //  - If the retrieval of the previous block was unsuccessful
 func (s *Service) fetchAndWrite(r basics.Round, prevFetchCompleteChan chan bool, lookbackComplete chan bool, peerSelector *peerSelector) bool {
+	// If sync-ing this round is not intended, don't fetch it
+	if dontSyncRound := s.GetDisableSyncRound(); dontSyncRound != 0 && r >= basics.Round(dontSyncRound) {
+		return false
+	}
 	i := 0
 	hasLookback := false
 	for true {

catchup/service_test.go

@@ -181,6 +181,96 @@ func (cl *periodicSyncLogger) Warnf(s string, args ...interface{}) {
 	cl.Logger.Warnf(s, args...)
 }
 
+func TestSyncRound(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	// Make Ledger
+	local := new(mockedLedger)
+	local.blocks = append(local.blocks, bookkeeping.Block{})
+
+	remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{})
+	if err != nil {
+		t.Fatal(err)
+		return
+	}
+	addBlocks(t, remote, blk, 10)
+
+	// Create a network and block service
+	blockServiceConfig := config.GetDefaultLocal()
+	net := &httpTestPeerSource{}
+	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
+
+	nodeA := basicRPCNode{}
+	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	nodeA.start()
+	defer nodeA.stop()
+	rootURL := nodeA.rootURL()
+	net.addPeer(rootURL)
+
+	auth := &mockedAuthenticator{fail: true}
+	initialLocalRound := local.LastRound()
+	require.True(t, 0 == initialLocalRound)
+
+	// Make Service
+	localCfg := config.GetDefaultLocal()
+	s := MakeService(logging.Base(), localCfg, net, local, auth, nil, nil)
+	s.log = &periodicSyncLogger{Logger: logging.Base()}
+	s.deadlineTimeout = 2 * time.Second
+
+	// Set disable round success
+	err = s.SetDisableSyncRound(3)
+	require.NoError(t, err)
+
+	s.Start()
+	defer s.Stop()
+	// wait past the initial sync - which is known to fail due to the above "auth"
+	time.Sleep(s.deadlineTimeout*2 - 200*time.Millisecond)
+	require.Equal(t, initialLocalRound, local.LastRound())
+	auth.alter(-1, false)
+
+	// wait until the catchup is done. Since we've might have missed the sleep window, we need to wait
+	// until the synchronization is complete.
+	waitStart := time.Now()
+	for time.Since(waitStart) < 2*s.deadlineTimeout {
+		if remote.LastRound() == local.LastRound() {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	// Assert that the last block is the one we expect--i.e. disableSyncRound - 1
+	rnd := s.GetDisableSyncRound()
+	rr, lr := basics.Round(rnd-1), local.LastRound()
+	require.Equal(t, rr, lr)
+
+	for r := basics.Round(1); r < rr; r++ {
+		localBlock, err := local.Block(r)
+		require.NoError(t, err)
+		remoteBlock, err := remote.Block(r)
+		require.NoError(t, err)
+		require.Equal(t, remoteBlock.Hash(), localBlock.Hash())
+	}
+
+	// unset syncRound and make sure we finish catching up
+	s.UnsetDisableSyncRound()
+	// wait until the catchup is done
+	waitStart = time.Now()
+	for time.Now().Sub(waitStart) < 8*s.deadlineTimeout {
+		if remote.LastRound() == local.LastRound() {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	rr, lr = remote.LastRound(), local.LastRound()
+	require.Equal(t, rr, lr)
+	for r := basics.Round(1); r < remote.LastRound(); r++ {
+		localBlock, err := local.Block(r)
+		require.NoError(t, err)
+		remoteBlock, err := remote.Block(r)
+		require.NoError(t, err)
+		require.Equal(t, remoteBlock.Hash(), localBlock.Hash())
+	}
+}
+
 func TestPeriodicSync(t *testing.T) {
 	partitiontest.PartitionTest(t)
 

cmd/catchpointdump/database.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/algorand/go-algorand/crypto/merkletrie"
 	"github.com/algorand/go-algorand/ledger"
+	"github.com/algorand/go-algorand/ledger/store"
 	"github.com/algorand/go-algorand/util/db"
 )
 
@@ -106,11 +107,11 @@ func checkDatabase(databaseName string, outFile *os.File) error {
 
 	var stats merkletrie.Stats
 	err = dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
-		committer, err := ledger.MakeMerkleCommitter(tx, ledgerTrackerStaging)
+		committer, err := store.MakeMerkleCommitter(tx, ledgerTrackerStaging)
 		if err != nil {
 			return err
 		}
-		trie, err := merkletrie.MakeTrie(committer, ledger.TrieMemoryConfig)
+		trie, err := merkletrie.MakeTrie(committer, store.TrieMemoryConfig)
 		if err != nil {
 			return err
 		}

cmd/catchpointdump/file.go

@@ -39,6 +39,7 @@ import (
 	"github.com/algorand/go-algorand/data/transactions/logic"
 	"github.com/algorand/go-algorand/ledger"
 	"github.com/algorand/go-algorand/ledger/ledgercore"
+	"github.com/algorand/go-algorand/ledger/store"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/util/db"
@@ -318,6 +319,8 @@ func printAccountsDatabase(databaseName string, stagingTables bool, fileHeader l
 			totals.RewardsLevel)
 	}
 	return dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
+		arw := store.NewAccountsSQLReaderWriter(tx)
+
 		fmt.Printf("\n")
 		printDumpingCatchpointProgressLine(0, 50, 0)
 
@@ -417,7 +420,7 @@ func printAccountsDatabase(databaseName string, stagingTables bool, fileHeader l
 				progress++
 				acctCount++
 			}
-			_, err = ledger.LoadAllFullAccounts(context.Background(), tx, balancesTable, resourcesTable, acctCb)
+			_, err = arw.LoadAllFullAccounts(context.Background(), balancesTable, resourcesTable, acctCb)
 			if err != nil {
 				return
 			}