Merge remote-tracking branch 'upstream/master' into add-peerID

.circleci/config.yml

@@ -47,19 +47,20 @@ executors:
     resource_class: arm.large
   mac_amd64_medium:
     macos:
-      xcode: 13.2.1
-    resource_class: medium
+      xcode: 14.2.0
+    resource_class: macos.x86.medium.gen2
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
   mac_amd64_large:
     macos:
-      xcode: 13.2.1
-    resource_class: large
+      xcode: 14.2.0
+    # Since they removed the large class for amd64, we will use medium here too.
+    resource_class: macos.x86.medium.gen2
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
   mac_arm64: &executor-mac-arm64
     machine: true
-    resource_class: algorand/macstadium-m1-macos11
+    resource_class: algorand/macstadium-m1
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
   # these are required b/c jobs explicitly assign sizes to the executors

.github/workflows/pr-type-category.yml

@@ -10,7 +10,7 @@ jobs:
     name: Check PR Category and Type
     steps:
       - name: Checking for correct number of required github pr labels
-        uses: mheap/github-action-required-labels@v2
+        uses: mheap/github-action-required-labels@v5
         with:
           mode: exactly
           count: 1

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04 as builder
+FROM ubuntu:20.04 as builder
 
 ARG GO_VERSION="1.20.5"
 

Makefile

@@ -49,24 +49,16 @@ export CPATH=/opt/homebrew/include
 export LIBRARY_PATH=/opt/homebrew/lib
 endif
 endif
+
 ifeq ($(UNAME), Linux)
 EXTLDFLAGS := -static-libstdc++ -static-libgcc
-ifeq ($(ARCH), amd64)
 # the following predicate is abit misleading; it tests if we're not in centos.
 ifeq (,$(wildcard /etc/centos-release))
 EXTLDFLAGS  += -static
 endif
 GOTAGSLIST  += osusergo netgo static_build
 GOBUILDMODE := -buildmode pie
 endif
-ifeq ($(ARCH), arm)
-ifneq ("$(wildcard /etc/alpine-release)","")
-EXTLDFLAGS  += -static
-GOTAGSLIST  += osusergo netgo static_build
-GOBUILDMODE := -buildmode pie
-endif
-endif
-endif
 
 ifneq (, $(findstring MINGW,$(UNAME)))
 EXTLDFLAGS := -static -static-libstdc++ -static-libgcc
@@ -99,7 +91,7 @@ ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd d
 
 GOMOD_DIRS := ./tools/block-generator ./tools/x-repo-types
 
-MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2
+MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2
 
 default: build
 

README.md

@@ -22,7 +22,7 @@ the [official Go documentation website](https://golang.org/doc/).
 
 ### Linux / OSX
 
-We currently strive to support Debian-based distributions with Ubuntu 22.04
+We currently strive to support Debian-based distributions with Ubuntu 20.04
 being our official release target.
 Building on Arch Linux works as well.
 Our core engineering team uses Linux and OSX, so both environments are well

catchup/service.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -744,17 +745,19 @@ func (s *Service) fetchRound(cert agreement.Certificate, verifier *agreement.Asy
 		if cert.Round == fetchedCert.Round &&
 			cert.Proposal.BlockDigest != fetchedCert.Proposal.BlockDigest &&
 			fetchedCert.Authenticate(*block, s.ledger, verifier) == nil {
-			s := "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-			s += "!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n"
-			s += "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-			s += "fetchRound called with a cert authenticating block with hash %v.\n"
-			s += "We fetched a valid cert authenticating a different block, %v. This indicates a fork.\n\n"
-			s += "Cert from our agreement service:\n%#v\n\n"
-			s += "Cert from the fetcher:\n%#v\n\n"
-			s += "Block from the fetcher:\n%#v\n\n"
-			s += "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-			s += "!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n"
-			s += "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+			var builder strings.Builder
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			builder.WriteString("fetchRound called with a cert authenticating block with hash %v.\n")
+			builder.WriteString("We fetched a valid cert authenticating a different block, %v. This indicates a fork.\n\n")
+			builder.WriteString("Cert from our agreement service:\n%#v\n\n")
+			builder.WriteString("Cert from the fetcher:\n%#v\n\n")
+			builder.WriteString("Block from the fetcher:\n%#v\n\n")
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			s := builder.String()
 			s = fmt.Sprintf(s, cert.Proposal.BlockDigest, fetchedCert.Proposal.BlockDigest, cert, fetchedCert, block)
 			fmt.Println(s)
 			logging.Base().Error(s)

cmd/goal/clerk.go

@@ -73,9 +73,11 @@ var (
 	simulateAllowMoreLogging      bool
 	simulateAllowMoreOpcodeBudget bool
 	simulateExtraOpcodeBudget     uint64
-	simulateEnableRequestTrace    bool
-	simulateStackChange           bool
-	simulateScratchChange         bool
+
+	simulateFullTrace          bool
+	simulateEnableRequestTrace bool
+	simulateStackChange        bool
+	simulateScratchChange      bool
 )
 
 func init() {
@@ -99,7 +101,7 @@ func init() {
 	sendCmd.Flags().Uint64VarP(&amount, "amount", "a", 0, "The amount to be transferred (required), in microAlgos")
 	sendCmd.Flags().StringVarP(&closeToAddress, "close-to", "c", "", "Close account and send remainder to this address")
 	sendCmd.Flags().StringVar(&rekeyToAddress, "rekey-to", "", "Rekey account to the given spending key/address. (Future transactions from this account will need to be signed with the new key.)")
-	sendCmd.Flags().StringVarP(&programSource, "from-program", "F", "", "Program source to use as account logic")
+	sendCmd.Flags().StringVarP(&programSource, "from-program", "F", "", "Program source file to use as account logic")
 	sendCmd.Flags().StringVarP(&progByteFile, "from-program-bytes", "P", "", "Program binary to use as account logic")
 	sendCmd.Flags().StringSliceVar(&argB64Strings, "argb64", nil, "Base64 encoded args to pass to transaction logic")
 	sendCmd.Flags().StringVarP(&logicSigFile, "logic-sig", "L", "", "LogicSig to apply to transaction")
@@ -119,7 +121,7 @@ func init() {
 	signCmd.Flags().StringVarP(&txFilename, "infile", "i", "", "Partially-signed transaction file to add signature to")
 	signCmd.Flags().StringVarP(&outFilename, "outfile", "o", "", "Filename for writing the signed transaction")
 	signCmd.Flags().StringVarP(&signerAddress, "signer", "S", "", "Address of key to sign with, if different from transaction \"from\" address due to rekeying")
-	signCmd.Flags().StringVarP(&programSource, "program", "p", "", "Program source to use as account logic")
+	signCmd.Flags().StringVarP(&programSource, "program", "p", "", "Program source file to use as account logic")
 	signCmd.Flags().StringVarP(&logicSigFile, "logic-sig", "L", "", "LogicSig to apply to transaction")
 	signCmd.Flags().StringSliceVar(&argB64Strings, "argb64", nil, "Base64 encoded args to pass to transaction logic")
 	signCmd.Flags().StringVarP(&protoVersion, "proto", "P", "", "Consensus protocol version id string")
@@ -164,6 +166,8 @@ func init() {
 	simulateCmd.Flags().BoolVar(&simulateAllowMoreLogging, "allow-more-logging", false, "Lift the limits on log opcode during simulation")
 	simulateCmd.Flags().BoolVar(&simulateAllowMoreOpcodeBudget, "allow-more-opcode-budget", false, "Apply max extra opcode budget for apps per transaction group (default 320000) during simulation")
 	simulateCmd.Flags().Uint64Var(&simulateExtraOpcodeBudget, "extra-opcode-budget", 0, "Apply extra opcode budget for apps per transaction group during simulation")
+
+	simulateCmd.Flags().BoolVar(&simulateFullTrace, "full-trace", false, "Enable all options for simulation execution trace")
 	simulateCmd.Flags().BoolVar(&simulateEnableRequestTrace, "trace", false, "Enable simulation time execution trace of app calls")
 	simulateCmd.Flags().BoolVar(&simulateStackChange, "stack", false, "Report stack change during simulation time")
 	simulateCmd.Flags().BoolVar(&simulateScratchChange, "scratch", false, "Report scratch change during simulation time")
@@ -1368,9 +1372,17 @@ func decodeTxnsFromFile(file string) []transactions.SignedTxn {
 }
 
 func traceCmdOptionToSimulateTraceConfigModel() simulation.ExecTraceConfig {
-	return simulation.ExecTraceConfig{
-		Enable:  simulateEnableRequestTrace,
-		Stack:   simulateStackChange,
-		Scratch: simulateScratchChange,
+	var traceConfig simulation.ExecTraceConfig
+	if simulateFullTrace {
+		traceConfig = simulation.ExecTraceConfig{
+			Enable:  true,
+			Stack:   true,
+			Scratch: true,
+		}
 	}
+	traceConfig.Enable = traceConfig.Enable || simulateEnableRequestTrace
+	traceConfig.Stack = traceConfig.Stack || simulateStackChange
+	traceConfig.Scratch = traceConfig.Scratch || simulateScratchChange
+
+	return traceConfig
 }

docker/Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:22.04
 ARG GOLANG_VERSION
 ENV DEBIAN_FRONTEND noninteractive
 
-RUN apt update && apt-get install -y git wget sqlite3 autoconf sudo tzdata bsdmainutils
+RUN apt update && apt-get install -y git wget autoconf sudo tzdata bsdmainutils
 
 WORKDIR /root
 RUN wget --quiet https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz && tar -xvf go${GOLANG_VERSION}.linux-amd64.tar.gz && mv go /usr/local

docker/build/Dockerfile

@@ -1,7 +1,7 @@
-FROM ubuntu:22.04
+FROM ubuntu:20.04
 ARG GOLANG_VERSION
 
-RUN apt-get update && apt-get install -y git wget sqlite3 autoconf build-essential shellcheck
+RUN apt-get update && apt-get install -y git wget autoconf build-essential shellcheck
 WORKDIR /root
 RUN wget --quiet https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz && tar -xvf go${GOLANG_VERSION}.linux-amd64.tar.gz && mv go /usr/local
 ENV GOROOT=/usr/local/go \

docker/build/Dockerfile-deploy

@@ -1,7 +1,7 @@
-FROM --platform=linux/amd64 ubuntu:22.04
+FROM --platform=linux/amd64 ubuntu:20.04
 ARG GOLANG_VERSION
 
-RUN apt-get update && apt-get install -y git wget sqlite3 autoconf jq bsdmainutils shellcheck
+RUN apt-get update && apt-get install -y git wget autoconf jq bsdmainutils shellcheck
 WORKDIR /root
 RUN wget --quiet https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz && tar -xvf go${GOLANG_VERSION}.linux-amd64.tar.gz && mv go /usr/local
 ENV GOROOT=/usr/local/go \

docker/build/aptly.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:20.04
 
 ARG ARCH=amd64
 ARG GOLANG_VERSION

docker/build/cicd.alpine.Dockerfile

@@ -13,8 +13,7 @@ RUN apk update && \
     apk add automake && \
     apk add fmt && \
     apk add build-base && \
-    apk add musl-dev && \
-    apk add sqlite
+    apk add musl-dev
 
 RUN apk add dpkg && \
     wget http://deb.debian.org/debian/pool/main/s/shellcheck/shellcheck_0.5.0-3_armhf.deb && \

docker/build/cicd.centos.Dockerfile

@@ -5,7 +5,7 @@ ARG GOLANG_VERSION
 ARG ARCH="amd64"
 RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
     yum update -y && \
-    yum install -y autoconf wget awscli git gnupg2 nfs-utils python3-devel sqlite3 expect jq \
+    yum install -y autoconf wget awscli git gnupg2 nfs-utils python3-devel expect jq \
     libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 which ShellCheck \
     libffi-devel openssl-devel
 WORKDIR /root

docker/build/cicd.centos8.Dockerfile

@@ -10,7 +10,6 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.n
     libffi-devel openssl-devel
 RUN dnf install -y epel-release && \
     dnf update && \
-    dnf -y install sqlite && \
     dnf -y --enablerepo=powertools install libstdc++-static && \
     dnf -y install make
 RUN echo "${BOLD}Downloading and installing binaries...${RESET}" && \

docker/build/cicd.ubuntu.Dockerfile

@@ -1,11 +1,11 @@
 ARG ARCH="amd64"
 
-FROM ${ARCH}/ubuntu:22.04
+FROM ${ARCH}/ubuntu:20.04
 ARG GOLANG_VERSION
 ARG ARCH="amd64"
 ARG GOARCH="amd64"
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && apt-get install -y build-essential git wget sqlite3 autoconf jq bsdmainutils shellcheck awscli libtool
+RUN apt-get update && apt-get install -y build-essential git wget autoconf jq bsdmainutils shellcheck awscli libtool
 WORKDIR /root
 RUN wget https://dl.google.com/go/go${GOLANG_VERSION}.linux-${GOARCH}.tar.gz \
     && tar -xvf go${GOLANG_VERSION}.linux-${GOARCH}.tar.gz && \

docker/build/docker.ubuntu.Dockerfile

@@ -1,6 +1,6 @@
 ARG ARCH="amd64"
 
-FROM ${ARCH}/ubuntu:22.04
+FROM ${ARCH}/ubuntu:20.04
 ARG GOLANG_VERSION
 ARG ARCH="amd64"
 RUN apt-get update && apt-get install curl python python3.7 python3-pip build-essential apt-transport-https ca-certificates software-properties-common -y && \

docker/build/releases-page.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:20.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install git python3 python3-pip -y && \

go.mod

@@ -24,12 +24,15 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/google/go-querystring v1.0.0
 	github.com/gorilla/mux v1.8.0
+	github.com/ipfs/go-datastore v0.6.0
+	github.com/ipfs/go-ds-pebble v0.2.4
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/karalabe/usb v0.0.2
 	github.com/labstack/echo/v4 v4.9.1
 	github.com/libp2p/go-libp2p v0.29.0
 	github.com/mattn/go-sqlite3 v1.10.0
 	github.com/miekg/dns v1.1.55
+	github.com/multiformats/go-multiaddr v0.10.1
 	github.com/olivere/elastic v6.2.14+incompatible
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.3.0
@@ -44,42 +47,79 @@ require (
 
 require (
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.8.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230227185959-8285e8dd5c08 // indirect
+	github.com/cockroachdb/redact v1.0.8 // indirect
+	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/fortytw2/leaktest v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.5 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/invopop/yaml v0.1.0 // indirect
+	github.com/ipfs/go-cid v0.4.1 // indirect
+	github.com/ipfs/go-log/v2 v2.5.1 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/jmespath/go-jmespath v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/kr/pretty v0.2.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/labstack/gommon v0.4.0 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multicodec v0.9.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-multistream v0.4.1 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.1 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.12.0 // indirect
 	golang.org/x/term v0.10.0 // indirect
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
 	golang.org/x/tools v0.11.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
 )