Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tools: new endpoints for block generator #5257

Merged
merged 17 commits into from
Apr 12, 2023
70 changes: 59 additions & 11 deletions tools/block-generator/generator/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package generator

import (
"encoding/binary"
"encoding/json"
"fmt"
"io"
Expand All @@ -26,6 +25,9 @@ import (
"time"

cconfig "github.com/algorand/go-algorand/config"
"github.com/algorand/go-algorand/ledger"
"github.com/algorand/go-algorand/ledger/ledgercore"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/protocol"

"github.com/algorand/go-algorand/agreement"
Expand Down Expand Up @@ -133,6 +135,7 @@ func MakeGenerator(config GenerationConfig) (Generator, error) {
gen.genesisHash[31] = 3

gen.initializeAccounting()
gen.initializeLedger()

for _, val := range getTransactionOptions() {
switch val {
Expand Down Expand Up @@ -177,7 +180,9 @@ type Generator interface {
WriteBlock(output io.Writer, round uint64) error
WriteAccount(output io.Writer, accountString string) error
WriteStatus(output io.Writer) error
WriteDeltas(output io.Writer, round uint64) error
Accounts() <-chan basics.Address
Stop()
}

type generator struct {
Expand Down Expand Up @@ -224,6 +229,9 @@ type generator struct {

// Reporting information from transaction type to data
reportData Report

// ledger
ledger *ledger.Ledger
}

type assetData struct {
Expand Down Expand Up @@ -410,20 +418,31 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error {
if err != nil {
return err
}

err = g.ledger.AddBlock(cert.Block, agreement.Certificate{})
if err != nil {
return err
}
g.ledger.WaitForCommit(basics.Round(g.round))
g.finishRound(numTxnForBlock)
return nil
}

func indexToAccount(i uint64) (addr basics.Address) {
// Make sure we don't generate a zero address by adding 1 to i
binary.LittleEndian.PutUint64(addr[:], i+1)
return
}

func accountToIndex(a basics.Address) (addr uint64) {
// Make sure we don't generate a zero address by adding 1 to i
return binary.LittleEndian.Uint64(a[:]) - 1
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

moved to utils.

// WriteDeltas generates returns the deltas for payset.
func (g *generator) WriteDeltas(output io.Writer, round uint64) error {
delta, err := g.ledger.GetStateDeltaForRound(basics.Round(round))
if err != nil {
return fmt.Errorf("err getting state delta for round %d, %v", round, err)
}
// msgp encode deltas
data, err := encode(protocol.CodecHandle, delta)
if err != nil {
return err
}
_, err = output.Write(data)
if err != nil {
return err
}
return nil
}

// initializeAccounting creates the genesis accounts.
Expand Down Expand Up @@ -686,6 +705,35 @@ func (g *generator) generateAssetTxn(round uint64, intra uint64) (transactions.S
return signTxn(txn), transactions.ApplyData{}, nil
}

func (g *generator) initializeLedger() {
genBal := convertToGenesisBalances(g.balances)
// add rewards pool with min balance
genBal[g.rewardsPool] = basics.AccountData{
MicroAlgos: basics.MicroAlgos{Raw: g.params.MinBalance},
}
bal := bookkeeping.MakeGenesisBalances(genBal, g.feeSink, g.rewardsPool)
block, err := bookkeeping.MakeGenesisBlock(g.protocol, bal, g.genesisID, g.genesisHash)
if err != nil {
fmt.Printf("error making genesis: %v\n.", err)
os.Exit(1)
}
l, err := ledger.OpenLedger(logging.Base(), "block-generator", true, ledgercore.InitState{
Block: block,
Accounts: bal.Balances,
GenesisHash: g.genesisHash,
}, cconfig.GetDefaultLocal())
if err != nil {
fmt.Printf("error initializing ledger: %v\n.", err)
os.Exit(1)
}
g.ledger = l
}

// Stop cleans up allocated resources.
func (g *generator) Stop() {
g.ledger.Close()
}

func (g *generator) WriteAccount(output io.Writer, accountString string) error {
addr, err := basics.UnmarshalChecksumAddress(accountString)
if err != nil {
Expand Down
47 changes: 41 additions & 6 deletions tools/block-generator/generator/generate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@ package generator

import (
"bytes"
"net/http"
"net/http/httptest"
"testing"

"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/data/transactions"
"github.com/algorand/go-algorand/protocol"
"github.com/algorand/go-algorand/rpcs"
Expand All @@ -31,7 +34,7 @@ func makePrivateGenerator(t *testing.T) *generator {
partitiontest.PartitionTest(t)
publicGenerator, err := MakeGenerator(GenerationConfig{
NumGenesisAccounts: 10,
GenesisAccountInitialBalance: 10000000000000000000,
GenesisAccountInitialBalance: 1000000000000,
shiqizng marked this conversation as resolved.
Show resolved Hide resolved
PaymentTransactionFraction: 1.0,
PaymentNewAccountFraction: 1.0,
AssetCreateFraction: 1.0,
Expand Down Expand Up @@ -204,13 +207,45 @@ func TestWriteRound(t *testing.T) {
var block rpcs.EncodedBlockCert
protocol.Decode(data, &block)
require.Len(t, block.Block.Payset, int(g.config.TxnPerBlock))
require.NotNil(t, g.ledger)
require.Equal(t, basics.Round(1), g.ledger.Latest())
_, err := g.ledger.GetStateDeltaForRound(1)
require.NoError(t, err)
}

func TestIndexToAccountAndAccountToIndex(t *testing.T) {
func TestHandlers(t *testing.T) {
partitiontest.PartitionTest(t)
for i := uint64(0); i < uint64(100000); i++ {
acct := indexToAccount(i)
result := accountToIndex(acct)
require.Equal(t, i, result)
g := makePrivateGenerator(t)
handler := getBlockHandler(g)
var testcases = []struct {
name string
url string
err string
}{
{
name: "no block",
url: "/v2/blocks/?nothing",
err: "invalid request path, /",
},
{
name: "blocks: round must be numeric",
url: "/v2/blocks/round",
err: `strconv.ParseUint: parsing "round": invalid syntax`,
},
{
name: "deltas: round must be numeric",
url: "/v2/deltas/round",
err: `strconv.ParseUint: parsing "round": invalid syntax`,
},
}

for _, testcase := range testcases {
t.Run(testcase.name, func(t *testing.T) {
req := httptest.NewRequest("GET", testcase.url, nil)
w := httptest.NewRecorder()
handler(w, req)
require.Equal(t, http.StatusBadRequest, w.Code)
require.Contains(t, w.Body.String(), testcase.err)
})
}
}
68 changes: 29 additions & 39 deletions tools/block-generator/generator/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"fmt"
"net/http"
"os"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -67,6 +68,8 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B
mux.HandleFunc("/genesis", getGenesisHandler(gen))
mux.HandleFunc("/report", getReportHandler(gen))
mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen))
mux.HandleFunc("/v2/ledger/sync/", func(w http.ResponseWriter, r *http.Request) {})
mux.HandleFunc("/v2/deltas/", getDeltasHandler(gen))

return &http.Server{
Addr: addr,
Expand Down Expand Up @@ -107,68 +110,55 @@ func getGenesisHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques
func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
// The generator doesn't actually care about the block...
round, err := parseRound(r.URL.Path)
s, err := parseURL(r.URL.Path)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
round, err := strconv.ParseUint(s, 0, 64)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}

maybeWriteError(w, gen.WriteBlock(w, round))
}
}

func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
// The generator doesn't actually care about the block...
account, err := parseAccount(r.URL.Path)
account, err := parseURL(r.URL.Path)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}

maybeWriteError(w, gen.WriteAccount(w, account))
}
}

const blockQueryPrefix = "/v2/blocks/"
const blockQueryBlockIdx = len(blockQueryPrefix)
const accountsQueryPrefix = "/v2/accounts/"
const accountsQueryAccountIdx = len(accountsQueryPrefix)

func parseRound(path string) (uint64, error) {
if !strings.HasPrefix(path, blockQueryPrefix) {
return 0, fmt.Errorf("not a blocks query: %s", path)
}

result := uint64(0)
pathlen := len(path)

if pathlen == blockQueryBlockIdx {
return 0, fmt.Errorf("no block in path")
}

for i := blockQueryBlockIdx; i < pathlen; i++ {
if path[i] < '0' || path[i] > '9' {
if i == blockQueryBlockIdx {
return 0, fmt.Errorf("no block in path")
}
break
func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
s, err := parseURL(r.URL.Path)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
result = (uint64(10) * result) + uint64(int(path[i])-'0')
round, err := strconv.ParseUint(s, 0, 64)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
maybeWriteError(w, gen.WriteDeltas(w, round))
}
return result, nil
}

func parseAccount(path string) (string, error) {
if !strings.HasPrefix(path, accountsQueryPrefix) {
return "", fmt.Errorf("not a accounts query: %s", path)
func parseURL(path string) (string, error) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

generalizing the parser for all endpoints. I didn't think it was necessary to check the path prefix since each handler function is associated with only one endpoint.

i := strings.LastIndex(path, "/")
if i == len(path)-1 {
return "", fmt.Errorf("invalid request path, %s", path)
}

pathlen := len(path)

if pathlen == accountsQueryAccountIdx {
return "", fmt.Errorf("no address in path")
if strings.Contains(path[i+1:], "?") {
return strings.Split(path[i+1:], "?")[0], nil
}

return path[accountsQueryAccountIdx:], nil
return path[i+1:], nil
}
Loading