Merge remote-tracking branch 'origin/develop' into cve-fix-integration

CHANGELOG.md

@@ -46,10 +46,20 @@ Upgrading to this version requires going through v1.5.x first. Removed migration
   Ensure that your key file in `data/identities` is named `local.key` if you run a supervised node or with the change
   the node will not start.
 
+* [#5965](https://github.com/spacemeshos/go-spacemesh/pull/5965) Start considering ballots that still contain the
+  deprecated inlined activeset as invalid. go-spacemesh references the active set via hash since v1.3.0, and has been
+  pruning the data of old ballots since then as well.
+
 * [#5927](https://github.com/spacemeshos/go-spacemesh/pull/5927) Fixed vulnerability in the way a node handles incoming
   ATXs. This vulnerability allows an attacker to claim rewards for a full tick amount although they should not be
   eligible for them.
 
+## Release v1.5.4
+
+### Improvements
+
+* [#5963](https://github.com/spacemeshos/go-spacemesh/pull/5963) Increase supported number of ATXs to 5.5 Mio.
+
 ## Release v1.5.3
 
 ### Improvements

common/types/ballot.go

@@ -73,7 +73,7 @@ type Ballot struct {
 	// only present in smesher's first ballot of the epoch
 	// this field isn't actually used any more (replaced by InnerBallot.EpochData)
 	// TODO (mafa): remove this field in Ballot v2
-	ActiveSet []ATXID `scale:"max=3500000"`
+	ActiveSet []ATXID `scale:"max=1"`
 
 	// the following fields are kept private and from being serialized
 	ballotID BallotID

common/types/ballot_test.go

@@ -1,8 +1,11 @@
 package types_test
 
 import (
+	"bytes"
 	"testing"
 
+	fuzz "github.com/google/gofuzz"
+	"github.com/spacemeshos/go-scale"
 	"github.com/spacemeshos/go-scale/tester"
 	"github.com/stretchr/testify/require"
 
@@ -101,5 +104,18 @@ func FuzzVotingEligibilityProofSafety(f *testing.F) {
 }
 
 func TestBallotEncoding(t *testing.T) {
-	types.CheckLayerFirstEncoding(t, func(object types.Ballot) types.LayerID { return object.Layer })
+	var ballot types.Ballot
+	f := fuzz.NewWithSeed(1001)
+	f.Fuzz(&ballot)
+	ballot.ActiveSet = nil
+
+	buf := bytes.NewBuffer(nil)
+	enc := scale.NewEncoder(buf)
+	_, err := ballot.EncodeScale(enc)
+	require.NoError(t, err)
+
+	var lid types.LayerID
+	_, err = lid.DecodeScale(scale.NewDecoder(buf))
+	require.NoError(t, err)
+	require.Equal(t, ballot.Layer, lid)
 }

common/types/proposal_test.go

@@ -1,8 +1,11 @@
 package types_test
 
 import (
+	"bytes"
 	"testing"
 
+	fuzz "github.com/google/gofuzz"
+	"github.com/spacemeshos/go-scale"
 	"github.com/spacemeshos/go-scale/tester"
 	"github.com/stretchr/testify/require"
 
@@ -58,5 +61,18 @@ func FuzzInnerProposalSafety(f *testing.F) {
 }
 
 func TestProposalEncoding(t *testing.T) {
-	types.CheckLayerFirstEncoding(t, func(object types.Proposal) types.LayerID { return object.Layer })
+	var proposal types.Proposal
+	f := fuzz.NewWithSeed(1001)
+	f.Fuzz(&proposal)
+	proposal.ActiveSet = nil
+
+	buf := bytes.NewBuffer(nil)
+	enc := scale.NewEncoder(buf)
+	_, err := proposal.EncodeScale(enc)
+	require.NoError(t, err)
+
+	var lid types.LayerID
+	_, err = lid.DecodeScale(scale.NewDecoder(buf))
+	require.NoError(t, err)
+	require.Equal(t, proposal.Layer, lid)
 }

miner/active_set_generator.go

@@ -222,7 +222,7 @@ type gradedActiveSet struct {
 }
 
 // activeSetFromGrades includes activations with the highest grade.
-// Such activations were received atleast 4 network delays before the epoch start, and no malfeasence proof for
+// Such activations were received at least 4 network delays before the epoch start, and no malfeasance proof for
 // identity was received before the epoch start.
 //
 // On mainnet we use 30minutes as a network delay parameter.

proposals/handler.go

@@ -444,7 +444,7 @@ func (h *Handler) processBallot(ctx context.Context, logger log.Log, b *types.Ba
 	if err != nil {
 		return nil, err
 	}
-	b.ActiveSet = nil
+	b.ActiveSet = nil // the active set is not needed anymore
 
 	t1 := time.Now()
 	proof, err := h.mesh.AddBallot(ctx, b)

sql/ballots/ballots.go

@@ -62,17 +62,6 @@ func Has(db sql.Executor, id types.BallotID) (bool, error) {
 	return rows > 0, nil
 }
 
-func UpdateBlob(db sql.Executor, bid types.BallotID, blob []byte) error {
-	if _, err := db.Exec(`update ballots set ballot = ?2 where id = ?1;`,
-		func(stmt *sql.Statement) {
-			stmt.BindBytes(1, bid.Bytes())
-			stmt.BindBytes(2, blob[:])
-		}, nil); err != nil {
-		return fmt.Errorf("update blob %s: %w", bid.String(), err)
-	}
-	return nil
-}
-
 // GetBlobSizes returns the sizes of the blobs corresponding to ballots with specified
 // ids. For non-existent ballots, the corresponding items are set to -1.
 func GetBlobSizes(db sql.Executor, ids [][]byte) (sizes []int, err error) {

sql/ballots/ballots_test.go

@@ -84,26 +84,6 @@ func TestAdd(t *testing.T) {
 	require.True(t, stored.IsMalicious())
 }
 
-func TestUpdateBlob(t *testing.T) {
-	db := sql.InMemory()
-	nodeID := types.RandomNodeID()
-	ballot := types.NewExistingBallot(types.BallotID{1}, types.RandomEdSignature(), nodeID, types.LayerID(0))
-	ballot.EpochData = &types.EpochData{
-		ActiveSetHash: types.RandomHash(),
-	}
-	ballot.ActiveSet = types.RandomActiveSet(199)
-	require.NoError(t, Add(db, &ballot))
-	got, err := Get(db, types.BallotID{1})
-	require.NoError(t, err)
-	require.Equal(t, ballot, *got)
-
-	ballot.ActiveSet = nil
-	require.NoError(t, UpdateBlob(db, types.BallotID{1}, codec.MustEncode(&ballot)))
-	got, err = Get(db, types.BallotID{1})
-	require.NoError(t, err)
-	require.Empty(t, got.ActiveSet)
-}
-
 func TestHas(t *testing.T) {
 	db := sql.InMemory()
 	ballot := types.NewExistingBallot(types.BallotID{1}, types.EmptyEdSignature, types.EmptyNodeID, types.LayerID(0))