Try #5791:

CHANGELOG.md

@@ -2,6 +2,18 @@
 
 See [RELEASE](./RELEASE.md) for workflow instructions.
 
+## UNRELEASED
+
+### Upgrade information
+
+### Highlights
+
+### Features
+
+### Improvements
+
+* [#5791](https://github.com/spacemeshos/go-spacemesh/pull/5791) Speed up cache warmup
+
 ## Release v1.4.4
 
 ### Improvements

atxsdata/warmup.go

@@ -2,12 +2,12 @@ package atxsdata
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/spacemeshos/go-spacemesh/common/types"
 	"github.com/spacemeshos/go-spacemesh/sql"
 	"github.com/spacemeshos/go-spacemesh/sql/atxs"
-	"github.com/spacemeshos/go-spacemesh/sql/identities"
 	"github.com/spacemeshos/go-spacemesh/sql/layers"
 )
 
@@ -49,28 +49,23 @@ func Warmup(db sql.Executor, cache *Data, keep types.EpochID) error {
 			weight,
 			base,
 			height uint64,
+			nonce *types.VRFPostIndex,
+			isMalicious bool,
 		) bool {
-			target := epoch + 1
-			nonce, err := atxs.VRFNonce(db, node, target)
-			if err != nil {
-				ierr = fmt.Errorf("missing nonce %w", err)
-				return false
-			}
-			malicious, err := identities.IsMalicious(db, node)
-			if err != nil {
-				ierr = err
+			if nonce == nil {
+				ierr = errors.New("missing nonce")
 				return false
 			}
 			cache.Add(
-				target,
+				epoch+1,
 				node,
 				coinbase,
 				id,
 				weight,
 				base,
 				height,
-				nonce,
-				malicious,
+				*nonce,
+				isMalicious,
 			)
 			return true
 		}); err != nil {

atxsdata/warmup_test.go

@@ -103,7 +103,7 @@ func TestWarmup(t *testing.T) {
 				return tx.Exec(q, enc, dec)
 			}).
 			AnyTimes()
-		for i := 0; i < 5; i++ {
+		for i := 0; i < 3; i++ {
 			c := New()
 			require.Error(t, Warmup(exec, c, 1))
 			fail++

sql/atxs/atxs.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"time"
 
+	sqlite "github.com/go-llsqlite/crawshaw"
+
 	"github.com/spacemeshos/go-spacemesh/codec"
 	"github.com/spacemeshos/go-spacemesh/common/types"
 	"github.com/spacemeshos/go-spacemesh/sql"
@@ -569,27 +571,53 @@ func IterateAtxsData(
 		weight uint64,
 		base uint64,
 		height uint64,
+		nonce *types.VRFPostIndex,
+		isMalicious bool,
 	) bool,
 ) error {
 	_, err := db.Exec(
-		`select id, pubkey, epoch, coinbase, effective_num_units, base_tick_height, tick_count
-		from atxs where epoch between ?1 and ?2;`,
-		func(stmt *sql.Statement) {
-			stmt.BindInt64(1, int64(from.Uint32()))
-			stmt.BindInt64(2, int64(to.Uint32()))
-		},
+		`select
+                   a.id, a.pubkey, a.epoch, a.coinbase, a.effective_num_units,
+                   a.base_tick_height, a.tick_count,
+                   coalesce(a.nonce, (
+		     select a1.nonce
+		     from atxs a1
+		     where a1.pubkey = a.pubkey and a1.epoch < a.epoch and a1.nonce is not null
+		     order by a1.epoch desc
+		     limit 1
+		   )) as nonce,
+                   iif(idn.proof is null, 0, 1) as is_malicious
+		from atxs a left join identities idn on a.pubkey = idn.pubkey`,
+		// SQLite happens to process the query much faster if we don't
+		// filter it by epoch
+		// where a.epoch between ? and ?`,
+		// func(stmt *sql.Statement) {
+		// 	stmt.BindInt64(1, int64(from.Uint32()))
+		// 	stmt.BindInt64(2, int64(to.Uint32()))
+		// },
+		nil,
 		func(stmt *sql.Statement) bool {
+			epoch := types.EpochID(uint32(stmt.ColumnInt64(2)))
+			if epoch < from || epoch > to {
+				return true
+			}
 			var id types.ATXID
 			stmt.ColumnBytes(0, id[:])
 			var node types.NodeID
 			stmt.ColumnBytes(1, node[:])
-			epoch := types.EpochID(uint32(stmt.ColumnInt64(2)))
 			var coinbase types.Address
 			stmt.ColumnBytes(3, coinbase[:])
 			effectiveUnits := uint64(stmt.ColumnInt64(4))
 			baseHeight := uint64(stmt.ColumnInt64(5))
 			ticks := uint64(stmt.ColumnInt64(6))
-			return fn(id, node, epoch, coinbase, effectiveUnits*ticks, baseHeight, baseHeight+ticks)
+			var vrfNonce *types.VRFPostIndex
+			if stmt.ColumnType(7) != sqlite.SQLITE_NULL {
+				nonce := types.VRFPostIndex(stmt.ColumnInt64(7))
+				vrfNonce = &nonce
+			}
+			isMalicious := stmt.ColumnInt(8) != 0
+			return fn(id, node, epoch, coinbase, effectiveUnits*ticks,
+				baseHeight, baseHeight+ticks, vrfNonce, isMalicious)
 		},
 	)
 	if err != nil {