Merge branch 'develop' into grading-for-miner

.github/workflows/ci.yml

@@ -137,11 +137,10 @@ jobs:
     needs: filter-changes
     if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         os:
           - ubuntu-latest
-          - [self-hosted, linux, arm64]
           - macos-latest
           - [self-hosted, macos, arm64]
     steps:
@@ -210,6 +209,7 @@ jobs:
       fail-fast: false
       matrix:
         os:
+          - [self-hosted, linux, arm64]
           - windows-latest
     steps:
       # as we use some request to localhost, sometimes it gives us flaky tests. try to disable tcp offloading for fix it

Makefile-libs.Inc

@@ -50,7 +50,7 @@ else
 	endif
 endif
 
-POSTRS_SETUP_REV = 0.4.0
+POSTRS_SETUP_REV = 0.4.2
 POSTRS_SETUP_ZIP = libpost-$(platform)-v$(POSTRS_SETUP_REV).zip
 POSTRS_SETUP_URL_ZIP ?= https://github.com/spacemeshos/post-rs/releases/download/v$(POSTRS_SETUP_REV)/$(POSTRS_SETUP_ZIP)
 POSTRS_PROFILER_ZIP = profiler-$(platform)-v$(POSTRS_SETUP_REV).zip
@@ -75,10 +75,10 @@ $(BINDIR_POSTRS_SETUP_LIBS): $(PROJ_DIR)$(POSTRS_SETUP_ZIP)
 	touch $@
 
 $(PROJ_DIR)$(POSTRS_SETUP_ZIP):
-	curl -L $(POSTRS_SETUP_URL_ZIP) -o $(PROJ_DIR)$(POSTRS_SETUP_ZIP)
+	curl -sSL --retry 5 --retry-max-time 60 $(POSTRS_SETUP_URL_ZIP) -o $(PROJ_DIR)$(POSTRS_SETUP_ZIP)
 
 $(BIN_DIR)$(POSTRS_PROFILER_BIN):
-	curl -L $(POSTRS_PROFILER_URL) -o $(PROJ_DIR)$(POSTRS_PROFILER_ZIP)
+	curl -sSL --retry 5 --retry-max-time 60 $(POSTRS_PROFILER_URL) -o $(PROJ_DIR)$(POSTRS_PROFILER_ZIP)
 	unzip -o -j $(PROJ_DIR)$(POSTRS_PROFILER_ZIP) -d $(BIN_DIR)
 
 get-postrs-lib: $(PROJ_DIR)$(POSTRS_SETUP_ZIP) $(BINDIR_POSTRS_SETUP_LIBS)

activation/activation.go

@@ -7,6 +7,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"sync"
 	"time"
@@ -22,6 +23,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/datastore"
 	"github.com/spacemeshos/go-spacemesh/events"
 	"github.com/spacemeshos/go-spacemesh/log"
+	"github.com/spacemeshos/go-spacemesh/metrics/public"
 	"github.com/spacemeshos/go-spacemesh/p2p/pubsub"
 	"github.com/spacemeshos/go-spacemesh/signing"
 	"github.com/spacemeshos/go-spacemesh/sql"
@@ -74,6 +76,7 @@ type Builder struct {
 	nipostBuilder     nipostBuilder
 	postSetupProvider postSetupProvider
 	initialPost       *types.Post
+	validator         nipostValidator
 
 	// smeshingMutex protects `StartSmeshing` and `StopSmeshing` from concurrent access
 	smeshingMutex sync.Mutex
@@ -125,6 +128,12 @@ func WithPoetConfig(c PoetConfig) BuilderOption {
 	}
 }
 
+func WithValidator(v nipostValidator) BuilderOption {
+	return func(b *Builder) {
+		b.validator = v
+	}
+}
+
 // NewBuilder returns an atx builder that will start a routine that will attempt to create an atx upon each new layer.
 func NewBuilder(
 	conf Config,
@@ -211,6 +220,7 @@ func (b *Builder) StartSmeshing(coinbase types.Address, opts PostSetupOpts) erro
 			return nil
 		case err != nil:
 			b.log.Panic("initialization failed: %v", err)
+			return err
 		}
 
 		b.run(ctx)
@@ -242,6 +252,19 @@ func (b *Builder) StopSmeshing(deleteFiles bool) error {
 			b.log.With().Error("failed to delete post files", log.Err(err))
 			return err
 		}
+		if err := discardBuilderState(b.nipostBuilder.DataDir()); err != nil && !errors.Is(err, fs.ErrNotExist) {
+			b.log.With().Error("failed to delete builder state", log.Err(err))
+			return err
+		}
+		if err := discardNipostChallenge(b.nipostBuilder.DataDir()); err != nil && !errors.Is(err, fs.ErrNotExist) {
+			b.log.With().Error("failed to delete nipost challenge", log.Err(err))
+			return err
+		}
+		if err := discardPost(b.nipostBuilder.DataDir()); err != nil && !errors.Is(err, fs.ErrNotExist) {
+			b.log.With().Error("failed to delete post", log.Err(err))
+			return err
+		}
+
 		return nil
 	default:
 		return fmt.Errorf("failed to stop post data creation session: %w", err)
@@ -254,7 +277,8 @@ func (b *Builder) SmesherID() types.NodeID {
 }
 
 func (b *Builder) run(ctx context.Context) {
-	if err := b.generateInitialPost(ctx); err != nil {
+	err := b.generateInitialPost(ctx)
+	if err != nil {
 		b.log.Error("Failed to generate proof: %s", err)
 		return
 	}
@@ -274,28 +298,57 @@ func (b *Builder) generateInitialPost(ctx context.Context) error {
 	}
 	// ...and if we don't have an initial POST persisted already.
 	if post, err := loadPost(b.nipostBuilder.DataDir()); err == nil {
-		b.initialPost = post
-		return nil
+		b.log.Info("loaded the initial post from disk")
+		return b.verifyInitialPost(ctx, post, &types.PostMetadata{
+			Challenge:     shared.ZeroChallenge,
+			LabelsPerUnit: b.postSetupProvider.Config().LabelsPerUnit,
+		})
 	}
 
 	// Create the initial post and save it.
 	startTime := time.Now()
 	var err error
 	events.EmitPostStart(shared.ZeroChallenge)
-	b.initialPost, _, err = b.postSetupProvider.GenerateProof(ctx, shared.ZeroChallenge, proving.WithPowCreator(b.nodeID.Bytes()))
+	post, metadata, err := b.postSetupProvider.GenerateProof(ctx, shared.ZeroChallenge, proving.WithPowCreator(b.nodeID.Bytes()))
 	if err != nil {
 		events.EmitPostFailure()
 		return fmt.Errorf("post execution: %w", err)
 	}
 	events.EmitPostComplete(shared.ZeroChallenge)
 	metrics.PostDuration.Set(float64(time.Since(startTime).Nanoseconds()))
+	public.PostSeconds.Set(float64(time.Since(startTime)))
+	b.log.Info("created the initial post")
+	if b.verifyInitialPost(ctx, post, metadata) != nil {
+		return err
+	}
 
-	if err := savePost(b.nipostBuilder.DataDir(), b.initialPost); err != nil {
+	if err := savePost(b.nipostBuilder.DataDir(), post); err != nil {
 		b.log.With().Warning("failed to save initial post: %w", log.Err(err))
 	}
 	return nil
 }
 
+func (b *Builder) verifyInitialPost(ctx context.Context, post *types.Post, metadata *types.PostMetadata) error {
+	b.log.With().Info("verifying the initial post", log.Object("post", post), log.Object("metadata", metadata))
+	commitmentAtxId, err := b.postSetupProvider.CommitmentAtx()
+	if err != nil {
+		b.log.With().Panic("failed to fetch commitment ATX ID.", log.Err(err))
+	}
+	err = b.validator.Post(ctx, types.EpochID(0), b.nodeID, commitmentAtxId, post, metadata, b.postSetupProvider.LastOpts().NumUnits)
+	switch {
+	case errors.Is(err, context.Canceled):
+		// If the context was canceled, we don't want to emit or log errors just propagate the cancellation signal.
+		return err
+	case err != nil:
+		events.EmitInvalidPostProof()
+		b.log.With().Fatal("initial POST proof is invalid. Probably the initialized POST data is corrupted. Please verify the data with postcli and regenerate the corrupted files.", log.Err(err))
+		return err
+	default:
+		b.initialPost = post
+		return nil
+	}
+}
+
 func (b *Builder) receivePendingPoetClients() *[]PoetProvingServiceClient {
 	return b.pendingPoetClients.Swap(nil)
 }
@@ -496,9 +549,11 @@ func (b *Builder) PublishActivationTx(ctx context.Context) error {
 
 	challenge, err := b.loadChallenge()
 	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			logger.With().Warning("failed to load atx challenge", log.Err(err))
+		}
 		logger.With().Info("building new atx challenge",
 			log.Stringer("current_epoch", b.currentEpoch()),
-			log.Err(err),
 		)
 		challenge, err = b.buildNIPostChallenge(ctx)
 		if err != nil {

activation/activation_test.go

@@ -96,12 +96,13 @@ type testAtxBuilder struct {
 	coinbase    types.Address
 	goldenATXID types.ATXID
 
-	mhdlr   *MockatxHandler
-	mpub    *mocks.MockPublisher
-	mnipost *MocknipostBuilder
-	mpost   *MockpostSetupProvider
-	mclock  *MocklayerClock
-	msync   *Mocksyncer
+	mhdlr      *MockatxHandler
+	mpub       *mocks.MockPublisher
+	mnipost    *MocknipostBuilder
+	mpost      *MockpostSetupProvider
+	mclock     *MocklayerClock
+	msync      *Mocksyncer
+	mValidator *MocknipostValidator
 }
 
 func newTestBuilder(tb testing.TB, opts ...BuilderOption) *testAtxBuilder {
@@ -121,8 +122,11 @@ func newTestBuilder(tb testing.TB, opts ...BuilderOption) *testAtxBuilder {
 		mpost:       NewMockpostSetupProvider(ctrl),
 		mclock:      NewMocklayerClock(ctrl),
 		msync:       NewMocksyncer(ctrl),
+		mValidator:  NewMocknipostValidator(ctrl),
 	}
 
+	opts = append(opts, WithValidator(tab.mValidator))
+
 	cfg := Config{
 		CoinbaseAccount: tab.coinbase,
 		GoldenATXID:     tab.goldenATXID,
@@ -254,7 +258,10 @@ func TestBuilder_StartSmeshingCoinbase(t *testing.T) {
 
 	tab.mpost.EXPECT().PrepareInitializer(gomock.Any(), gomock.Any()).AnyTimes()
 	tab.mpost.EXPECT().StartSession(gomock.Any()).AnyTimes()
+	tab.mpost.EXPECT().LastOpts().Return(&PostSetupOpts{}).AnyTimes()
+	tab.mpost.EXPECT().CommitmentAtx().Return(tab.goldenATXID, nil).AnyTimes()
 	tab.mpost.EXPECT().GenerateProof(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(&types.Post{}, &types.PostMetadata{}, nil)
+	tab.mValidator.EXPECT().Post(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	tab.mclock.EXPECT().AwaitLayer(gomock.Any()).Return(make(chan struct{})).AnyTimes()
 	require.NoError(t, tab.StartSmeshing(coinbase, postSetupOpts))
 	require.Equal(t, coinbase, tab.Coinbase())
@@ -271,9 +278,15 @@ func TestBuilder_RestartSmeshing(t *testing.T) {
 	getBuilder := func(t *testing.T) *Builder {
 		tab := newTestBuilder(t)
 		tab.mpost.EXPECT().PrepareInitializer(gomock.Any(), gomock.Any()).AnyTimes()
+		tab.mpost.EXPECT().CommitmentAtx().Return(types.EmptyATXID, nil).AnyTimes()
+		tab.mpost.EXPECT().LastOpts().Return(&PostSetupOpts{}).AnyTimes()
 		tab.mpost.EXPECT().StartSession(gomock.Any()).AnyTimes()
-		tab.mpost.EXPECT().GenerateProof(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(&types.Post{}, &types.PostMetadata{}, nil)
+		tab.mpost.EXPECT().GenerateProof(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(&types.Post{}, &types.PostMetadata{
+			Challenge: shared.ZeroChallenge,
+		}, nil)
 		tab.mpost.EXPECT().Reset().AnyTimes()
+		tab.mValidator.EXPECT().Post(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
+		tab.mpost.EXPECT().Config().AnyTimes()
 		ch := make(chan struct{})
 		close(ch)
 		tab.mclock.EXPECT().AwaitLayer(gomock.Any()).Return(ch).AnyTimes()
@@ -311,6 +324,45 @@ func TestBuilder_RestartSmeshing(t *testing.T) {
 	})
 }
 
+func TestBuilder_StopSmeshing_Delete(t *testing.T) {
+	tab := newTestBuilder(t)
+
+	tab.mpost.EXPECT().PrepareInitializer(gomock.Any(), gomock.Any()).AnyTimes()
+	tab.mpost.EXPECT().StartSession(gomock.Any()).DoAndReturn(func(ctx context.Context) error {
+		// wait for stop to be called
+		<-ctx.Done()
+		return ctx.Err()
+	}).AnyTimes()
+
+	// Create state files
+	require.NoError(t, saveBuilderState(tab.nipostBuilder.DataDir(), &types.NIPostBuilderState{}))
+	require.NoError(t, savePost(tab.nipostBuilder.DataDir(), &types.Post{}))
+	require.NoError(t, SaveNipostChallenge(tab.nipostBuilder.DataDir(), &types.NIPostChallenge{}))
+	files, err := os.ReadDir(tab.nipostBuilder.DataDir())
+	require.NoError(t, err)
+	require.Len(t, files, 3) // 3 state files created
+
+	require.NoError(t, tab.StartSmeshing(types.Address{}, PostSetupOpts{}))
+	require.NoError(t, tab.StopSmeshing(false))
+	files, err = os.ReadDir(tab.nipostBuilder.DataDir())
+	require.NoError(t, err)
+	require.Len(t, files, 3) // state files still present
+
+	require.NoError(t, tab.StartSmeshing(types.Address{}, PostSetupOpts{}))
+	tab.mpost.EXPECT().Reset().Return(nil)
+	require.NoError(t, tab.StopSmeshing(true))
+	files, err = os.ReadDir(tab.nipostBuilder.DataDir())
+	require.NoError(t, err)
+	require.Len(t, files, 0) // state files deleted
+
+	require.NoError(t, tab.StartSmeshing(types.Address{}, PostSetupOpts{}))
+	tab.mpost.EXPECT().Reset().Return(nil)
+	require.NoError(t, tab.StopSmeshing(true)) // no-op
+	files, err = os.ReadDir(tab.nipostBuilder.DataDir())
+	require.NoError(t, err)
+	require.Len(t, files, 0) // state files still deleted
+}
+
 func TestBuilder_StoppingSmeshingBefore_Initialized(t *testing.T) {
 	tab := newTestBuilder(t)
 	tab.mpost.EXPECT().PrepareInitializer(gomock.Any(), gomock.Any()).AnyTimes()
@@ -386,7 +438,10 @@ func TestBuilder_StopSmeshing_OnPoSTError(t *testing.T) {
 	tab := newTestBuilder(t)
 	tab.mpost.EXPECT().PrepareInitializer(gomock.Any(), gomock.Any()).AnyTimes()
 	tab.mpost.EXPECT().StartSession(gomock.Any()).Return(nil).AnyTimes()
+	tab.mpost.EXPECT().CommitmentAtx().Return(types.EmptyATXID, nil).AnyTimes()
+	tab.mpost.EXPECT().LastOpts().Return(&PostSetupOpts{}).AnyTimes()
 	tab.mpost.EXPECT().GenerateProof(gomock.Any(), gomock.Any(), gomock.Any()).Return(&types.Post{}, &types.PostMetadata{}, nil).AnyTimes()
+	tab.mValidator.EXPECT().Post(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	ch := make(chan struct{})
 	close(ch)
 	now := time.Now()
@@ -1084,6 +1139,9 @@ func TestBuilder_RetryPublishActivationTx(t *testing.T) {
 func TestBuilder_InitialProofGeneratedOnce(t *testing.T) {
 	tab := newTestBuilder(t, WithPoetConfig(PoetConfig{PhaseShift: layerDuration * 4}))
 	tab.mpost.EXPECT().GenerateProof(gomock.Any(), shared.ZeroChallenge, gomock.Any()).Return(&types.Post{}, &types.PostMetadata{}, nil)
+	tab.mpost.EXPECT().LastOpts().Return(&PostSetupOpts{})
+	tab.mpost.EXPECT().CommitmentAtx().Return(tab.goldenATXID, nil)
+	tab.mValidator.EXPECT().Post(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	require.NoError(t, tab.generateInitialPost(context.Background()))
 
 	posEpoch := postGenesisEpoch + 1
@@ -1109,7 +1167,13 @@ func TestBuilder_InitialProofGeneratedOnce(t *testing.T) {
 
 func TestBuilder_InitialPostIsPersisted(t *testing.T) {
 	tab := newTestBuilder(t, WithPoetConfig(PoetConfig{PhaseShift: layerDuration * 4}))
-	tab.mpost.EXPECT().GenerateProof(gomock.Any(), shared.ZeroChallenge, gomock.Any()).Return(&types.Post{}, &types.PostMetadata{}, nil)
+	tab.mpost.EXPECT().Config().AnyTimes().Return(PostConfig{})
+	tab.mpost.EXPECT().LastOpts().Return(&PostSetupOpts{}).Times(3)
+	tab.mpost.EXPECT().CommitmentAtx().Return(tab.goldenATXID, nil).Times(3)
+	tab.mpost.EXPECT().GenerateProof(gomock.Any(), shared.ZeroChallenge, gomock.Any()).Return(&types.Post{}, &types.PostMetadata{
+		Challenge: shared.ZeroChallenge,
+	}, nil)
+	tab.mValidator.EXPECT().Post(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	require.NoError(t, tab.generateInitialPost(context.Background()))
 
 	// GenerateProof() should not be called again