diff --git a/activation/activation.go b/activation/activation.go index 11aa48898c..f858875c99 100644 --- a/activation/activation.go +++ b/activation/activation.go @@ -31,6 +31,8 @@ import ( "github.com/spacemeshos/go-spacemesh/sql/localsql/nipost" ) +var ErrNotFound = errors.New("not found") + // PoetConfig is the configuration to interact with the poet server. type PoetConfig struct { PhaseShift time.Duration `mapstructure:"phase-shift"` @@ -82,6 +84,7 @@ type Builder struct { syncer syncer log *zap.Logger parentCtx context.Context + poets []PoetClient poetCfg PoetConfig poetRetryInterval time.Duration // delay before PoST in ATX is considered valid (counting from the time it was received) @@ -138,6 +141,12 @@ func WithPoetConfig(c PoetConfig) BuilderOption { } } +func WithPoets(poets ...PoetClient) BuilderOption { + return func(b *Builder) { + b.poets = poets + } +} + func WithValidator(v nipostValidator) BuilderOption { return func(b *Builder) { b.validator = v @@ -323,7 +332,7 @@ func (b *Builder) buildInitialPost(ctx context.Context, nodeID types.NodeID) err return nil } // ...and if we haven't stored an initial post yet. - _, err := nipost.InitialPost(b.localDB, nodeID) + _, err := nipost.GetPost(b.localDB, nodeID) switch { case err == nil: b.log.Info("load initial post from db") @@ -347,9 +356,10 @@ func (b *Builder) buildInitialPost(ctx context.Context, nodeID types.NodeID) err return errors.New("nil VRF nonce") } initialPost := nipost.Post{ - Nonce: post.Nonce, - Indices: post.Indices, - Pow: post.Pow, + Nonce: post.Nonce, + Indices: post.Indices, + Pow: post.Pow, + Challenge: shared.ZeroChallenge, NumUnits: postInfo.NumUnits, CommitmentATX: postInfo.CommitmentATX, @@ -361,7 +371,7 @@ func (b *Builder) buildInitialPost(ctx context.Context, nodeID types.NodeID) err }, postInfo.NumUnits) if err != nil { b.log.Error("initial POST is invalid", log.ZShortStringer("smesherID", nodeID), zap.Error(err)) - if err := nipost.RemoveInitialPost(b.localDB, nodeID); err != nil { + if err := nipost.RemovePost(b.localDB, nodeID); err != nil { b.log.Fatal("failed to remove initial post", log.ZShortStringer("smesherID", nodeID), zap.Error(err)) } return fmt.Errorf("initial POST is invalid: %w", err) @@ -371,7 +381,7 @@ func (b *Builder) buildInitialPost(ctx context.Context, nodeID types.NodeID) err public.PostSeconds.Set(float64(time.Since(startTime))) b.log.Info("created the initial post") - return nipost.AddInitialPost(b.localDB, nodeID, initialPost) + return nipost.AddPost(b.localDB, nodeID, initialPost) } func (b *Builder) run(ctx context.Context, sig *signing.EdSigner) { @@ -390,6 +400,17 @@ func (b *Builder) run(ctx context.Context, sig *signing.EdSigner) { case <-b.layerClock.AwaitLayer(currentLayer.Add(1)): } } + var eg errgroup.Group + for _, poet := range b.poets { + eg.Go(func() error { + _, err := poet.Certify(ctx, sig.NodeID()) + if err != nil { + b.log.Warn("failed to certify poet", zap.Error(err), log.ZShortStringer("smesherID", sig.NodeID())) + } + return nil + }) + } + eg.Wait() for { err := b.PublishActivationTx(ctx, sig) @@ -515,7 +536,7 @@ func (b *Builder) BuildNIPostChallenge(ctx context.Context, nodeID types.NodeID) switch { case errors.Is(err, sql.ErrNotFound): logger.Info("no previous ATX found, creating an initial nipost challenge") - post, err := nipost.InitialPost(b.localDB, nodeID) + post, err := nipost.GetPost(b.localDB, nodeID) if err != nil { return nil, fmt.Errorf("get initial post: %w", err) } @@ -531,7 +552,7 @@ func (b *Builder) BuildNIPostChallenge(ctx context.Context, nodeID types.NodeID) }, post.NumUnits) if err != nil { logger.Error("initial POST is invalid", zap.Error(err)) - if err := nipost.RemoveInitialPost(b.localDB, nodeID); err != nil { + if err := nipost.RemovePost(b.localDB, nodeID); err != nil { logger.Fatal("failed to remove initial post", zap.Error(err)) } return nil, fmt.Errorf("initial POST is invalid: %w", err) diff --git a/activation/activation_multi_test.go b/activation/activation_multi_test.go index 41e868d2d6..55dcc360ac 100644 --- a/activation/activation_multi_test.go +++ b/activation/activation_multi_test.go @@ -252,10 +252,12 @@ func Test_Builder_Multi_InitialPost(t *testing.T) { }, nil, ) - require.NoError(t, tab.buildInitialPost(context.Background(), sig.NodeID())) + err := tab.buildInitialPost(context.Background(), sig.NodeID()) + require.NoError(t, err) // postClient.Proof() should not be called again - require.NoError(t, tab.buildInitialPost(context.Background(), sig.NodeID())) + err = tab.buildInitialPost(context.Background(), sig.NodeID()) + require.NoError(t, err) return nil }) } @@ -397,7 +399,9 @@ func Test_Builder_Multi_HappyPath(t *testing.T) { VRFNonce: types.VRFPostIndex(rand.Uint64()), } nipostState[sig.NodeID()] = state - tab.mnipost.EXPECT().BuildNIPost(gomock.Any(), sig, ref.PublishEpoch, ref.Hash()).Return(state, nil) + tab.mnipost.EXPECT(). + BuildNIPost(gomock.Any(), sig, ref.PublishEpoch, ref.Hash()). + Return(state, nil) // awaiting atx publication epoch log tab.mclock.EXPECT().CurrentLayer().DoAndReturn( diff --git a/activation/activation_test.go b/activation/activation_test.go index e6131e3b52..08eb2b7022 100644 --- a/activation/activation_test.go +++ b/activation/activation_test.go @@ -666,8 +666,9 @@ func TestBuilder_PublishActivationTx_NoPrevATX(t *testing.T) { NumUnits: uint32(12), CommitmentATX: types.RandomATXID(), VRFNonce: types.VRFPostIndex(rand.Uint64()), + Challenge: shared.ZeroChallenge, } - require.NoError(t, nipost.AddInitialPost(tab.localDb, sig.NodeID(), post)) + require.NoError(t, nipost.AddPost(tab.localDb, sig.NodeID(), post)) initialPost := &types.Post{ Nonce: post.Nonce, Indices: post.Indices, @@ -706,8 +707,9 @@ func TestBuilder_PublishActivationTx_NoPrevATX_PublishFails_InitialPost_preserve NumUnits: uint32(12), CommitmentATX: types.RandomATXID(), VRFNonce: types.VRFPostIndex(rand.Uint64()), + Challenge: shared.ZeroChallenge, } - require.NoError(t, nipost.AddInitialPost(tab.localDb, sig.NodeID(), refPost)) + require.NoError(t, nipost.AddPost(tab.localDb, sig.NodeID(), refPost)) initialPost := &types.Post{ Nonce: refPost.Nonce, Indices: refPost.Indices, @@ -728,12 +730,10 @@ func TestBuilder_PublishActivationTx_NoPrevATX_PublishFails_InitialPost_preserve genesis := time.Now().Add(-time.Duration(currLayer) * layerDuration) return genesis.Add(layerDuration * time.Duration(got)) }).AnyTimes() - tab.mnipost.EXPECT(). BuildNIPost(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, ErrATXChallengeExpired) tab.mnipost.EXPECT().ResetState(sig.NodeID()).Return(nil) - ch := make(chan struct{}) tab.mclock.EXPECT().AwaitLayer(currLayer.Add(1)).Do(func(got types.LayerID) <-chan struct{} { close(ch) @@ -758,7 +758,7 @@ func TestBuilder_PublishActivationTx_NoPrevATX_PublishFails_InitialPost_preserve } // initial post is preserved - post, err := nipost.InitialPost(tab.localDB, sig.NodeID()) + post, err := nipost.GetPost(tab.localDB, sig.NodeID()) require.NoError(t, err) require.NotNil(t, post) require.Equal(t, refPost, *post) @@ -824,11 +824,13 @@ func TestBuilder_PublishActivationTx_PrevATXWithoutPrevATX(t *testing.T) { LabelsPerUnit: DefaultPostConfig().LabelsPerUnit, }, nil).AnyTimes() - tab.mnipost.EXPECT().BuildNIPost(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ *signing.EdSigner, _ types.EpochID, _ types.Hash32) (*nipost.NIPostState, error) { - currentLayer = currentLayer.Add(5) - return newNIPostWithPoet(t, poetBytes), nil - }) + tab.mnipost.EXPECT(). + BuildNIPost(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn( + func(_ context.Context, _ *signing.EdSigner, _ types.EpochID, _ types.Hash32) (*nipost.NIPostState, error) { + currentLayer = currentLayer.Add(5) + return newNIPostWithPoet(t, poetBytes), nil + }) tab.mValidator.EXPECT().VerifyChain(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) @@ -938,8 +940,9 @@ func TestBuilder_PublishActivationTx_TargetsEpochBasedOnPosAtx(t *testing.T) { NumUnits: uint32(12), CommitmentATX: types.RandomATXID(), VRFNonce: types.VRFPostIndex(rand.Uint64()), + Challenge: shared.ZeroChallenge, } - require.NoError(t, nipost.AddInitialPost(tab.localDb, sig.NodeID(), post)) + require.NoError(t, nipost.AddPost(tab.localDb, sig.NodeID(), post)) initialPost := &types.Post{ Nonce: post.Nonce, Indices: post.Indices, @@ -979,7 +982,9 @@ func TestBuilder_PublishActivationTx_FailsWhenNIPostBuilderFails(t *testing.T) { return genesis.Add(layerDuration * time.Duration(got)) }).AnyTimes() nipostErr := errors.New("NIPost builder error") - tab.mnipost.EXPECT().BuildNIPost(gomock.Any(), sig, gomock.Any(), gomock.Any()).Return(nil, nipostErr) + tab.mnipost.EXPECT(). + BuildNIPost(gomock.Any(), sig, gomock.Any(), gomock.Any()). + Return(nil, nipostErr) tab.mValidator.EXPECT().VerifyChain(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) require.ErrorIs(t, tab.PublishActivationTx(context.Background(), sig), nipostErr) @@ -1027,7 +1032,6 @@ func TestBuilder_RetryPublishActivationTx(t *testing.T) { require.NoError(t, atxs.Add(tab.db, toAtx(t, prevAtx))) tab.atxsdata.AddFromAtx(toAtx(t, prevAtx), false) - publishEpoch := prevAtx.PublishEpoch + 1 currLayer := prevAtx.PublishEpoch.FirstLayer() tab.mclock.EXPECT().CurrentLayer().DoAndReturn(func() types.LayerID { return currLayer }).AnyTimes() tab.mclock.EXPECT().LayerToTime(gomock.Any()).DoAndReturn( @@ -1038,7 +1042,7 @@ func TestBuilder_RetryPublishActivationTx(t *testing.T) { }).AnyTimes() done := make(chan struct{}) close(done) - tab.mclock.EXPECT().AwaitLayer(publishEpoch.FirstLayer()).DoAndReturn( + tab.mclock.EXPECT().AwaitLayer(prevAtx.PublishEpoch.Add(1).FirstLayer()).DoAndReturn( func(got types.LayerID) <-chan struct{} { // advance to publish layer if currLayer.Before(got) { @@ -1125,9 +1129,6 @@ func TestBuilder_RetryPublishActivationTx(t *testing.T) { } // state is cleaned up - _, err = nipost.InitialPost(tab.localDB, sig.NodeID()) - require.ErrorIs(t, err, sql.ErrNotFound) - _, err = nipost.Challenge(tab.localDB, sig.NodeID()) require.ErrorIs(t, err, sql.ErrNotFound) } @@ -1169,7 +1170,8 @@ func TestBuilder_InitialProofGeneratedOnce(t *testing.T) { tab.mValidator.EXPECT().Post(gomock.Any(), sig.NodeID(), post.CommitmentATX, initialPost, metadata, post.NumUnits) require.NoError(t, tab.buildInitialPost(context.Background(), sig.NodeID())) // postClient.Proof() should not be called again - require.NoError(t, tab.buildInitialPost(context.Background(), sig.NodeID())) + err := tab.buildInitialPost(context.Background(), sig.NodeID()) + require.NoError(t, err) } func TestBuilder_InitialPostIsPersisted(t *testing.T) { @@ -1201,10 +1203,12 @@ func TestBuilder_InitialPostIsPersisted(t *testing.T) { LabelsPerUnit: tab.conf.LabelsPerUnit, } tab.mValidator.EXPECT().Post(gomock.Any(), sig.NodeID(), commitmentATX, initialPost, metadata, numUnits) - require.NoError(t, tab.buildInitialPost(context.Background(), sig.NodeID())) + err := tab.buildInitialPost(context.Background(), sig.NodeID()) + require.NoError(t, err) // postClient.Proof() should not be called again - require.NoError(t, tab.buildInitialPost(context.Background(), sig.NodeID())) + err = tab.buildInitialPost(context.Background(), sig.NodeID()) + require.NoError(t, err) } func TestBuilder_InitialPostLogErrorMissingVRFNonce(t *testing.T) { @@ -1234,7 +1238,8 @@ func TestBuilder_InitialPostLogErrorMissingVRFNonce(t *testing.T) { LabelsPerUnit: tab.conf.LabelsPerUnit, } tab.mValidator.EXPECT().Post(gomock.Any(), sig.NodeID(), commitmentATX, initialPost, metadata, numUnits) - require.ErrorContains(t, tab.buildInitialPost(context.Background(), sig.NodeID()), "nil VRF nonce") + err := tab.buildInitialPost(context.Background(), sig.NodeID()) + require.ErrorContains(t, err, "nil VRF nonce") observedLogs := tab.observedLogs.FilterLevelExact(zapcore.ErrorLevel) require.Equal(t, 1, observedLogs.Len(), "expected 1 log message") @@ -1256,7 +1261,8 @@ func TestBuilder_InitialPostLogErrorMissingVRFNonce(t *testing.T) { }, nil, ) - require.NoError(t, tab.buildInitialPost(context.Background(), sig.NodeID())) + err = tab.buildInitialPost(context.Background(), sig.NodeID()) + require.NoError(t, err) } func TestWaitPositioningAtx(t *testing.T) { @@ -1287,7 +1293,8 @@ func TestWaitPositioningAtx(t *testing.T) { // everything else are stubs that are irrelevant for the test tab.mpostClient.EXPECT().Info(gomock.Any()).Return(&types.PostInfo{}, nil).AnyTimes() tab.mnipost.EXPECT().ResetState(sig.NodeID()).Return(nil) - tab.mnipost.EXPECT().BuildNIPost(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + tab.mnipost.EXPECT(). + BuildNIPost(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(&nipost.NIPostState{}, nil) closed := make(chan struct{}) close(closed) @@ -1309,8 +1316,9 @@ func TestWaitPositioningAtx(t *testing.T) { NumUnits: uint32(12), CommitmentATX: types.RandomATXID(), VRFNonce: types.VRFPostIndex(rand.Uint64()), + Challenge: shared.ZeroChallenge, } - require.NoError(t, nipost.AddInitialPost(tab.localDb, sig.NodeID(), post)) + require.NoError(t, nipost.AddPost(tab.localDb, sig.NodeID(), post)) initialPost := &types.Post{ Nonce: post.Nonce, Indices: post.Indices, diff --git a/activation/certifier.go b/activation/certifier.go new file mode 100644 index 0000000000..4ac3c4aced --- /dev/null +++ b/activation/certifier.go @@ -0,0 +1,349 @@ +package activation + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/hashicorp/go-retryablehttp" + "github.com/spacemeshos/poet/shared" + "go.uber.org/zap" + "golang.org/x/sync/singleflight" + + "github.com/spacemeshos/go-spacemesh/activation/wire" + "github.com/spacemeshos/go-spacemesh/codec" + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/atxs" + "github.com/spacemeshos/go-spacemesh/sql/localsql" + certifierdb "github.com/spacemeshos/go-spacemesh/sql/localsql/certifier" + "github.com/spacemeshos/go-spacemesh/sql/localsql/nipost" +) + +type CertifierClientConfig struct { + // Base delay between retries, scaled with the number of retries. + RetryDelay time.Duration `mapstructure:"retry-delay"` + // Maximum time to wait between retries + MaxRetryDelay time.Duration `mapstructure:"max-retry-delay"` + // Maximum number of retries + MaxRetries int `mapstructure:"max-retries"` +} + +type CertifierConfig struct { + Client CertifierClientConfig `mapstructure:"client"` +} + +func DefaultCertifierClientConfig() CertifierClientConfig { + return CertifierClientConfig{ + RetryDelay: 1 * time.Second, + MaxRetryDelay: 30 * time.Second, + MaxRetries: 5, + } +} + +func DefaultCertifierConfig() CertifierConfig { + return CertifierConfig{ + Client: DefaultCertifierClientConfig(), + } +} + +type ProofToCertify struct { + Nonce uint32 `json:"nonce"` + Indices []byte `json:"indices"` + Pow uint64 `json:"pow"` +} + +type ProofToCertifyMetadata struct { + NodeId []byte `json:"node_id"` + CommitmentAtxId []byte `json:"commitment_atx_id"` + + Challenge []byte `json:"challenge"` + NumUnits uint32 `json:"num_units"` +} + +type CertifyRequest struct { + Proof ProofToCertify `json:"proof"` + Metadata ProofToCertifyMetadata `json:"metadata"` +} + +type CertifyResponse struct { + Certificate []byte `json:"certificate"` + Signature []byte `json:"signature"` + PubKey []byte `json:"pub_key"` +} + +type Certifier struct { + logger *zap.Logger + db *localsql.Database + client certifierClient + + certifications singleflight.Group +} + +func NewCertifier( + db *localsql.Database, + logger *zap.Logger, + client certifierClient, +) *Certifier { + c := &Certifier{ + client: client, + logger: logger, + db: db, + } + + return c +} + +func (c *Certifier) Certificate( + ctx context.Context, + id types.NodeID, + certifier *url.URL, + pubkey []byte, +) (*certifierdb.PoetCert, error) { + // We index certs in DB by node ID and pubkey. To avoid redundant queries, we allow only 1 + // request per (nodeID, pubkey) pair to be in flight at a time. + key := string(append(id.Bytes(), pubkey...)) + cert, err, _ := c.certifications.Do(key, func() (any, error) { + cert, err := certifierdb.Certificate(c.db, id, pubkey) + switch { + case err == nil: + return cert, nil + case !errors.Is(err, sql.ErrNotFound): + return nil, fmt.Errorf("getting certificate from DB for: %w", err) + } + return c.Recertify(ctx, id, certifier, pubkey) + }) + + if err != nil { + return nil, err + } + return cert.(*certifierdb.PoetCert), nil +} + +func (c *Certifier) Recertify( + ctx context.Context, + id types.NodeID, + certifier *url.URL, + pubkey []byte, +) (*certifierdb.PoetCert, error) { + cert, err := c.client.Certify(ctx, id, certifier, pubkey) + if err != nil { + return nil, fmt.Errorf("certifying POST at %v: %w", certifier, err) + } + + if err := certifierdb.AddCertificate(c.db, id, *cert, pubkey); err != nil { + c.logger.Warn("failed to persist poet cert", zap.Error(err)) + } + return cert, nil +} + +type CertifierClient struct { + client *retryablehttp.Client + logger *zap.Logger + db sql.Executor + localDb *localsql.Database +} + +type certifierClientOpts func(*CertifierClient) + +func WithCertifierClientConfig(cfg CertifierClientConfig) certifierClientOpts { + return func(c *CertifierClient) { + c.client.RetryMax = cfg.MaxRetries + c.client.RetryWaitMin = cfg.RetryDelay + c.client.RetryWaitMax = cfg.MaxRetryDelay + } +} + +func NewCertifierClient( + db sql.Executor, + localDb *localsql.Database, + logger *zap.Logger, + opts ...certifierClientOpts, +) *CertifierClient { + c := &CertifierClient{ + client: retryablehttp.NewClient(), + logger: logger, + db: db, + localDb: localDb, + } + config := DefaultCertifierClientConfig() + c.client.RetryMax = config.MaxRetries + c.client.RetryWaitMin = config.RetryDelay + c.client.RetryWaitMax = config.MaxRetryDelay + c.client.Logger = &retryableHttpLogger{logger} + c.client.ResponseLogHook = func(logger retryablehttp.Logger, resp *http.Response) { + c.logger.Info("response received", zap.Stringer("url", resp.Request.URL), zap.Int("status", resp.StatusCode)) + } + + for _, opt := range opts { + opt(c) + } + + return c +} + +func (c *CertifierClient) obtainPostFromLastAtx(ctx context.Context, nodeId types.NodeID) (*nipost.Post, error) { + atxid, err := atxs.GetLastIDByNodeID(c.db, nodeId) + if err != nil { + return nil, fmt.Errorf("no existing ATX found: %w", err) + } + atx, err := atxs.Get(c.db, atxid) + if err != nil { + return nil, fmt.Errorf("failed to retrieve ATX: %w", err) + } + atxNipost, err := loadNipost(ctx, c.db, atxid) + if err != nil { + return nil, errors.New("no NIPoST found in last ATX") + } + if atx.CommitmentATX == nil { + if commitmentAtx, err := atxs.CommitmentATX(c.db, nodeId); err != nil { + return nil, fmt.Errorf("failed to retrieve commitment ATX: %w", err) + } else { + atx.CommitmentATX = &commitmentAtx + } + } + + c.logger.Info("found POST in an existing ATX", zap.String("atx_id", atxid.Hash32().ShortString())) + return &nipost.Post{ + Nonce: atxNipost.Post.Nonce, + Indices: atxNipost.Post.Indices, + Pow: atxNipost.Post.Pow, + Challenge: atxNipost.PostMetadata.Challenge, + NumUnits: atx.NumUnits, + CommitmentATX: *atx.CommitmentATX, + // VRF nonce is not needed + }, nil +} + +func (c *CertifierClient) obtainPost(ctx context.Context, id types.NodeID) (*nipost.Post, error) { + c.logger.Info("looking for POST for poet certification") + post, err := nipost.GetPost(c.localDb, id) + switch { + case err == nil: + c.logger.Info("found POST in local DB") + return post, nil + case errors.Is(err, sql.ErrNotFound): + // no post found + default: + return nil, fmt.Errorf("loading initial post from db: %w", err) + } + + c.logger.Info("POST not found in local DB. Trying to obtain POST from an existing ATX") + if post, err := c.obtainPostFromLastAtx(ctx, id); err == nil { + c.logger.Info("found POST in an existing ATX") + if err := nipost.AddPost(c.localDb, id, *post); err != nil { + c.logger.Error("failed to save post", zap.Error(err)) + } + return post, nil + } + + return nil, errors.New("PoST not found") +} + +func (c *CertifierClient) Certify( + ctx context.Context, + id types.NodeID, + url *url.URL, + pubkey []byte, +) (*certifierdb.PoetCert, error) { + post, err := c.obtainPost(ctx, id) + if err != nil { + return nil, fmt.Errorf("obtaining PoST: %w", err) + } + + request := CertifyRequest{ + Proof: ProofToCertify{ + Pow: post.Pow, + Nonce: post.Nonce, + Indices: post.Indices, + }, + Metadata: ProofToCertifyMetadata{ + NodeId: id[:], + CommitmentAtxId: post.CommitmentATX[:], + NumUnits: post.NumUnits, + Challenge: post.Challenge, + }, + } + + jsonRequest, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("marshaling request: %w", err) + } + + req, err := retryablehttp.NewRequestWithContext(ctx, "POST", url.JoinPath("/certify").String(), jsonRequest) + if err != nil { + return nil, fmt.Errorf("creating HTTP request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("sending request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + var body []byte + if resp.Body != nil { + body, _ = io.ReadAll(resp.Body) + } + return nil, fmt.Errorf("request failed with code %d (message: %s)", resp.StatusCode, body) + } + + certResponse := CertifyResponse{} + if err := json.NewDecoder(resp.Body).Decode(&certResponse); err != nil { + return nil, fmt.Errorf("decoding JSON response: %w", err) + } + if !bytes.Equal(certResponse.PubKey, pubkey) { + return nil, errors.New("pubkey is invalid") + } + + opaqueCert := &shared.OpaqueCert{ + Data: certResponse.Certificate, + Signature: certResponse.Signature, + } + + cert, err := shared.VerifyCertificate(opaqueCert, pubkey, id.Bytes()) + if err != nil { + return nil, fmt.Errorf("verifying certificate: %w", err) + } + + if cert.Expiration != nil { + c.logger.Info("certificate has expiration date", zap.Time("expiration", *cert.Expiration)) + if time.Until(*cert.Expiration) < 0 { + return nil, errors.New("certificate is expired") + } + } + + return &certifierdb.PoetCert{ + Data: opaqueCert.Data, + Signature: opaqueCert.Signature, + }, nil +} + +// load NIPoST for the given ATX from the database. +func loadNipost(ctx context.Context, db sql.Executor, id types.ATXID) (*types.NIPost, error) { + var blob sql.Blob + version, err := atxs.LoadBlob(ctx, db, id.Bytes(), &blob) + if err != nil { + return nil, fmt.Errorf("getting blob for %s: %w", id, err) + } + + switch version { + case types.AtxV1: + var atx wire.ActivationTxV1 + if err := codec.Decode(blob.Bytes, &atx); err != nil { + return nil, fmt.Errorf("decoding ATX blob: %w", err) + } + return wire.NiPostFromWireV1(atx.NIPost), nil + case types.AtxV2: + // TODO: support ATX V2 + } + panic("unsupported ATX version") +} diff --git a/activation/certifier_test.go b/activation/certifier_test.go new file mode 100644 index 0000000000..cc329737da --- /dev/null +++ b/activation/certifier_test.go @@ -0,0 +1,162 @@ +package activation + +import ( + "context" + "net/url" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "go.uber.org/zap/zaptest" + "golang.org/x/sync/errgroup" + + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/atxs" + "github.com/spacemeshos/go-spacemesh/sql/localsql" + certdb "github.com/spacemeshos/go-spacemesh/sql/localsql/certifier" + "github.com/spacemeshos/go-spacemesh/sql/localsql/nipost" +) + +func TestPersistsCerts(t *testing.T) { + client := NewMockcertifierClient(gomock.NewController(t)) + id := types.RandomNodeID() + db := localsql.InMemory() + cert := &certdb.PoetCert{Data: []byte("cert"), Signature: []byte("sig")} + certifierAddress := &url.URL{Scheme: "http", Host: "certifier.org"} + pubkey := []byte("pubkey") + { + c := NewCertifier(db, zaptest.NewLogger(t), client) + client.EXPECT(). + Certify(gomock.Any(), id, certifierAddress, pubkey). + Return(cert, nil) + + _, err := certdb.Certificate(db, id, pubkey) + require.ErrorIs(t, err, sql.ErrNotFound) + got, err := c.Certificate(context.Background(), id, certifierAddress, pubkey) + require.NoError(t, err) + require.Equal(t, cert, got) + + got, err = c.Certificate(context.Background(), id, certifierAddress, pubkey) + require.NoError(t, err) + require.Equal(t, cert, got) + + got, err = certdb.Certificate(db, id, pubkey) + require.NoError(t, err) + require.Equal(t, cert, got) + } + { + // Create new certifier and check that it loads the certs back. + c := NewCertifier(db, zaptest.NewLogger(t), client) + got, err := c.Certificate(context.Background(), id, certifierAddress, pubkey) + require.NoError(t, err) + require.Equal(t, cert, got) + } +} + +func TestAvoidsRedundantQueries(t *testing.T) { + db := localsql.InMemory() + client := NewMockcertifierClient(gomock.NewController(t)) + id1 := types.RandomNodeID() + id2 := types.RandomNodeID() + cert1 := &certdb.PoetCert{Data: []byte("1"), Signature: []byte("sig")} + cert2 := &certdb.PoetCert{Data: []byte("2"), Signature: []byte("sig")} + cert3 := &certdb.PoetCert{Data: []byte("3"), Signature: []byte("sig")} + certifierAddress := &url.URL{Scheme: "http", Host: "certifier.org"} + pubkey := []byte("pubkey") + pubkey2 := []byte("pubkey2") + + c := NewCertifier(db, zaptest.NewLogger(t), client) + // The key is (id, pubkey) so we should only have one request in flight at a time + // for a given pair. + client.EXPECT().Certify(gomock.Any(), id1, certifierAddress, pubkey).Return(cert1, nil) + client.EXPECT().Certify(gomock.Any(), id2, certifierAddress, pubkey).Return(cert2, nil) + client.EXPECT().Certify(gomock.Any(), id1, certifierAddress, pubkey2).Return(cert3, nil) + + var eg errgroup.Group + for i := 0; i < 10; i++ { + eg.Go(func() error { + got, err := c.Certificate(context.Background(), id1, certifierAddress, pubkey) + require.NoError(t, err) + require.Equal(t, cert1, got) + return nil + }) + eg.Go(func() error { + got, err := c.Certificate(context.Background(), id2, certifierAddress, pubkey) + require.NoError(t, err) + require.Equal(t, cert2, got) + return nil + }) + eg.Go(func() error { + got, err := c.Certificate(context.Background(), id1, certifierAddress, pubkey2) + require.NoError(t, err) + require.Equal(t, cert3, got) + return nil + }) + } + eg.Wait() + + got, err := certdb.Certificate(db, id1, pubkey) + require.NoError(t, err) + require.Equal(t, cert1, got) + // different id - different cert + got, err = certdb.Certificate(db, id2, pubkey) + require.NoError(t, err) + require.Equal(t, cert2, got) + // different pubkey - different cert + got, err = certdb.Certificate(db, id1, pubkey2) + require.NoError(t, err) + require.Equal(t, cert3, got) +} + +func TestObtainingPost(t *testing.T) { + id := types.RandomNodeID() + + t.Run("no POST or ATX", func(t *testing.T) { + db := sql.InMemory() + localDb := localsql.InMemory() + + certifier := NewCertifierClient(db, localDb, zaptest.NewLogger(t)) + _, err := certifier.obtainPost(context.Background(), id) + require.ErrorContains(t, err, "PoST not found") + }) + t.Run("initial POST available", func(t *testing.T) { + db := sql.InMemory() + localDb := localsql.InMemory() + + post := nipost.Post{ + Nonce: 30, + Indices: types.RandomBytes(20), + Pow: 17, + Challenge: types.RandomBytes(32), + NumUnits: 2, + CommitmentATX: types.RandomATXID(), + VRFNonce: 15, + } + err := nipost.AddPost(localDb, id, post) + require.NoError(t, err) + + certifier := NewCertifierClient(db, localDb, zaptest.NewLogger(t)) + got, err := certifier.obtainPost(context.Background(), id) + require.NoError(t, err) + require.Equal(t, post, *got) + }) + t.Run("initial POST unavailable but ATX exists", func(t *testing.T) { + db := sql.InMemory() + localDb := localsql.InMemory() + + atx := newInitialATXv1(t, types.RandomATXID()) + atx.SmesherID = id + require.NoError(t, atxs.Add(db, toAtx(t, atx))) + + certifier := NewCertifierClient(db, localDb, zaptest.NewLogger(t)) + got, err := certifier.obtainPost(context.Background(), id) + require.NoError(t, err) + require.Equal(t, atx.NIPost.Post.Indices, got.Indices) + require.Equal(t, atx.NIPost.Post.Nonce, got.Nonce) + require.Equal(t, atx.NIPost.Post.Pow, got.Pow) + require.Equal(t, atx.NIPost.PostMetadata.Challenge, got.Challenge) + require.Equal(t, atx.NumUnits, got.NumUnits) + require.Equal(t, atx.CommitmentATXID, &got.CommitmentATX) + }) +} diff --git a/activation/e2e/activation_test.go b/activation/e2e/activation_test.go index 352516517f..7bbdb3be8b 100644 --- a/activation/e2e/activation_test.go +++ b/activation/e2e/activation_test.go @@ -2,12 +2,16 @@ package activation_test import ( "context" + "net/url" "sync" "sync/atomic" "testing" "time" + "github.com/spacemeshos/poet/registration" + poetShared "github.com/spacemeshos/poet/shared" "github.com/spacemeshos/post/initialization" + "github.com/spacemeshos/post/verifying" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -50,6 +54,7 @@ func Test_BuilderWithMultipleClients(t *testing.T) { goldenATX := types.ATXID{2, 3, 4} cfg := activation.DefaultPostConfig() db := sql.InMemory() + localDB := localsql.InMemory() syncer := activation.NewMocksyncer(ctrl) syncer.EXPECT().RegisterForATXSynced().DoAndReturn(func() <-chan struct{} { @@ -103,39 +108,52 @@ func Test_BuilderWithMultipleClients(t *testing.T) { RequestRetryDelay: epoch / 50, MaxRequestRetries: 10, } + + pubkey, address := spawnTestCertifier( + t, + cfg, + func(id []byte) *poetShared.Cert { + exp := time.Now().Add(epoch) + return &poetShared.Cert{Pubkey: id, Expiration: &exp} + }, + verifying.WithLabelScryptParams(opts.Scrypt), + ) + poetProver := spawnPoet( t, WithGenesis(genesis), WithEpochDuration(epoch), WithPhaseShift(poetCfg.PhaseShift), WithCycleGap(poetCfg.CycleGap), + WithCertifier(®istration.CertifierConfig{ + URL: (&url.URL{Scheme: "http", Host: address.String()}).String(), + PubKey: registration.Base64Enc(pubkey), + }), ) + certClient := activation.NewCertifierClient(db, localDB, logger.Named("certifier")) + certifier := activation.NewCertifier(localDB, logger, certClient) + poetDb := activation.NewPoetDb(db, log.NewFromLog(logger).Named("poetDb")) + client, err := poetProver.Client(poetDb, poetCfg, logger, activation.WithCertifier(certifier)) + require.NoError(t, err) clock, err := timesync.NewClock( timesync.WithGenesisTime(genesis), timesync.WithLayerDuration(layerDuration), timesync.WithTickInterval(100*time.Millisecond), - timesync.WithLogger(logger), + timesync.WithLogger(zap.NewNop()), ) require.NoError(t, err) t.Cleanup(clock.Close) - poetDb := activation.NewPoetDb(db, log.NewFromLog(logger).Named("poetDb")) - postStates := activation.NewMockPostStates(ctrl) - localDB := localsql.InMemory() nb, err := activation.NewNIPostBuilder( localDB, - poetDb, svc, - []types.PoetServer{{ - Pubkey: types.NewBase64Enc(poetProver.Service.PublicKey()), - Address: poetProver.RestURL().String(), - }}, logger.Named("nipostBuilder"), poetCfg, clock, activation.NipostbuilderWithPostStates(postStates), + activation.WithPoetClients(client), ) require.NoError(t, err) @@ -191,6 +209,7 @@ func Test_BuilderWithMultipleClients(t *testing.T) { activation.WithPoetConfig(poetCfg), activation.WithValidator(v), activation.WithPostStates(postStates), + activation.WithPoets(client), ) for _, sig := range signers { gomock.InOrder( diff --git a/activation/e2e/certifier_client_test.go b/activation/e2e/certifier_client_test.go new file mode 100644 index 0000000000..26da2c7df4 --- /dev/null +++ b/activation/e2e/certifier_client_test.go @@ -0,0 +1,225 @@ +package activation_test + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "fmt" + "net" + "net/http" + "net/url" + "testing" + "time" + + poetShared "github.com/spacemeshos/poet/shared" + "github.com/spacemeshos/post/initialization" + "github.com/spacemeshos/post/shared" + "github.com/spacemeshos/post/verifying" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "go.uber.org/zap/zaptest" + "golang.org/x/sync/errgroup" + + "github.com/spacemeshos/go-spacemesh/activation" + "github.com/spacemeshos/go-spacemesh/api/grpcserver" + "github.com/spacemeshos/go-spacemesh/atxsdata" + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/signing" + "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/localsql" + "github.com/spacemeshos/go-spacemesh/sql/localsql/nipost" +) + +func TestCertification(t *testing.T) { + sig, err := signing.NewEdSigner() + require.NoError(t, err) + + logger := zaptest.NewLogger(t) + cfg := activation.DefaultPostConfig() + db := sql.InMemory() + localDb := localsql.InMemory() + + syncer := activation.NewMocksyncer(gomock.NewController(t)) + synced := make(chan struct{}) + close(synced) + syncer.EXPECT().RegisterForATXSynced().AnyTimes().Return(synced) + + validator := activation.NewMocknipostValidator(gomock.NewController(t)) + validator.EXPECT(). + Post(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + AnyTimes() + validator.EXPECT().VerifyChain(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + mgr, err := activation.NewPostSetupManager(cfg, logger, db, atxsdata.New(), types.ATXID{2, 3, 4}, syncer, validator) + require.NoError(t, err) + + opts := activation.DefaultPostSetupOpts() + opts.DataDir = t.TempDir() + opts.ProviderID.SetUint32(initialization.CPUProviderID()) + opts.Scrypt.N = 2 // Speedup initialization in tests. + initPost(t, mgr, opts, sig.NodeID()) + + svc := grpcserver.NewPostService(logger) + svc.AllowConnections(true) + grpcCfg, cleanup := launchServer(t, svc) + t.Cleanup(cleanup) + t.Cleanup(launchPostSupervisor(t, logger, mgr, sig, grpcCfg, opts)) + + var postClient activation.PostClient + require.Eventually(t, func() bool { + var err error + postClient, err = svc.Client(sig.NodeID()) + return err == nil + }, 10*time.Second, 100*time.Millisecond, "timed out waiting for connection") + post, info, err := postClient.Proof(context.Background(), shared.ZeroChallenge) + require.NoError(t, err) + + fullPost := nipost.Post{ + Nonce: post.Nonce, + Indices: post.Indices, + Pow: post.Pow, + Challenge: shared.ZeroChallenge, + NumUnits: info.NumUnits, + CommitmentATX: info.CommitmentATX, + VRFNonce: *info.Nonce, + } + err = nipost.AddPost(localDb, sig.NodeID(), fullPost) + require.NoError(t, err) + + t.Run("certify accepts valid cert", func(t *testing.T) { + pubKey, addr := spawnTestCertifier(t, cfg, nil, verifying.WithLabelScryptParams(opts.Scrypt)) + + client := activation.NewCertifierClient(db, localDb, zaptest.NewLogger(t)) + _, err := client. + Certify(context.Background(), sig.NodeID(), &url.URL{Scheme: "http", Host: addr.String()}, pubKey) + require.NoError(t, err) + }) + t.Run("certify rejects invalid cert (expired)", func(t *testing.T) { + makeCert := func(nodeID []byte) *poetShared.Cert { + expired := time.Now().Add(-time.Hour) + return &poetShared.Cert{ + Pubkey: nodeID, + Expiration: &expired, + } + } + pubKey, addr := spawnTestCertifier(t, cfg, makeCert, verifying.WithLabelScryptParams(opts.Scrypt)) + + client := activation.NewCertifierClient(db, localDb, zaptest.NewLogger(t)) + cert, err := client. + Certify(context.Background(), sig.NodeID(), &url.URL{Scheme: "http", Host: addr.String()}, pubKey) + require.Error(t, err) + require.Nil(t, cert) + }) + t.Run("certify rejects invalid cert (wrong ID)", func(t *testing.T) { + makeCert := func(_ []byte) *poetShared.Cert { + return &poetShared.Cert{Pubkey: []byte("wrong")} + } + pubKey, addr := spawnTestCertifier(t, cfg, makeCert, verifying.WithLabelScryptParams(opts.Scrypt)) + + client := activation.NewCertifierClient(db, localDb, zaptest.NewLogger(t)) + cert, err := client. + Certify(context.Background(), sig.NodeID(), &url.URL{Scheme: "http", Host: addr.String()}, pubKey) + require.Error(t, err) + require.Nil(t, cert) + }) +} + +// A testCertifier for use in tests. +// Will verify any certificate valid POST proofs. +type testCertifier struct { + privKey ed25519.PrivateKey + postVerifier activation.PostVerifier + opts []verifying.OptionFunc + cfg activation.PostConfig + + makeCert func(nodeID []byte) *poetShared.Cert +} + +func (c *testCertifier) certify(w http.ResponseWriter, r *http.Request) { + var req activation.CertifyRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, fmt.Sprintf("decoding request: %v", err), http.StatusBadRequest) + return + } + + // Verify the POST proof. + proof := &shared.Proof{ + Nonce: req.Proof.Nonce, + Indices: req.Proof.Indices, + Pow: req.Proof.Pow, + } + metadata := &shared.ProofMetadata{ + NodeId: req.Metadata.NodeId, + CommitmentAtxId: req.Metadata.CommitmentAtxId, + Challenge: req.Metadata.Challenge, + NumUnits: req.Metadata.NumUnits, + LabelsPerUnit: c.cfg.LabelsPerUnit, + } + if err := c.postVerifier.Verify(context.Background(), proof, metadata, c.opts...); err != nil { + http.Error(w, fmt.Sprintf("verifying POST: %v", err), http.StatusBadRequest) + return + } + + var cert *poetShared.Cert + if c.makeCert != nil { + cert = c.makeCert(req.Metadata.NodeId) + } else { + cert = &poetShared.Cert{Pubkey: req.Metadata.NodeId} + } + certData, err := poetShared.EncodeCert(cert) + if err != nil { + panic(fmt.Sprintf("encoding cert: %v", err)) + } + + resp := activation.CertifyResponse{ + Certificate: certData, + Signature: ed25519.Sign(c.privKey, certData), + PubKey: c.privKey.Public().(ed25519.PublicKey), + } + if err := json.NewEncoder(w).Encode(resp); err != nil { + http.Error(w, fmt.Sprintf("encoding response: %v", err), http.StatusInternalServerError) + return + } +} + +func spawnTestCertifier( + t *testing.T, + cfg activation.PostConfig, + // optional - if nil, will create valid certs + makeCert func(nodeID []byte) *poetShared.Cert, + opts ...verifying.OptionFunc, +) (ed25519.PublicKey, net.Addr) { + t.Helper() + + pub, private, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + + postVerifier, err := activation.NewPostVerifier( + cfg, + zaptest.NewLogger(t), + ) + require.NoError(t, err) + var eg errgroup.Group + l, err := net.Listen("tcp", ":0") + require.NoError(t, err) + eg.Go(func() error { + certifier := &testCertifier{ + privKey: private, + postVerifier: postVerifier, + opts: opts, + cfg: cfg, + makeCert: makeCert, + } + + mux := http.NewServeMux() + mux.HandleFunc("/certify", certifier.certify) + http.Serve(l, mux) + return nil + }) + t.Cleanup(func() { + require.NoError(t, l.Close()) + require.NoError(t, eg.Wait()) + }) + + return pub, l.Addr() +} diff --git a/activation/e2e/nipost_test.go b/activation/e2e/nipost_test.go index 78f7adf809..2522daf673 100644 --- a/activation/e2e/nipost_test.go +++ b/activation/e2e/nipost_test.go @@ -9,7 +9,10 @@ import ( "time" "github.com/spacemeshos/poet/logging" + "github.com/spacemeshos/poet/registration" "github.com/spacemeshos/post/initialization" + "github.com/spacemeshos/post/shared" + "github.com/spacemeshos/post/verifying" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -26,6 +29,7 @@ import ( "github.com/spacemeshos/go-spacemesh/signing" "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sql/localsql" + "github.com/spacemeshos/go-spacemesh/sql/localsql/nipost" ) const ( @@ -40,6 +44,18 @@ func TestMain(m *testing.M) { os.Exit(res) } +func fullPost(post *types.Post, info *types.PostInfo, challenge []byte) *nipost.Post { + return &nipost.Post{ + Nonce: post.Nonce, + Indices: post.Indices, + Pow: post.Pow, + Challenge: challenge, + NumUnits: info.NumUnits, + CommitmentATX: info.CommitmentATX, + VRFNonce: *info.Nonce, + } +} + func spawnPoet(tb testing.TB, opts ...HTTPPoetOpt) *HTTPPoetTestHarness { tb.Helper() ctx, cancel := context.WithCancel(logging.NewContext(context.Background(), zaptest.NewLogger(tb))) @@ -121,6 +137,7 @@ func TestNIPostBuilderWithClients(t *testing.T) { cfg := activation.DefaultPostConfig() db := sql.InMemory() cdb := datastore.NewCachedDB(db, log.NewFromLog(logger)) + localDb := localsql.InMemory() syncer := activation.NewMocksyncer(ctrl) syncer.EXPECT().RegisterForATXSynced().AnyTimes().DoAndReturn(func() <-chan struct{} { @@ -150,12 +167,20 @@ func TestNIPostBuilderWithClients(t *testing.T) { RequestRetryDelay: epoch / 50, MaxRequestRetries: 10, } + + pubKey, addr := spawnTestCertifier(t, cfg, nil, verifying.WithLabelScryptParams(opts.Scrypt)) + certifierCfg := ®istration.CertifierConfig{ + URL: "http://" + addr.String(), + PubKey: registration.Base64Enc(pubKey), + } + poetProver := spawnPoet( t, WithGenesis(genesis), WithEpochDuration(epoch), WithPhaseShift(poetCfg.PhaseShift), WithCycleGap(poetCfg.CycleGap), + WithCertifier(certifierCfg), ) mclock := activation.NewMocklayerClock(ctrl) @@ -178,168 +203,40 @@ func TestNIPostBuilderWithClients(t *testing.T) { t.Cleanup(launchPostSupervisor(t, logger, mgr, sig, grpcCfg, opts)) + var postClient activation.PostClient require.Eventually(t, func() bool { - _, err := svc.Client(sig.NodeID()) + var err error + postClient, err = svc.Client(sig.NodeID()) return err == nil }, 10*time.Second, 100*time.Millisecond, "timed out waiting for connection") - localDB := localsql.InMemory() - nb, err := activation.NewNIPostBuilder( - localDB, - poetDb, - svc, - []types.PoetServer{{Pubkey: types.NewBase64Enc([]byte("foobar")), Address: poetProver.RestURL().String()}}, - logger.Named("nipostBuilder"), - poetCfg, - mclock, - ) - require.NoError(t, err) - - challenge := types.RandomHash() - nipost, err := nb.BuildNIPost(context.Background(), sig, 7, challenge) + post, info, err := postClient.Proof(context.Background(), shared.ZeroChallenge) require.NoError(t, err) - - v := activation.NewValidator(nil, poetDb, cfg, opts.Scrypt, verifier) - _, err = v.NIPost( - context.Background(), - sig.NodeID(), - goldenATX, - nipost.NIPost, - challenge, - nipost.NumUnits, - ) - require.NoError(t, err) -} - -func TestNIPostBuilder_Close(t *testing.T) { - ctrl := gomock.NewController(t) - - sig, err := signing.NewEdSigner() + err = nipost.AddPost(localDb, sig.NodeID(), *fullPost(post, info, shared.ZeroChallenge)) require.NoError(t, err) - logger := zaptest.NewLogger(t) - - poetProver := spawnPoet(t, WithGenesis(time.Now()), WithEpochDuration(time.Second)) - poetDb := activation.NewMockpoetDbAPI(ctrl) - - mclock := activation.NewMocklayerClock(ctrl) - mclock.EXPECT().LayerToTime(gomock.Any()).AnyTimes().DoAndReturn( - func(got types.LayerID) time.Time { - // time.Now() ~= currentLayer - genesis := time.Now().Add(-time.Duration(postGenesisEpoch.FirstLayer()) * layerDuration) - return genesis.Add(layerDuration * time.Duration(got)) - }, - ) - - svc := grpcserver.NewPostService(logger) - svc.AllowConnections(true) - - db := localsql.InMemory() - nb, err := activation.NewNIPostBuilder( - db, + client, err := activation.NewPoetClient( poetDb, - svc, - []types.PoetServer{{Pubkey: types.NewBase64Enc([]byte("foobar")), Address: poetProver.RestURL().String()}}, - logger.Named("nipostBuilder"), - activation.PoetConfig{}, - mclock, + poetProver.ServerCfg(), + poetCfg, + logger, ) require.NoError(t, err) - challenge := types.RandomHash() - ctx, cancel := context.WithCancel(context.Background()) - cancel() - nipost, err := nb.BuildNIPost(ctx, sig, 7, challenge) - require.ErrorIs(t, err, context.Canceled) - require.Nil(t, nipost) -} - -func TestNewNIPostBuilderNotInitialized(t *testing.T) { - ctrl := gomock.NewController(t) - - sig, err := signing.NewEdSigner() - require.NoError(t, err) - - logger := zaptest.NewLogger(t) - goldenATX := types.ATXID{2, 3, 4} - cfg := activation.DefaultPostConfig() - db := sql.InMemory() - - syncer := activation.NewMocksyncer(ctrl) - syncer.EXPECT().RegisterForATXSynced().AnyTimes().DoAndReturn(func() <-chan struct{} { - synced := make(chan struct{}) - close(synced) - return synced - }) - - validator := activation.NewMocknipostValidator(ctrl) - mgr, err := activation.NewPostSetupManager(cfg, logger, db, atxsdata.New(), goldenATX, syncer, validator) - require.NoError(t, err) - - // ensure that genesis aligns with layer timings - genesis := time.Now().Add(layerDuration).Round(layerDuration) - epoch := layersPerEpoch * layerDuration - poetCfg := activation.PoetConfig{ - PhaseShift: epoch / 2, - CycleGap: epoch / 4, - GracePeriod: epoch / 5, - RequestTimeout: epoch / 5, - RequestRetryDelay: epoch / 50, - MaxRequestRetries: 10, - } - poetProver := spawnPoet( - t, - WithGenesis(genesis), - WithEpochDuration(epoch), - WithPhaseShift(poetCfg.PhaseShift), - WithCycleGap(poetCfg.CycleGap), - ) - - mclock := activation.NewMocklayerClock(ctrl) - mclock.EXPECT().LayerToTime(gomock.Any()).AnyTimes().DoAndReturn( - func(got types.LayerID) time.Time { - return genesis.Add(layerDuration * time.Duration(got)) - }, - ) - - poetDb := activation.NewPoetDb(db, log.NewFromLog(logger).Named("poetDb")) - - svc := grpcserver.NewPostService(logger) - svc.AllowConnections(true) - grpcCfg, cleanup := launchServer(t, svc) - t.Cleanup(cleanup) - localDB := localsql.InMemory() nb, err := activation.NewNIPostBuilder( localDB, - poetDb, svc, - []types.PoetServer{{Pubkey: types.NewBase64Enc([]byte("foobar")), Address: poetProver.RestURL().String()}}, logger.Named("nipostBuilder"), poetCfg, mclock, + activation.WithPoetClients(client), ) require.NoError(t, err) - opts := activation.DefaultPostSetupOpts() - opts.DataDir = t.TempDir() - opts.ProviderID.SetUint32(initialization.CPUProviderID()) - opts.Scrypt.N = 2 // Speedup initialization in tests. - t.Cleanup(launchPostSupervisor(t, logger, mgr, sig, grpcCfg, opts)) - - require.Eventually(t, func() bool { - _, err := svc.Client(sig.NodeID()) - return err == nil - }, 10*time.Second, 100*time.Millisecond, "timed out waiting for connection") - challenge := types.RandomHash() nipost, err := nb.BuildNIPost(context.Background(), sig, 7, challenge) require.NoError(t, err) - require.NotNil(t, nipost) - - verifier, err := activation.NewPostVerifier(cfg, logger.Named("verifier")) - require.NoError(t, err) - t.Cleanup(func() { assert.NoError(t, verifier.Close()) }) v := activation.NewValidator(nil, poetDb, cfg, opts.Scrypt, verifier) _, err = v.NIPost( @@ -425,6 +322,15 @@ func Test_NIPostBuilderWithMultipleClients(t *testing.T) { WithCycleGap(poetCfg.CycleGap), ) + poetDb := activation.NewPoetDb(db, log.NewFromLog(logger).Named("poetDb")) + client, err := activation.NewPoetClient( + poetDb, + poetProver.ServerCfg(), + poetCfg, + logger, + ) + require.NoError(t, err) + mclock := activation.NewMocklayerClock(ctrl) mclock.EXPECT().LayerToTime(gomock.Any()).AnyTimes().DoAndReturn( func(got types.LayerID) time.Time { @@ -436,23 +342,25 @@ func Test_NIPostBuilderWithMultipleClients(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { assert.NoError(t, verifier.Close()) }) - poetDb := activation.NewPoetDb(db, log.NewFromLog(logger).Named("poetDb")) - localDB := localsql.InMemory() nb, err := activation.NewNIPostBuilder( localDB, - poetDb, svc, - []types.PoetServer{{Pubkey: types.NewBase64Enc([]byte("foobar")), Address: poetProver.RestURL().String()}}, logger.Named("nipostBuilder"), poetCfg, mclock, + activation.WithPoetClients(client), ) require.NoError(t, err) challenge := types.RandomHash() for _, sig := range signers { eg.Go(func() error { + post, info, err := nb.Proof(context.Background(), sig.NodeID(), shared.ZeroChallenge) + require.NoError(t, err) + err = nipost.AddPost(localDB, sig.NodeID(), *fullPost(post, info, shared.ZeroChallenge)) + require.NoError(t, err) + nipost, err := nb.BuildNIPost(context.Background(), sig, 7, challenge) require.NoError(t, err) diff --git a/activation/e2e/poet_test.go b/activation/e2e/poet_test.go index abfd9b7361..6d873b3363 100644 --- a/activation/e2e/poet_test.go +++ b/activation/e2e/poet_test.go @@ -3,20 +3,24 @@ package activation_test import ( "bytes" "context" + "crypto/ed25519" "errors" "net/url" "testing" "time" + "github.com/spacemeshos/poet/registration" "github.com/spacemeshos/poet/server" "github.com/spacemeshos/poet/shared" "github.com/stretchr/testify/require" + "go.uber.org/zap" "go.uber.org/zap/zaptest" "golang.org/x/sync/errgroup" "github.com/spacemeshos/go-spacemesh/activation" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/signing" + "github.com/spacemeshos/go-spacemesh/sql/localsql/certifier" ) // HTTPPoetTestHarness utilizes a local self-contained poet server instance @@ -32,6 +36,25 @@ func (h *HTTPPoetTestHarness) RestURL() *url.URL { } } +func (h *HTTPPoetTestHarness) Client( + db *activation.PoetDb, + cfg activation.PoetConfig, + logger *zap.Logger, + opts ...activation.PoetClientOpt, +) (activation.PoetClient, error) { + return activation.NewPoetClient( + db, + h.ServerCfg(), + cfg, + logger, + opts..., + ) +} + +func (h *HTTPPoetTestHarness) ServerCfg() types.PoetServer { + return types.PoetServer{Pubkey: types.NewBase64Enc(h.Service.PublicKey()), Address: h.RestURL().String()} +} + type HTTPPoetOpt func(*server.Config) func WithGenesis(genesis time.Time) HTTPPoetOpt { @@ -58,6 +81,12 @@ func WithCycleGap(gap time.Duration) HTTPPoetOpt { } } +func WithCertifier(certifier *registration.CertifierConfig) HTTPPoetOpt { + return func(cfg *server.Config) { + cfg.Registration.Certifier = certifier + } +} + // NewHTTPPoetTestHarness returns a new instance of HTTPPoetHarness. func NewHTTPPoetTestHarness(ctx context.Context, poetdir string, opts ...HTTPPoetOpt) (*HTTPPoetTestHarness, error) { cfg := server.DefaultConfig() @@ -91,7 +120,13 @@ func TestHTTPPoet(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - c, err := NewHTTPPoetTestHarness(ctx, poetDir) + + certPubKey, certPrivKey, err := ed25519.GenerateKey(nil) + r.NoError(err) + + c, err := NewHTTPPoetTestHarness(ctx, poetDir, WithCertifier(®istration.CertifierConfig{ + PubKey: registration.Base64Enc(certPubKey), + })) r.NoError(err) r.NotNil(c) @@ -107,39 +142,47 @@ func TestHTTPPoet(t *testing.T) { ) require.NoError(t, err) - resp, err := client.PowParams(context.Background()) - r.NoError(err) - signer, err := signing.NewEdSigner(signing.WithPrefix([]byte("prefix"))) require.NoError(t, err) ch := types.RandomHash() - nonce, err := shared.FindSubmitPowNonce( - context.Background(), - resp.Challenge, - ch.Bytes(), - signer.NodeID().Bytes(), - uint(resp.Difficulty), - ) - r.NoError(err) - signature := signer.Sign(signing.POET, ch.Bytes()) prefix := bytes.Join([][]byte{signer.Prefix(), {byte(signing.POET)}}, nil) - poetRound, err := client.Submit( - context.Background(), - time.Time{}, - prefix, - ch.Bytes(), - signature, - signer.NodeID(), - activation.PoetPoW{ - Nonce: nonce, - Params: *resp, - }, - ) - r.NoError(err) - r.NotNil(poetRound) + t.Run("submit with cert", func(t *testing.T) { + cert := shared.Cert{Pubkey: signer.NodeID().Bytes()} + encoded, err := shared.EncodeCert(&cert) + require.NoError(t, err) + + poetRound, err := client.Submit( + context.Background(), + time.Time{}, + prefix, + ch.Bytes(), + signature, + signer.NodeID(), + activation.PoetAuth{ + PoetCert: &certifier.PoetCert{ + Data: encoded, + Signature: ed25519.Sign(certPrivKey, encoded), + }, + }, + ) + require.NoError(t, err) + require.NotNil(t, poetRound) + }) + t.Run("return proper error code on rejected cert", func(t *testing.T) { + _, err := client.Submit( + context.Background(), + time.Time{}, + prefix, + ch.Bytes(), + signature, + signer.NodeID(), + activation.PoetAuth{PoetCert: &certifier.PoetCert{Data: []byte("oops")}}, + ) + require.ErrorIs(t, err, activation.ErrUnauthorized) + }) } func TestSubmitTooLate(t *testing.T) { @@ -168,22 +211,10 @@ func TestSubmitTooLate(t *testing.T) { ) require.NoError(t, err) - resp, err := client.PowParams(context.Background()) - r.NoError(err) - signer, err := signing.NewEdSigner(signing.WithPrefix([]byte("prefix"))) require.NoError(t, err) ch := types.RandomHash() - nonce, err := shared.FindSubmitPowNonce( - context.Background(), - resp.Challenge, - ch.Bytes(), - signer.NodeID().Bytes(), - uint(resp.Difficulty), - ) - r.NoError(err) - signature := signer.Sign(signing.POET, ch.Bytes()) prefix := bytes.Join([][]byte{signer.Prefix(), {byte(signing.POET)}}, nil) @@ -194,10 +225,72 @@ func TestSubmitTooLate(t *testing.T) { ch.Bytes(), signature, signer.NodeID(), - activation.PoetPoW{ - Nonce: nonce, - Params: *resp, - }, + activation.PoetAuth{}, ) r.ErrorIs(err, activation.ErrInvalidRequest) } + +func TestCertifierInfo(t *testing.T) { + t.Parallel() + r := require.New(t) + + var eg errgroup.Group + poetDir := t.TempDir() + t.Cleanup(func() { r.NoError(eg.Wait()) }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, err := NewHTTPPoetTestHarness(ctx, poetDir, WithCertifier(®istration.CertifierConfig{ + URL: "http://localhost:8080", + PubKey: []byte("pubkey"), + })) + r.NoError(err) + r.NotNil(c) + + eg.Go(func() error { + err := c.Service.Start(ctx) + return errors.Join(err, c.Service.Close()) + }) + + client, err := activation.NewHTTPPoetClient( + types.PoetServer{Address: c.RestURL().String()}, + activation.DefaultPoetConfig(), + activation.WithLogger(zaptest.NewLogger(t)), + ) + require.NoError(t, err) + + url, pubkey, err := client.CertifierInfo(context.Background()) + r.NoError(err) + r.Equal("http://localhost:8080", url.String()) + r.Equal([]byte("pubkey"), pubkey) +} + +func TestNoCertifierInfo(t *testing.T) { + t.Parallel() + r := require.New(t) + + var eg errgroup.Group + poetDir := t.TempDir() + t.Cleanup(func() { r.NoError(eg.Wait()) }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, err := NewHTTPPoetTestHarness(ctx, poetDir) + r.NoError(err) + r.NotNil(c) + + eg.Go(func() error { + err := c.Service.Start(ctx) + return errors.Join(err, c.Service.Close()) + }) + + client, err := activation.NewHTTPPoetClient( + types.PoetServer{Address: c.RestURL().String()}, + activation.DefaultPoetConfig(), + activation.WithLogger(zaptest.NewLogger(t)), + ) + require.NoError(t, err) + + _, _, err = client.CertifierInfo(context.Background()) + r.ErrorContains(err, "poet doesn't support certificates") +} diff --git a/activation/e2e/validation_test.go b/activation/e2e/validation_test.go index 9552489949..94feca41a6 100644 --- a/activation/e2e/validation_test.go +++ b/activation/e2e/validation_test.go @@ -68,6 +68,14 @@ func TestValidator_Validate(t *testing.T) { WithPhaseShift(poetCfg.PhaseShift), WithCycleGap(poetCfg.CycleGap), ) + poetDb := activation.NewPoetDb(sql.InMemory(), log.NewFromLog(logger).Named("poetDb")) + client, err := activation.NewPoetClient( + poetDb, + types.PoetServer{Address: poetProver.RestURL().String()}, + poetCfg, + logger, + ) + require.NoError(t, err) mclock := activation.NewMocklayerClock(ctrl) mclock.EXPECT().LayerToTime(gomock.Any()).AnyTimes().DoAndReturn( @@ -80,8 +88,6 @@ func TestValidator_Validate(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { assert.NoError(t, verifier.Close()) }) - poetDb := activation.NewPoetDb(sql.InMemory(), log.NewFromLog(logger).Named("poetDb")) - svc := grpcserver.NewPostService(logger) svc.AllowConnections(true) grpcCfg, cleanup := launchServer(t, svc) @@ -97,12 +103,11 @@ func TestValidator_Validate(t *testing.T) { challenge := types.RandomHash() nb, err := activation.NewNIPostBuilder( localsql.InMemory(), - poetDb, svc, - []types.PoetServer{{Address: poetProver.RestURL().String()}}, logger.Named("nipostBuilder"), poetCfg, mclock, + activation.WithPoetClients(client), ) require.NoError(t, err) diff --git a/activation/interface.go b/activation/interface.go index 71f82c1956..eb451e1c0d 100644 --- a/activation/interface.go +++ b/activation/interface.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io" + "net/url" "time" "github.com/spacemeshos/post/shared" @@ -12,6 +13,7 @@ import ( "github.com/spacemeshos/go-spacemesh/activation/wire" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/signing" + "github.com/spacemeshos/go-spacemesh/sql/localsql/certifier" "github.com/spacemeshos/go-spacemesh/sql/localsql/nipost" ) @@ -117,9 +119,9 @@ type SmeshingProvider interface { SetCoinbase(coinbase types.Address) } -// poetClient servers as an interface to communicate with a PoET server. +// PoetClient servers as an interface to communicate with a PoET server. // It is used to submit challenges and fetch proofs. -type poetClient interface { +type PoetClient interface { Address() string // Submit registers a challenge in the proving service current open round. @@ -131,10 +133,45 @@ type poetClient interface { nodeID types.NodeID, ) (*types.PoetRound, error) + Certify(ctx context.Context, id types.NodeID) (*certifier.PoetCert, error) + // Proof returns the proof for the given round ID. Proof(ctx context.Context, roundID string) (*types.PoetProof, []types.Hash32, error) } +// A certifier client that the certifierService uses to obtain certificates +// The implementation can use any method to obtain the certificate, +// for example, POST verification. +type certifierClient interface { + // Certify obtains a certificate in a remote certifier service. + Certify( + ctx context.Context, + id types.NodeID, + certifierAddress *url.URL, + pubkey []byte, + ) (*certifier.PoetCert, error) +} + +// certifierService is used to certify nodeID for registering in the poet. +// It holds the certificates and can recertify if needed. +type certifierService interface { + // Acquire a certificate for the ID in the given certifier. + // The certificate confirms that the ID is verified and it can be later used to submit in poet. + Certificate( + ctx context.Context, + id types.NodeID, + certifierAddress *url.URL, + pubkey []byte, + ) (*certifier.PoetCert, error) + + Recertify( + ctx context.Context, + id types.NodeID, + certifierAddress *url.URL, + pubkey []byte, + ) (*certifier.PoetCert, error) +} + type poetDbAPI interface { Proof(types.PoetProofRef) (*types.PoetProof, *types.Hash32, error) ProofForRound(poetID []byte, roundID string) (*types.PoetProof, error) diff --git a/activation/mocks.go b/activation/mocks.go index bb1759dbc7..e74c3cc7f4 100644 --- a/activation/mocks.go +++ b/activation/mocks.go @@ -11,12 +11,14 @@ package activation import ( context "context" + url "net/url" reflect "reflect" time "time" wire "github.com/spacemeshos/go-spacemesh/activation/wire" types "github.com/spacemeshos/go-spacemesh/common/types" signing "github.com/spacemeshos/go-spacemesh/signing" + certifier "github.com/spacemeshos/go-spacemesh/sql/localsql/certifier" nipost "github.com/spacemeshos/go-spacemesh/sql/localsql/nipost" shared "github.com/spacemeshos/post/shared" verifying "github.com/spacemeshos/post/verifying" @@ -1450,31 +1452,31 @@ func (c *MockSmeshingProviderStopSmeshingCall) DoAndReturn(f func(bool) error) * return c } -// MockpoetClient is a mock of poetClient interface. -type MockpoetClient struct { +// MockPoetClient is a mock of PoetClient interface. +type MockPoetClient struct { ctrl *gomock.Controller - recorder *MockpoetClientMockRecorder + recorder *MockPoetClientMockRecorder } -// MockpoetClientMockRecorder is the mock recorder for MockpoetClient. -type MockpoetClientMockRecorder struct { - mock *MockpoetClient +// MockPoetClientMockRecorder is the mock recorder for MockPoetClient. +type MockPoetClientMockRecorder struct { + mock *MockPoetClient } -// NewMockpoetClient creates a new mock instance. -func NewMockpoetClient(ctrl *gomock.Controller) *MockpoetClient { - mock := &MockpoetClient{ctrl: ctrl} - mock.recorder = &MockpoetClientMockRecorder{mock} +// NewMockPoetClient creates a new mock instance. +func NewMockPoetClient(ctrl *gomock.Controller) *MockPoetClient { + mock := &MockPoetClient{ctrl: ctrl} + mock.recorder = &MockPoetClientMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockpoetClient) EXPECT() *MockpoetClientMockRecorder { +func (m *MockPoetClient) EXPECT() *MockPoetClientMockRecorder { return m.recorder } // Address mocks base method. -func (m *MockpoetClient) Address() string { +func (m *MockPoetClient) Address() string { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Address") ret0, _ := ret[0].(string) @@ -1482,37 +1484,76 @@ func (m *MockpoetClient) Address() string { } // Address indicates an expected call of Address. -func (mr *MockpoetClientMockRecorder) Address() *MockpoetClientAddressCall { +func (mr *MockPoetClientMockRecorder) Address() *MockPoetClientAddressCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Address", reflect.TypeOf((*MockpoetClient)(nil).Address)) - return &MockpoetClientAddressCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Address", reflect.TypeOf((*MockPoetClient)(nil).Address)) + return &MockPoetClientAddressCall{Call: call} } -// MockpoetClientAddressCall wrap *gomock.Call -type MockpoetClientAddressCall struct { +// MockPoetClientAddressCall wrap *gomock.Call +type MockPoetClientAddressCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *MockpoetClientAddressCall) Return(arg0 string) *MockpoetClientAddressCall { +func (c *MockPoetClientAddressCall) Return(arg0 string) *MockPoetClientAddressCall { c.Call = c.Call.Return(arg0) return c } // Do rewrite *gomock.Call.Do -func (c *MockpoetClientAddressCall) Do(f func() string) *MockpoetClientAddressCall { +func (c *MockPoetClientAddressCall) Do(f func() string) *MockPoetClientAddressCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockpoetClientAddressCall) DoAndReturn(f func() string) *MockpoetClientAddressCall { +func (c *MockPoetClientAddressCall) DoAndReturn(f func() string) *MockPoetClientAddressCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Certify mocks base method. +func (m *MockPoetClient) Certify(ctx context.Context, id types.NodeID) (*certifier.PoetCert, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Certify", ctx, id) + ret0, _ := ret[0].(*certifier.PoetCert) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Certify indicates an expected call of Certify. +func (mr *MockPoetClientMockRecorder) Certify(ctx, id any) *MockPoetClientCertifyCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Certify", reflect.TypeOf((*MockPoetClient)(nil).Certify), ctx, id) + return &MockPoetClientCertifyCall{Call: call} +} + +// MockPoetClientCertifyCall wrap *gomock.Call +type MockPoetClientCertifyCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockPoetClientCertifyCall) Return(arg0 *certifier.PoetCert, arg1 error) *MockPoetClientCertifyCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockPoetClientCertifyCall) Do(f func(context.Context, types.NodeID) (*certifier.PoetCert, error)) *MockPoetClientCertifyCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockPoetClientCertifyCall) DoAndReturn(f func(context.Context, types.NodeID) (*certifier.PoetCert, error)) *MockPoetClientCertifyCall { c.Call = c.Call.DoAndReturn(f) return c } // Proof mocks base method. -func (m *MockpoetClient) Proof(ctx context.Context, roundID string) (*types.PoetProof, []types.Hash32, error) { +func (m *MockPoetClient) Proof(ctx context.Context, roundID string) (*types.PoetProof, []types.Hash32, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Proof", ctx, roundID) ret0, _ := ret[0].(*types.PoetProof) @@ -1522,37 +1563,37 @@ func (m *MockpoetClient) Proof(ctx context.Context, roundID string) (*types.Poet } // Proof indicates an expected call of Proof. -func (mr *MockpoetClientMockRecorder) Proof(ctx, roundID any) *MockpoetClientProofCall { +func (mr *MockPoetClientMockRecorder) Proof(ctx, roundID any) *MockPoetClientProofCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Proof", reflect.TypeOf((*MockpoetClient)(nil).Proof), ctx, roundID) - return &MockpoetClientProofCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Proof", reflect.TypeOf((*MockPoetClient)(nil).Proof), ctx, roundID) + return &MockPoetClientProofCall{Call: call} } -// MockpoetClientProofCall wrap *gomock.Call -type MockpoetClientProofCall struct { +// MockPoetClientProofCall wrap *gomock.Call +type MockPoetClientProofCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *MockpoetClientProofCall) Return(arg0 *types.PoetProof, arg1 []types.Hash32, arg2 error) *MockpoetClientProofCall { +func (c *MockPoetClientProofCall) Return(arg0 *types.PoetProof, arg1 []types.Hash32, arg2 error) *MockPoetClientProofCall { c.Call = c.Call.Return(arg0, arg1, arg2) return c } // Do rewrite *gomock.Call.Do -func (c *MockpoetClientProofCall) Do(f func(context.Context, string) (*types.PoetProof, []types.Hash32, error)) *MockpoetClientProofCall { +func (c *MockPoetClientProofCall) Do(f func(context.Context, string) (*types.PoetProof, []types.Hash32, error)) *MockPoetClientProofCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockpoetClientProofCall) DoAndReturn(f func(context.Context, string) (*types.PoetProof, []types.Hash32, error)) *MockpoetClientProofCall { +func (c *MockPoetClientProofCall) DoAndReturn(f func(context.Context, string) (*types.PoetProof, []types.Hash32, error)) *MockPoetClientProofCall { c.Call = c.Call.DoAndReturn(f) return c } // Submit mocks base method. -func (m *MockpoetClient) Submit(ctx context.Context, deadline time.Time, prefix, challenge []byte, signature types.EdSignature, nodeID types.NodeID) (*types.PoetRound, error) { +func (m *MockPoetClient) Submit(ctx context.Context, deadline time.Time, prefix, challenge []byte, signature types.EdSignature, nodeID types.NodeID) (*types.PoetRound, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Submit", ctx, deadline, prefix, challenge, signature, nodeID) ret0, _ := ret[0].(*types.PoetRound) @@ -1561,31 +1602,194 @@ func (m *MockpoetClient) Submit(ctx context.Context, deadline time.Time, prefix, } // Submit indicates an expected call of Submit. -func (mr *MockpoetClientMockRecorder) Submit(ctx, deadline, prefix, challenge, signature, nodeID any) *MockpoetClientSubmitCall { +func (mr *MockPoetClientMockRecorder) Submit(ctx, deadline, prefix, challenge, signature, nodeID any) *MockPoetClientSubmitCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Submit", reflect.TypeOf((*MockPoetClient)(nil).Submit), ctx, deadline, prefix, challenge, signature, nodeID) + return &MockPoetClientSubmitCall{Call: call} +} + +// MockPoetClientSubmitCall wrap *gomock.Call +type MockPoetClientSubmitCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockPoetClientSubmitCall) Return(arg0 *types.PoetRound, arg1 error) *MockPoetClientSubmitCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockPoetClientSubmitCall) Do(f func(context.Context, time.Time, []byte, []byte, types.EdSignature, types.NodeID) (*types.PoetRound, error)) *MockPoetClientSubmitCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockPoetClientSubmitCall) DoAndReturn(f func(context.Context, time.Time, []byte, []byte, types.EdSignature, types.NodeID) (*types.PoetRound, error)) *MockPoetClientSubmitCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// MockcertifierClient is a mock of certifierClient interface. +type MockcertifierClient struct { + ctrl *gomock.Controller + recorder *MockcertifierClientMockRecorder +} + +// MockcertifierClientMockRecorder is the mock recorder for MockcertifierClient. +type MockcertifierClientMockRecorder struct { + mock *MockcertifierClient +} + +// NewMockcertifierClient creates a new mock instance. +func NewMockcertifierClient(ctrl *gomock.Controller) *MockcertifierClient { + mock := &MockcertifierClient{ctrl: ctrl} + mock.recorder = &MockcertifierClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockcertifierClient) EXPECT() *MockcertifierClientMockRecorder { + return m.recorder +} + +// Certify mocks base method. +func (m *MockcertifierClient) Certify(ctx context.Context, id types.NodeID, certifierAddress *url.URL, pubkey []byte) (*certifier.PoetCert, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Certify", ctx, id, certifierAddress, pubkey) + ret0, _ := ret[0].(*certifier.PoetCert) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Certify indicates an expected call of Certify. +func (mr *MockcertifierClientMockRecorder) Certify(ctx, id, certifierAddress, pubkey any) *MockcertifierClientCertifyCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Certify", reflect.TypeOf((*MockcertifierClient)(nil).Certify), ctx, id, certifierAddress, pubkey) + return &MockcertifierClientCertifyCall{Call: call} +} + +// MockcertifierClientCertifyCall wrap *gomock.Call +type MockcertifierClientCertifyCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockcertifierClientCertifyCall) Return(arg0 *certifier.PoetCert, arg1 error) *MockcertifierClientCertifyCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockcertifierClientCertifyCall) Do(f func(context.Context, types.NodeID, *url.URL, []byte) (*certifier.PoetCert, error)) *MockcertifierClientCertifyCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockcertifierClientCertifyCall) DoAndReturn(f func(context.Context, types.NodeID, *url.URL, []byte) (*certifier.PoetCert, error)) *MockcertifierClientCertifyCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// MockcertifierService is a mock of certifierService interface. +type MockcertifierService struct { + ctrl *gomock.Controller + recorder *MockcertifierServiceMockRecorder +} + +// MockcertifierServiceMockRecorder is the mock recorder for MockcertifierService. +type MockcertifierServiceMockRecorder struct { + mock *MockcertifierService +} + +// NewMockcertifierService creates a new mock instance. +func NewMockcertifierService(ctrl *gomock.Controller) *MockcertifierService { + mock := &MockcertifierService{ctrl: ctrl} + mock.recorder = &MockcertifierServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockcertifierService) EXPECT() *MockcertifierServiceMockRecorder { + return m.recorder +} + +// Certificate mocks base method. +func (m *MockcertifierService) Certificate(ctx context.Context, id types.NodeID, certifierAddress *url.URL, pubkey []byte) (*certifier.PoetCert, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Certificate", ctx, id, certifierAddress, pubkey) + ret0, _ := ret[0].(*certifier.PoetCert) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Certificate indicates an expected call of Certificate. +func (mr *MockcertifierServiceMockRecorder) Certificate(ctx, id, certifierAddress, pubkey any) *MockcertifierServiceCertificateCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Certificate", reflect.TypeOf((*MockcertifierService)(nil).Certificate), ctx, id, certifierAddress, pubkey) + return &MockcertifierServiceCertificateCall{Call: call} +} + +// MockcertifierServiceCertificateCall wrap *gomock.Call +type MockcertifierServiceCertificateCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockcertifierServiceCertificateCall) Return(arg0 *certifier.PoetCert, arg1 error) *MockcertifierServiceCertificateCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockcertifierServiceCertificateCall) Do(f func(context.Context, types.NodeID, *url.URL, []byte) (*certifier.PoetCert, error)) *MockcertifierServiceCertificateCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockcertifierServiceCertificateCall) DoAndReturn(f func(context.Context, types.NodeID, *url.URL, []byte) (*certifier.PoetCert, error)) *MockcertifierServiceCertificateCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Recertify mocks base method. +func (m *MockcertifierService) Recertify(ctx context.Context, id types.NodeID, certifierAddress *url.URL, pubkey []byte) (*certifier.PoetCert, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Recertify", ctx, id, certifierAddress, pubkey) + ret0, _ := ret[0].(*certifier.PoetCert) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Recertify indicates an expected call of Recertify. +func (mr *MockcertifierServiceMockRecorder) Recertify(ctx, id, certifierAddress, pubkey any) *MockcertifierServiceRecertifyCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Submit", reflect.TypeOf((*MockpoetClient)(nil).Submit), ctx, deadline, prefix, challenge, signature, nodeID) - return &MockpoetClientSubmitCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Recertify", reflect.TypeOf((*MockcertifierService)(nil).Recertify), ctx, id, certifierAddress, pubkey) + return &MockcertifierServiceRecertifyCall{Call: call} } -// MockpoetClientSubmitCall wrap *gomock.Call -type MockpoetClientSubmitCall struct { +// MockcertifierServiceRecertifyCall wrap *gomock.Call +type MockcertifierServiceRecertifyCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *MockpoetClientSubmitCall) Return(arg0 *types.PoetRound, arg1 error) *MockpoetClientSubmitCall { +func (c *MockcertifierServiceRecertifyCall) Return(arg0 *certifier.PoetCert, arg1 error) *MockcertifierServiceRecertifyCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockpoetClientSubmitCall) Do(f func(context.Context, time.Time, []byte, []byte, types.EdSignature, types.NodeID) (*types.PoetRound, error)) *MockpoetClientSubmitCall { +func (c *MockcertifierServiceRecertifyCall) Do(f func(context.Context, types.NodeID, *url.URL, []byte) (*certifier.PoetCert, error)) *MockcertifierServiceRecertifyCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockpoetClientSubmitCall) DoAndReturn(f func(context.Context, time.Time, []byte, []byte, types.EdSignature, types.NodeID) (*types.PoetRound, error)) *MockpoetClientSubmitCall { +func (c *MockcertifierServiceRecertifyCall) DoAndReturn(f func(context.Context, types.NodeID, *url.URL, []byte) (*certifier.PoetCert, error)) *MockcertifierServiceRecertifyCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/activation/nipost.go b/activation/nipost.go index b2e89f38fe..7fde9dcdc7 100644 --- a/activation/nipost.go +++ b/activation/nipost.go @@ -46,7 +46,7 @@ const ( type NIPostBuilder struct { localDB *localsql.Database - poetProvers map[string]poetClient + poetProvers map[string]PoetClient postService postService log *zap.Logger poetCfg PoetConfig @@ -56,10 +56,9 @@ type NIPostBuilder struct { type NIPostBuilderOption func(*NIPostBuilder) -// withPoetClients allows to pass in clients directly (for testing purposes). -func withPoetClients(clients []poetClient) NIPostBuilderOption { +func WithPoetClients(clients ...PoetClient) NIPostBuilderOption { return func(nb *NIPostBuilder) { - nb.poetProvers = make(map[string]poetClient, len(clients)) + nb.poetProvers = make(map[string]PoetClient, len(clients)) for _, client := range clients { nb.poetProvers[client.Address()] = client } @@ -75,27 +74,15 @@ func NipostbuilderWithPostStates(ps PostStates) NIPostBuilderOption { // NewNIPostBuilder returns a NIPostBuilder. func NewNIPostBuilder( db *localsql.Database, - poetDB poetDbAPI, postService postService, - poetServers []types.PoetServer, lg *zap.Logger, poetCfg PoetConfig, layerClock layerClock, opts ...NIPostBuilderOption, ) (*NIPostBuilder, error) { - poetClients := make(map[string]poetClient, len(poetServers)) - for _, server := range poetServers { - client, err := newPoetClient(poetDB, server, poetCfg, lg.Named("poet")) - if err != nil { - return nil, fmt.Errorf("cannot create poet client: %w", err) - } - poetClients[client.Address()] = client - } - b := &NIPostBuilder{ localDB: db, - poetProvers: poetClients, postService: postService, log: lg, poetCfg: poetCfg, @@ -232,7 +219,8 @@ func (nb *NIPostBuilder) BuildNIPost( submitCtx, cancel := context.WithDeadline(ctx, poetRoundStart) defer cancel() - if err := nb.submitPoetChallenges(submitCtx, signer, poetProofDeadline, challenge.Bytes()); err != nil { + err := nb.submitPoetChallenges(submitCtx, signer, poetProofDeadline, challenge.Bytes()) + if err != nil { return nil, fmt.Errorf("submitting to poets: %w", err) } count, err := nipost.PoetRegistrationCount(nb.localDB, signer.NodeID()) @@ -343,7 +331,7 @@ func (nb *NIPostBuilder) submitPoetChallenge( ctx context.Context, nodeID types.NodeID, deadline time.Time, - client poetClient, + client PoetClient, prefix, challenge []byte, signature types.EdSignature, ) error { @@ -353,11 +341,15 @@ func (nb *NIPostBuilder) submitPoetChallenge( log.ZShortStringer("smesherID", nodeID), ) - round, err := client.Submit(ctx, deadline, prefix, challenge, signature, nodeID) + logger.Debug("submitting challenge to poet proving service") + + submitCtx, cancel := withConditionalTimeout(ctx, nb.poetCfg.RequestTimeout) + defer cancel() + + round, err := client.Submit(submitCtx, deadline, prefix, challenge, signature, nodeID) if err != nil { return &PoetSvcUnstableError{msg: "failed to submit challenge to poet service", source: err} } - logger.Info("challenge submitted to poet proving service", zap.String("round", round.ID)) return nipost.AddPoetRegistration(nb.localDB, nodeID, nipost.PoETRegistration{ ChallengeHash: types.Hash32(challenge), @@ -407,7 +399,7 @@ func (nb *NIPostBuilder) submitPoetChallenges( return nil } -func (nb *NIPostBuilder) getPoetClient(ctx context.Context, address string) poetClient { +func (nb *NIPostBuilder) getPoetClient(ctx context.Context, address string) PoetClient { for _, client := range nb.poetProvers { if address == client.Address() { return client diff --git a/activation/nipost_test.go b/activation/nipost_test.go index f4ac51a347..f1f14116a8 100644 --- a/activation/nipost_test.go +++ b/activation/nipost_test.go @@ -24,8 +24,9 @@ import ( "github.com/spacemeshos/go-spacemesh/sql/localsql/nipost" ) -func defaultPoetServiceMock(ctrl *gomock.Controller, id []byte, address string) *MockpoetClient { - poetClient := NewMockpoetClient(ctrl) +func defaultPoetServiceMock(t *testing.T, ctrl *gomock.Controller, address string) *MockPoetClient { + t.Helper() + poetClient := NewMockPoetClient(ctrl) poetClient.EXPECT(). Submit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). AnyTimes(). @@ -38,7 +39,6 @@ func defaultLayerClockMock(ctrl *gomock.Controller) *MocklayerClock { mclock := NewMocklayerClock(ctrl) mclock.EXPECT().LayerToTime(gomock.Any()).AnyTimes().DoAndReturn( func(got types.LayerID) time.Time { - // time.Now() ~= currentLayer genesis := time.Now().Add(-time.Duration(postGenesisEpoch.FirstLayer()) * layerDuration) return genesis.Add(layerDuration * time.Duration(got)) }, @@ -88,9 +88,7 @@ func newTestNIPostBuilder(tb testing.TB) *testNIPostBuilder { nb, err := NewNIPostBuilder( tnb.mDb, - tnb.mPoetDb, tnb.mPostService, - []types.PoetServer{}, tnb.mLogger, PoetConfig{}, tnb.mClock, @@ -328,7 +326,6 @@ func Test_NIPostBuilder_ResetState(t *testing.T) { require.NoError(t, err) ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) postService := NewMockpostService(ctrl) mclock := defaultLayerClockMock(ctrl) @@ -336,9 +333,7 @@ func Test_NIPostBuilder_ResetState(t *testing.T) { nb, err := NewNIPostBuilder( db, - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, mclock, @@ -378,10 +373,9 @@ func Test_NIPostBuilder_WithMocks(t *testing.T) { challenge := types.RandomHash() ctrl := gomock.NewController(t) - poetProvider := defaultPoetServiceMock(ctrl, []byte("poet"), "http://localhost:9999") + poetProvider := defaultPoetServiceMock(t, ctrl, "http://localhost:9999") poetProvider.EXPECT().Proof(gomock.Any(), "").Return(&types.PoetProof{}, []types.Hash32{challenge}, nil) - poetDb := NewMockpoetDbAPI(ctrl) mclock := defaultLayerClockMock(ctrl) sig, err := signing.NewEdSigner() @@ -397,13 +391,11 @@ func Test_NIPostBuilder_WithMocks(t *testing.T) { nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, mclock, - withPoetClients([]poetClient{poetProvider}), + WithPoetClients(poetProvider), ) require.NoError(t, err) @@ -418,10 +410,9 @@ func TestPostSetup(t *testing.T) { require.NoError(t, err) ctrl := gomock.NewController(t) - poetProvider := defaultPoetServiceMock(ctrl, []byte("poet"), "http://localhost:9999") + poetProvider := defaultPoetServiceMock(t, ctrl, "http://localhost:9999") poetProvider.EXPECT().Proof(gomock.Any(), "").Return(&types.PoetProof{}, []types.Hash32{challenge}, nil) - poetDb := NewMockpoetDbAPI(ctrl) mclock := defaultLayerClockMock(ctrl) postClient := NewMockPostClient(ctrl) @@ -434,13 +425,11 @@ func TestPostSetup(t *testing.T) { nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, mclock, - withPoetClients([]poetClient{poetProvider}), + WithPoetClients(poetProvider), ) require.NoError(t, err) @@ -463,7 +452,7 @@ func TestNIPostBuilder_BuildNIPost(t *testing.T) { challengeHash := wire.NIPostChallengeToWireV1(&challenge).Hash() ctrl := gomock.NewController(t) - poetProver := defaultPoetServiceMock(ctrl, []byte("poet"), "http://localhost:9999") + poetProver := defaultPoetServiceMock(t, ctrl, "http://localhost:9999") poetProver.EXPECT().Proof(gomock.Any(), "").AnyTimes().Return( &types.PoetProof{}, []types.Hash32{ challengeHash, @@ -472,7 +461,6 @@ func TestNIPostBuilder_BuildNIPost(t *testing.T) { }, nil, ) - poetDb := NewMockpoetDbAPI(ctrl) mclock := defaultLayerClockMock(ctrl) postClient := NewMockPostClient(ctrl) @@ -488,13 +476,11 @@ func TestNIPostBuilder_BuildNIPost(t *testing.T) { nb, err := NewNIPostBuilder( db, - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, mclock, - withPoetClients([]poetClient{poetProver}), + WithPoetClients(poetProver), ) require.NoError(t, err) @@ -502,19 +488,15 @@ func TestNIPostBuilder_BuildNIPost(t *testing.T) { require.NoError(t, err) require.NotNil(t, nipost) - poetDb = NewMockpoetDbAPI(ctrl) - // fail post exec require.NoError(t, nb.ResetState(sig.NodeID())) nb, err = NewNIPostBuilder( db, - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, mclock, - withPoetClients([]poetClient{poetProver}), + WithPoetClients(poetProver), ) require.NoError(t, err) @@ -528,13 +510,11 @@ func TestNIPostBuilder_BuildNIPost(t *testing.T) { // successful post exec nb, err = NewNIPostBuilder( db, - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, mclock, - withPoetClients([]poetClient{poetProver}), + WithPoetClients(poetProver), ) require.NoError(t, err) postClient.EXPECT().Proof(gomock.Any(), gomock.Any()).Return( @@ -551,20 +531,6 @@ func TestNIPostBuilder_BuildNIPost(t *testing.T) { require.NotNil(t, nipost) } -func Test_NIPostBuilder_InvalidPoetAddresses(t *testing.T) { - nb, err := NewNIPostBuilder( - nil, - nil, - nil, - []types.PoetServer{{Address: ":invalid"}}, - zaptest.NewLogger(t).Named("nipostBuilder"), - PoetConfig{}, - nil, - ) - require.ErrorContains(t, err, "cannot create poet client") - require.Nil(t, nb) -} - func TestNIPostBuilder_ManyPoETs_SubmittingChallenge_DeadlineReached(t *testing.T) { t.Parallel() // Arrange @@ -572,12 +538,11 @@ func TestNIPostBuilder_ManyPoETs_SubmittingChallenge_DeadlineReached(t *testing. proof := &types.PoetProof{} ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) mclock := defaultLayerClockMock(ctrl) - poets := make([]poetClient, 0, 2) + poets := make([]PoetClient, 0, 2) { - poet := NewMockpoetClient(ctrl) + poet := NewMockPoetClient(ctrl) poet.EXPECT(). Submit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func( @@ -594,7 +559,7 @@ func TestNIPostBuilder_ManyPoETs_SubmittingChallenge_DeadlineReached(t *testing. poets = append(poets, poet) } { - poet := NewMockpoetClient(ctrl) + poet := NewMockPoetClient(ctrl) poet.EXPECT(). Submit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(&types.PoetRound{}, nil) @@ -622,13 +587,11 @@ func TestNIPostBuilder_ManyPoETs_SubmittingChallenge_DeadlineReached(t *testing. nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), poetCfg, mclock, - withPoetClients(poets), + WithPoetClients(poets...), ) require.NoError(t, err) @@ -650,17 +613,20 @@ func TestNIPostBuilder_ManyPoETs_AllFinished(t *testing.T) { proofBetter := &types.PoetProof{LeafCount: 999} ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) mclock := defaultLayerClockMock(ctrl) - poets := make([]poetClient, 0, 2) + poets := make([]PoetClient, 0, 2) { - poet := defaultPoetServiceMock(ctrl, []byte("poet0"), "http://localhost:9999") + poet := NewMockPoetClient(ctrl) + poet.EXPECT(). + Submit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(&types.PoetRound{}, nil) + poet.EXPECT().Address().AnyTimes().Return("http://localhost:9999") poet.EXPECT().Proof(gomock.Any(), "").Return(proofWorse, []types.Hash32{challenge}, nil) poets = append(poets, poet) } { - poet := defaultPoetServiceMock(ctrl, []byte("poet1"), "http://localhost:9998") + poet := defaultPoetServiceMock(t, ctrl, "http://localhost:9998") poet.EXPECT().Proof(gomock.Any(), "").Return(proofBetter, []types.Hash32{challenge}, nil) poets = append(poets, poet) } @@ -678,13 +644,11 @@ func TestNIPostBuilder_ManyPoETs_AllFinished(t *testing.T) { nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, mclock, - withPoetClients(poets), + WithPoetClients(poets...), ) require.NoError(t, err) @@ -703,31 +667,27 @@ func TestNIPSTBuilder_PoetUnstable(t *testing.T) { poetCfg := PoetConfig{ PhaseShift: layerDuration, } - sig, err := signing.NewEdSigner() require.NoError(t, err) t.Run("Submit fails", func(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) mclock := defaultLayerClockMock(ctrl) - poetProver := NewMockpoetClient(ctrl) + poetProver := NewMockPoetClient(ctrl) poetProver.EXPECT(). - Submit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Submit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), sig.NodeID()). Return(nil, errors.New("test")) poetProver.EXPECT().Address().AnyTimes().Return("http://localhost:9999") postService := NewMockpostService(ctrl) nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), poetCfg, mclock, - withPoetClients([]poetClient{poetProver}), + WithPoetClients(poetProver), ) require.NoError(t, err) @@ -738,11 +698,11 @@ func TestNIPSTBuilder_PoetUnstable(t *testing.T) { t.Run("Submit hangs", func(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) mclock := defaultLayerClockMock(ctrl) - poetProver := NewMockpoetClient(ctrl) + poetProver := NewMockPoetClient(ctrl) + poetProver.EXPECT(). - Submit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Submit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), sig.NodeID()). DoAndReturn(func( ctx context.Context, _ time.Time, @@ -758,15 +718,14 @@ func TestNIPSTBuilder_PoetUnstable(t *testing.T) { nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), poetCfg, mclock, - withPoetClients([]poetClient{poetProver}), + WithPoetClients(poetProver), ) require.NoError(t, err) + nipst, err := nb.BuildNIPost(context.Background(), sig, postGenesisEpoch+2, challenge) require.ErrorIs(t, err, ErrPoetServiceUnstable) require.Nil(t, nipst) @@ -774,23 +733,21 @@ func TestNIPSTBuilder_PoetUnstable(t *testing.T) { t.Run("GetProof fails", func(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) mclock := defaultLayerClockMock(ctrl) - poetProver := defaultPoetServiceMock(ctrl, []byte("poet"), "http://localhost:9999") + poetProver := defaultPoetServiceMock(t, ctrl, "http://localhost:9999") poetProver.EXPECT().Proof(gomock.Any(), "").Return(nil, nil, errors.New("failed")) postService := NewMockpostService(ctrl) nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), poetCfg, mclock, - withPoetClients([]poetClient{poetProver}), + WithPoetClients(poetProver), ) require.NoError(t, err) + nipst, err := nb.BuildNIPost(context.Background(), sig, postGenesisEpoch+2, challenge) require.ErrorIs(t, err, ErrPoetProofNotReceived) require.Nil(t, nipst) @@ -798,9 +755,8 @@ func TestNIPSTBuilder_PoetUnstable(t *testing.T) { t.Run("Challenge is not included in proof members", func(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) mclock := defaultLayerClockMock(ctrl) - poetProver := defaultPoetServiceMock(ctrl, []byte("poet"), "http://localhost:9999") + poetProver := defaultPoetServiceMock(t, ctrl, "http://localhost:9999") poetProver.EXPECT(). Proof(gomock.Any(), ""). Return(&types.PoetProof{}, []types.Hash32{}, nil) @@ -808,15 +764,14 @@ func TestNIPSTBuilder_PoetUnstable(t *testing.T) { nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), poetCfg, mclock, - withPoetClients([]poetClient{poetProver}), + WithPoetClients(poetProver), ) require.NoError(t, err) + nipst, err := nb.BuildNIPost(context.Background(), sig, postGenesisEpoch+2, challenge) require.ErrorIs(t, err, ErrPoetProofNotReceived) require.Nil(t, nipst) @@ -837,9 +792,8 @@ func TestNIPoSTBuilder_StaleChallenge(t *testing.T) { // Act & Verify t.Run("no requests, poet round started", func(t *testing.T) { ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) mclock := NewMocklayerClock(ctrl) - poetProver := NewMockpoetClient(ctrl) + poetProver := NewMockPoetClient(ctrl) poetProver.EXPECT().Address().Return("http://localhost:9999") mclock.EXPECT().LayerToTime(gomock.Any()).DoAndReturn( func(got types.LayerID) time.Time { @@ -850,13 +804,11 @@ func TestNIPoSTBuilder_StaleChallenge(t *testing.T) { nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, mclock, - withPoetClients([]poetClient{poetProver}), + WithPoetClients(poetProver), ) require.NoError(t, err) @@ -867,9 +819,8 @@ func TestNIPoSTBuilder_StaleChallenge(t *testing.T) { }) t.Run("no response before deadline", func(t *testing.T) { ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) mclock := NewMocklayerClock(ctrl) - poetProver := NewMockpoetClient(ctrl) + poetProver := NewMockPoetClient(ctrl) poetProver.EXPECT().Address().Return("http://localhost:9999") mclock.EXPECT().LayerToTime(gomock.Any()).DoAndReturn( func(got types.LayerID) time.Time { @@ -880,13 +831,11 @@ func TestNIPoSTBuilder_StaleChallenge(t *testing.T) { db := localsql.InMemory() nb, err := NewNIPostBuilder( db, - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, mclock, - withPoetClients([]poetClient{poetProver}), + WithPoetClients(poetProver), ) require.NoError(t, err) @@ -911,9 +860,8 @@ func TestNIPoSTBuilder_StaleChallenge(t *testing.T) { }) t.Run("too late for proof generation", func(t *testing.T) { ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) mclock := NewMocklayerClock(ctrl) - poetProver := NewMockpoetClient(ctrl) + poetProver := NewMockPoetClient(ctrl) poetProver.EXPECT().Address().Return("http://localhost:9999") mclock.EXPECT().LayerToTime(gomock.Any()).DoAndReturn( func(got types.LayerID) time.Time { @@ -924,13 +872,11 @@ func TestNIPoSTBuilder_StaleChallenge(t *testing.T) { db := localsql.InMemory() nb, err := NewNIPostBuilder( db, - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, mclock, - withPoetClients([]poetClient{poetProver}), + WithPoetClients(poetProver), ) require.NoError(t, err) @@ -966,15 +912,16 @@ func TestNIPoSTBuilder_Continues_After_Interrupted(t *testing.T) { // Arrange challenge := types.RandomHash() + sig, err := signing.NewEdSigner() + require.NoError(t, err) proof := &types.PoetProof{LeafCount: 777} ctrl := gomock.NewController(t) - poetDb := NewMockpoetDbAPI(ctrl) mclock := defaultLayerClockMock(ctrl) buildCtx, cancel := context.WithCancel(context.Background()) - poet := NewMockpoetClient(ctrl) + poet := NewMockPoetClient(ctrl) poet.EXPECT(). Submit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func( @@ -994,9 +941,6 @@ func TestNIPoSTBuilder_Continues_After_Interrupted(t *testing.T) { PhaseShift: layerDuration * layersPerEpoch / 2, } - sig, err := signing.NewEdSigner() - require.NoError(t, err) - postClient := NewMockPostClient(ctrl) nonce := types.VRFPostIndex(1) postClient.EXPECT().Proof(gomock.Any(), gomock.Any()).Return(&types.Post{}, &types.PostInfo{ @@ -1007,13 +951,11 @@ func TestNIPoSTBuilder_Continues_After_Interrupted(t *testing.T) { nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), poetCfg, mclock, - withPoetClients([]poetClient{poet}), + WithPoetClients(poet), ) require.NoError(t, err) @@ -1095,9 +1037,9 @@ func TestNIPostBuilder_Mainnet_Poet_Workaround(t *testing.T) { challenge := types.RandomHash() ctrl := gomock.NewController(t) - poets := make([]poetClient, 0, 2) + poets := make([]PoetClient, 0, 2) { - poetProvider := NewMockpoetClient(ctrl) + poetProvider := NewMockPoetClient(ctrl) poetProvider.EXPECT().Address().Return(tc.from) // PoET succeeds to submit @@ -1112,7 +1054,7 @@ func TestNIPostBuilder_Mainnet_Poet_Workaround(t *testing.T) { { // PoET fails submission - poetProvider := NewMockpoetClient(ctrl) + poetProvider := NewMockPoetClient(ctrl) poetProvider.EXPECT().Address().Return(tc.to) // proof is still fetched from PoET @@ -1121,7 +1063,6 @@ func TestNIPostBuilder_Mainnet_Poet_Workaround(t *testing.T) { poets = append(poets, poetProvider) } - poetDb := NewMockpoetDbAPI(ctrl) mclock := NewMocklayerClock(ctrl) genesis := time.Now().Add(-time.Duration(1) * layerDuration) mclock.EXPECT().LayerToTime(gomock.Any()).AnyTimes().DoAndReturn( @@ -1144,13 +1085,11 @@ func TestNIPostBuilder_Mainnet_Poet_Workaround(t *testing.T) { nb, err := NewNIPostBuilder( localsql.InMemory(), - poetDb, postService, - []types.PoetServer{}, zaptest.NewLogger(t), poetCfg, mclock, - withPoetClients(poets), + WithPoetClients(poets...), ) require.NoError(t, err) @@ -1200,3 +1139,31 @@ func TestCalculatingGetProofWaitTime(t *testing.T) { ) }) } + +func TestNIPostBuilder_Close(t *testing.T) { + ctrl := gomock.NewController(t) + sig, err := signing.NewEdSigner() + require.NoError(t, err) + + poet := defaultPoetServiceMock(t, ctrl, "http://localhost:9999") + poet.EXPECT().Proof(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(ctx context.Context, _ string) (*types.PoetProofMessage, []types.Hash32, error) { + return nil, nil, ctx.Err() + }) + nb, err := NewNIPostBuilder( + localsql.InMemory(), + NewMockpostService(ctrl), + zaptest.NewLogger(t), + PoetConfig{}, + defaultLayerClockMock(ctrl), + WithPoetClients(poet), + ) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + challenge := types.RandomHash() + + _, err = nb.BuildNIPost(ctx, sig, postGenesisEpoch+2, challenge) + require.Error(t, err) +} diff --git a/activation/poet.go b/activation/poet.go index 9a938cd3a6..ef0457fa6a 100644 --- a/activation/poet.go +++ b/activation/poet.go @@ -21,12 +21,13 @@ import ( "github.com/spacemeshos/go-spacemesh/activation/metrics" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" + "github.com/spacemeshos/go-spacemesh/sql/localsql/certifier" ) var ( - ErrNotFound = errors.New("not found") - ErrUnavailable = errors.New("unavailable") - ErrInvalidRequest = errors.New("invalid request") + ErrInvalidRequest = errors.New("invalid request") + ErrUnauthorized = errors.New("unauthorized") + errCertificatesNotSupported = errors.New("poet doesn't support certificates") ) type PoetPowParams struct { @@ -39,6 +40,11 @@ type PoetPoW struct { Params PoetPowParams } +type PoetAuth struct { + *PoetPoW + *certifier.PoetCert +} + // HTTPPoetClient implements PoetProvingServiceClient interface. type HTTPPoetClient struct { baseURL *url.URL @@ -152,6 +158,22 @@ func (c *HTTPPoetClient) PowParams(ctx context.Context) (*PoetPowParams, error) }, nil } +func (c *HTTPPoetClient) CertifierInfo(ctx context.Context) (*url.URL, []byte, error) { + info, err := c.info(ctx) + if err != nil { + return nil, nil, err + } + certifierInfo := info.GetCertifier() + if certifierInfo == nil { + return nil, nil, errCertificatesNotSupported + } + url, err := url.Parse(certifierInfo.Url) + if err != nil { + return nil, nil, fmt.Errorf("parsing certifier address: %w", err) + } + return url, certifierInfo.Pubkey, nil +} + // Submit registers a challenge in the proving service current open round. func (c *HTTPPoetClient) Submit( ctx context.Context, @@ -159,20 +181,29 @@ func (c *HTTPPoetClient) Submit( prefix, challenge []byte, signature types.EdSignature, nodeID types.NodeID, - pow PoetPoW, + auth PoetAuth, ) (*types.PoetRound, error) { request := rpcapi.SubmitRequest{ Prefix: prefix, Challenge: challenge, Signature: signature.Bytes(), Pubkey: nodeID.Bytes(), - Nonce: pow.Nonce, - PowParams: &rpcapi.PowParams{ - Challenge: pow.Params.Challenge, - Difficulty: uint32(pow.Params.Difficulty), - }, - Deadline: timestamppb.New(deadline), + Deadline: timestamppb.New(deadline), + } + if auth.PoetPoW != nil { + request.PowParams = &rpcapi.PowParams{ + Challenge: auth.PoetPoW.Params.Challenge, + Difficulty: uint32(auth.PoetPoW.Params.Difficulty), + } + request.Nonce = auth.PoetPoW.Nonce + } + if auth.PoetCert != nil { + request.Certificate = &rpcapi.SubmitRequest_Certificate{ + Data: auth.PoetCert.Data, + Signature: auth.PoetCert.Signature, + } } + resBody := rpcapi.SubmitResponse{} if err := c.req(ctx, http.MethodPost, "/v1/submit", &request, &resBody); err != nil { return nil, fmt.Errorf("submitting challenge: %w", err) @@ -185,6 +216,14 @@ func (c *HTTPPoetClient) Submit( return &types.PoetRound{ID: resBody.RoundId, End: roundEnd}, nil } +func (c *HTTPPoetClient) info(ctx context.Context) (*rpcapi.InfoResponse, error) { + resBody := rpcapi.InfoResponse{} + if err := c.req(ctx, http.MethodGet, "/v1/info", nil, &resBody); err != nil { + return nil, fmt.Errorf("getting poet info: %w", err) + } + return &resBody, nil +} + // Proof implements PoetProvingServiceClient. func (c *HTTPPoetClient) Proof(ctx context.Context, roundID string) (*types.PoetProofMessage, []types.Hash32, error) { resBody := rpcapi.ProofResponse{} @@ -249,12 +288,10 @@ func (c *HTTPPoetClient) req(ctx context.Context, method, path string, reqBody, switch res.StatusCode { case http.StatusOK: - case http.StatusNotFound: - return fmt.Errorf("%w: response status code: %s, body: %s", ErrNotFound, res.Status, string(data)) - case http.StatusServiceUnavailable: - return fmt.Errorf("%w: response status code: %s, body: %s", ErrUnavailable, res.Status, string(data)) case http.StatusBadRequest: return fmt.Errorf("%w: response status code: %s, body: %s", ErrInvalidRequest, res.Status, string(data)) + case http.StatusUnauthorized: + return fmt.Errorf("%w: response status code: %s, body: %s", ErrUnauthorized, res.Status, string(data)) default: return fmt.Errorf("unrecognized error: status code: %s, body: %s", res.Status, string(data)) } @@ -269,9 +306,9 @@ func (c *HTTPPoetClient) req(ctx context.Context, method, path string, reqBody, return nil } -// PoetClient is a higher-level interface to communicate with a PoET service. +// poetClient is a higher-level interface to communicate with a PoET service. // It wraps the HTTP client, adding additional functionality. -type PoetClient struct { +type poetClient struct { db poetDbAPI id []byte logger *zap.Logger @@ -282,14 +319,30 @@ type PoetClient struct { gettingProof sync.Mutex // cached members of the last queried proof proofMembers map[string][]types.Hash32 + + certifier certifierService +} + +type PoetClientOpt func(*poetClient) + +func WithCertifier(certifier certifierService) PoetClientOpt { + return func(c *poetClient) { + c.certifier = certifier + } } -func newPoetClient(db poetDbAPI, server types.PoetServer, cfg PoetConfig, logger *zap.Logger) (*PoetClient, error) { +func NewPoetClient( + db poetDbAPI, + server types.PoetServer, + cfg PoetConfig, + logger *zap.Logger, + opts ...PoetClientOpt, +) (*poetClient, error) { client, err := NewHTTPPoetClient(server, cfg, WithLogger(logger)) if err != nil { return nil, fmt.Errorf("creating HTTP poet client: %w", err) } - poetClient := &PoetClient{ + poetClient := &poetClient{ db: db, id: server.Pubkey.Bytes(), logger: logger, @@ -298,25 +351,35 @@ func newPoetClient(db poetDbAPI, server types.PoetServer, cfg PoetConfig, logger proofMembers: make(map[string][]types.Hash32, 1), } + for _, opt := range opts { + opt(poetClient) + } + return poetClient, nil } -func (c *PoetClient) Address() string { +func (c *poetClient) Address() string { return c.client.Address() } -func (c *PoetClient) Submit( +func (c *poetClient) authorize( ctx context.Context, - deadline time.Time, - prefix, challenge []byte, - signature types.EdSignature, nodeID types.NodeID, -) (*types.PoetRound, error) { - logger := c.logger.With( - log.ZContext(ctx), - zap.String("poet", c.Address()), - log.ZShortStringer("smesherID", nodeID), - ) + challenge []byte, + logger *zap.Logger, +) (*PoetAuth, error) { + logger.Debug("certifying node") + cert, err := c.Certify(ctx, nodeID) + switch { + case err == nil: + return &PoetAuth{PoetCert: cert}, nil + case errors.Is(err, errCertificatesNotSupported): + logger.Debug("poet doesn't support certificates") + default: + logger.Warn("failed to certify", zap.Error(err)) + } + // Fallback to PoW + // TODO: remove this fallback once we migrate to certificates fully. logger.Debug("querying for poet pow parameters") powCtx, cancel := withConditionalTimeout(ctx, c.requestTimeout) @@ -340,17 +403,51 @@ func (c *PoetClient) Submit( return nil, fmt.Errorf("running poet PoW: %w", err) } + return &PoetAuth{PoetPoW: &PoetPoW{ + Nonce: nonce, + Params: *powParams, + }}, nil +} + +func (c *poetClient) Submit( + ctx context.Context, + deadline time.Time, + prefix, challenge []byte, + signature types.EdSignature, + nodeID types.NodeID, +) (*types.PoetRound, error) { + logger := c.logger.With( + log.ZContext(ctx), + zap.String("poet", c.Address()), + log.ZShortStringer("smesherID", nodeID), + ) + + // Try obtain a certificate + auth, err := c.authorize(ctx, nodeID, challenge, logger) + if err != nil { + return nil, fmt.Errorf("authorizing: %w", err) + } + logger.Debug("submitting challenge to poet proving service") submitCtx, cancel := withConditionalTimeout(ctx, c.requestTimeout) defer cancel() - return c.client.Submit(submitCtx, deadline, prefix, challenge, signature, nodeID, PoetPoW{ - Nonce: nonce, - Params: *powParams, - }) + round, err := c.client.Submit(submitCtx, deadline, prefix, challenge, signature, nodeID, *auth) + switch { + case err == nil: + return round, nil + case errors.Is(err, ErrUnauthorized): + logger.Warn("failed to submit challenge as unathorized - recertifying", zap.Error(err)) + auth.PoetCert, err = c.recertify(ctx, nodeID) + if err != nil { + return nil, fmt.Errorf("recertifying: %w", err) + } + return c.client.Submit(submitCtx, deadline, prefix, challenge, signature, nodeID, *auth) + } + return nil, fmt.Errorf("submitting challenge: %w", err) } -func (c *PoetClient) Proof(ctx context.Context, roundID string) (*types.PoetProof, []types.Hash32, error) { +func (c *poetClient) Proof(ctx context.Context, roundID string) (*types.PoetProof, []types.Hash32, error) { getProofsCtx, cancel := withConditionalTimeout(ctx, c.requestTimeout) defer cancel() @@ -380,3 +477,25 @@ func (c *PoetClient) Proof(ctx context.Context, roundID string) (*types.PoetProo return &proof.PoetProof, members, nil } + +func (c *poetClient) Certify(ctx context.Context, id types.NodeID) (*certifier.PoetCert, error) { + if c.certifier == nil { + return nil, errors.New("certifier not configured") + } + url, pubkey, err := c.client.CertifierInfo(ctx) + if err != nil { + return nil, fmt.Errorf("getting certifier info: %w", err) + } + return c.certifier.Certificate(ctx, id, url, pubkey) +} + +func (c *poetClient) recertify(ctx context.Context, id types.NodeID) (*certifier.PoetCert, error) { + if c.certifier == nil { + return nil, errors.New("certifier not configured") + } + url, pubkey, err := c.client.CertifierInfo(ctx) + if err != nil { + return nil, fmt.Errorf("getting certifier info: %w", err) + } + return c.certifier.Recertify(ctx, id, url, pubkey) +} diff --git a/activation/poet_client_test.go b/activation/poet_client_test.go index 7d069ed104..1acfbbffda 100644 --- a/activation/poet_client_test.go +++ b/activation/poet_client_test.go @@ -2,8 +2,10 @@ package activation import ( "context" + "io" "net/http" "net/http/httptest" + "net/url" "sync/atomic" "testing" "time" @@ -17,6 +19,8 @@ import ( "google.golang.org/protobuf/encoding/protojson" "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/signing" + "github.com/spacemeshos/go-spacemesh/sql/localsql/certifier" ) func Test_HTTPPoetClient_ParsesURL(t *testing.T) { @@ -68,7 +72,7 @@ func Test_HTTPPoetClient_Submit(t *testing.T) { nil, types.EmptyEdSignature, types.NodeID{}, - PoetPoW{}, + PoetAuth{}, ) require.NoError(t, err) } @@ -169,7 +173,7 @@ func TestPoetClient_CachesProof(t *testing.T) { db.EXPECT().ValidateAndStore(ctx, gomock.Any()) db.EXPECT().ProofForRound(server.Pubkey.Bytes(), "1").Times(19) - poet, err := newPoetClient(db, server, DefaultPoetConfig(), zaptest.NewLogger(t)) + poet, err := NewPoetClient(db, server, DefaultPoetConfig(), zaptest.NewLogger(t)) require.NoError(t, err) poet.client.client.HTTPClient = ts.Client() @@ -206,7 +210,7 @@ func TestPoetClient_QueryProofTimeout(t *testing.T) { cfg := PoetConfig{ RequestTimeout: time.Millisecond * 100, } - poet, err := newPoetClient(nil, server, cfg, zaptest.NewLogger(t)) + poet, err := NewPoetClient(nil, server, cfg, zaptest.NewLogger(t)) require.NoError(t, err) poet.client.client.HTTPClient = ts.Client() @@ -222,3 +226,164 @@ func TestPoetClient_QueryProofTimeout(t *testing.T) { eg.Wait() require.WithinDuration(t, start.Add(cfg.RequestTimeout), time.Now(), time.Millisecond*300) } + +func TestPoetClient_Certify(t *testing.T) { + t.Parallel() + + sig, err := signing.NewEdSigner() + require.NoError(t, err) + + certifierAddress := &url.URL{Scheme: "http", Host: "certifier"} + certifierPubKey := []byte("certifier-pubkey") + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/info": + resp, err := protojson.Marshal(&rpcapi.InfoResponse{ + ServicePubkey: []byte("pubkey"), + Certifier: &rpcapi.InfoResponse_Cerifier{ + Url: certifierAddress.String(), + Pubkey: certifierPubKey, + }, + }) + require.NoError(t, err) + w.Write(resp) + } + })) + defer ts.Close() + + server := types.PoetServer{ + Address: ts.URL, + Pubkey: types.NewBase64Enc([]byte("pubkey")), + } + cfg := PoetConfig{RequestTimeout: time.Millisecond * 100} + cert := certifier.PoetCert{Data: []byte("abc")} + ctrl := gomock.NewController(t) + mCertifier := NewMockcertifierService(ctrl) + mCertifier.EXPECT(). + Certificate(gomock.Any(), sig.NodeID(), certifierAddress, certifierPubKey). + Return(&cert, nil) + + poet, err := NewPoetClient(nil, server, cfg, zaptest.NewLogger(t), WithCertifier(mCertifier)) + require.NoError(t, err) + poet.client.client.HTTPClient = ts.Client() + + got, err := poet.Certify(context.Background(), sig.NodeID()) + require.NoError(t, err) + require.Equal(t, cert, *got) +} + +func TestPoetClient_ObtainsCertOnSubmit(t *testing.T) { + t.Parallel() + + sig, err := signing.NewEdSigner() + require.NoError(t, err) + + certifierAddress := &url.URL{Scheme: "http", Host: "certifier"} + certifierPubKey := []byte("certifier-pubkey") + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/info": + resp, err := protojson.Marshal(&rpcapi.InfoResponse{ + ServicePubkey: []byte("pubkey"), + Certifier: &rpcapi.InfoResponse_Cerifier{ + Url: certifierAddress.String(), + Pubkey: certifierPubKey, + }, + }) + require.NoError(t, err) + w.Write(resp) + case "/v1/submit": + resp, err := protojson.Marshal(&rpcapi.SubmitResponse{}) + require.NoError(t, err) + w.Write(resp) + } + })) + + defer ts.Close() + + server := types.PoetServer{ + Address: ts.URL, + Pubkey: types.NewBase64Enc([]byte("pubkey")), + } + cfg := PoetConfig{RequestTimeout: time.Millisecond * 100} + cert := certifier.PoetCert{Data: []byte("abc")} + ctrl := gomock.NewController(t) + mCertifier := NewMockcertifierService(ctrl) + mCertifier.EXPECT(). + Certificate(gomock.Any(), sig.NodeID(), certifierAddress, certifierPubKey). + Return(&cert, nil) + + poet, err := NewPoetClient(nil, server, cfg, zaptest.NewLogger(t), WithCertifier(mCertifier)) + require.NoError(t, err) + poet.client.client.HTTPClient = ts.Client() + + _, err = poet.Submit(context.Background(), time.Time{}, nil, nil, types.RandomEdSignature(), sig.NodeID()) + require.NoError(t, err) +} + +func TestPoetClient_RecertifiesOnAuthFailure(t *testing.T) { + t.Parallel() + + sig, err := signing.NewEdSigner() + require.NoError(t, err) + + certifierAddress := &url.URL{Scheme: "http", Host: "certifier"} + certifierPubKey := []byte("certifier-pubkey") + submitCount := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/info": + resp, err := protojson.Marshal(&rpcapi.InfoResponse{ + ServicePubkey: []byte("pubkey"), + Certifier: &rpcapi.InfoResponse_Cerifier{ + Url: certifierAddress.String(), + Pubkey: certifierPubKey, + }, + }) + require.NoError(t, err) + w.Write(resp) + case "/v1/submit": + req := rpcapi.SubmitRequest{} + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.NoError(t, protojson.Unmarshal(body, &req)) + if submitCount == 0 { + require.EqualValues(t, "first", req.Certificate.Data) + w.WriteHeader(http.StatusUnauthorized) + } else { + require.EqualValues(t, "second", req.Certificate.Data) + resp, err := protojson.Marshal(&rpcapi.SubmitResponse{}) + require.NoError(t, err) + w.Write(resp) + } + submitCount++ + } + })) + + defer ts.Close() + + server := types.PoetServer{ + Address: ts.URL, + Pubkey: types.NewBase64Enc([]byte("pubkey")), + } + cfg := PoetConfig{RequestTimeout: time.Millisecond * 100} + + ctrl := gomock.NewController(t) + mCertifier := NewMockcertifierService(ctrl) + gomock.InOrder( + mCertifier.EXPECT(). + Certificate(gomock.Any(), sig.NodeID(), certifierAddress, certifierPubKey). + Return(&certifier.PoetCert{Data: []byte("first")}, nil), + mCertifier.EXPECT(). + Recertify(gomock.Any(), sig.NodeID(), certifierAddress, certifierPubKey). + Return(&certifier.PoetCert{Data: []byte("second")}, nil), + ) + + poet, err := NewPoetClient(nil, server, cfg, zaptest.NewLogger(t), WithCertifier(mCertifier)) + require.NoError(t, err) + poet.client.client.HTTPClient = ts.Client() + + _, err = poet.Submit(context.Background(), time.Time{}, nil, nil, types.RandomEdSignature(), sig.NodeID()) + require.NoError(t, err) + require.Equal(t, 2, submitCount) +} diff --git a/activation/post_states_test.go b/activation/post_states_test.go index c8e71ba6fd..b737f0fa43 100644 --- a/activation/post_states_test.go +++ b/activation/post_states_test.go @@ -48,10 +48,8 @@ func TestPostState_OnProof(t *testing.T) { mPostService := NewMockpostService(ctrl) mPostClient := NewMockPostClient(ctrl) nb, err := NewNIPostBuilder( - nil, nil, mPostService, - []types.PoetServer{}, zaptest.NewLogger(t), PoetConfig{}, nil, diff --git a/checkpoint/recovery.go b/checkpoint/recovery.go index 6f73994ac6..423f09ae08 100644 --- a/checkpoint/recovery.go +++ b/checkpoint/recovery.go @@ -388,6 +388,7 @@ func collectOwnAtxDeps( nipostCh, _ := nipost.Challenge(localDB, nodeID) if ref == types.EmptyATXID { if nipostCh == nil { + logger.Debug("there is no own atx and none is being built") return nil, nil, nil } if nipostCh.CommitmentATX != nil { @@ -428,6 +429,7 @@ func collectOwnAtxDeps( maps.Copy(deps, deps2) maps.Copy(proofs, proofs2) } + logger.With().Debug("collected atx deps", log.Any("deps", deps)) return deps, proofs, nil } diff --git a/cmd/merge-nodes/internal/merge_action.go b/cmd/merge-nodes/internal/merge_action.go index c773b5acff..43b9929fdc 100644 --- a/cmd/merge-nodes/internal/merge_action.go +++ b/cmd/merge-nodes/internal/merge_action.go @@ -157,7 +157,7 @@ func MergeDBs(ctx context.Context, dbLog *zap.Logger, from, to string) error { if _, err := tx.Exec("ATTACH DATABASE ?1 AS srcDB;", enc, nil); err != nil { return fmt.Errorf("attach source database: %w", err) } - if _, err := tx.Exec("INSERT INTO main.initial_post SELECT * FROM srcDB.initial_post;", nil, nil); err != nil { + if _, err := tx.Exec("INSERT INTO main.post SELECT * FROM srcDB.post;", nil, nil); err != nil { return fmt.Errorf("merge initial_post: %w", err) } if _, err := tx.Exec("INSERT INTO main.challenge SELECT * FROM srcDB.challenge;", nil, nil); err != nil { diff --git a/cmd/merge-nodes/internal/merge_action_test.go b/cmd/merge-nodes/internal/merge_action_test.go index 77fb22195b..81b4e3bb12 100644 --- a/cmd/merge-nodes/internal/merge_action_test.go +++ b/cmd/merge-nodes/internal/merge_action_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/spacemeshos/post/shared" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -218,12 +219,13 @@ func Test_MergeDBs_Successful_Existing_Node(t *testing.T) { require.NoError(t, err) sig1Post := nipost.Post{ - Nonce: rand.Uint32(), - Pow: rand.Uint64(), - Indices: types.RandomBytes(32), - NumUnits: rand.Uint32(), + Nonce: rand.Uint32(), + Pow: rand.Uint64(), + Indices: types.RandomBytes(32), + NumUnits: rand.Uint32(), + Challenge: shared.ZeroChallenge, } - err = nipost.AddInitialPost(dstDB, sig1.NodeID(), sig1Post) + err = nipost.AddPost(dstDB, sig1.NodeID(), sig1Post) require.NoError(t, err) sig1Poet1 := nipost.PoETRegistration{ @@ -276,12 +278,13 @@ func Test_MergeDBs_Successful_Existing_Node(t *testing.T) { require.NoError(t, err) sig2Post := nipost.Post{ - Nonce: rand.Uint32(), - Pow: rand.Uint64(), - Indices: types.RandomBytes(32), - NumUnits: rand.Uint32(), + Nonce: rand.Uint32(), + Pow: rand.Uint64(), + Indices: types.RandomBytes(32), + NumUnits: rand.Uint32(), + Challenge: shared.ZeroChallenge, } - err = nipost.AddInitialPost(srcDB, sig2.NodeID(), sig2Post) + err = nipost.AddPost(srcDB, sig2.NodeID(), sig2Post) require.NoError(t, err) sig2Poet1 := nipost.PoETRegistration{ @@ -317,7 +320,7 @@ func Test_MergeDBs_Successful_Existing_Node(t *testing.T) { require.NoError(t, err) require.Equal(t, sig1Ch, ch) - post, err := nipost.InitialPost(dstDB, sig1.NodeID()) + post, err := nipost.GetPost(dstDB, sig1.NodeID()) require.NoError(t, err) require.Equal(t, sig1Post, *post) @@ -330,7 +333,7 @@ func Test_MergeDBs_Successful_Existing_Node(t *testing.T) { require.NoError(t, err) require.Equal(t, sig2Ch, ch) - post, err = nipost.InitialPost(dstDB, sig2.NodeID()) + post, err = nipost.GetPost(dstDB, sig2.NodeID()) require.NoError(t, err) require.Equal(t, sig2Post, *post) @@ -376,12 +379,13 @@ func Test_MergeDBs_Successful_Empty_Dir(t *testing.T) { require.NoError(t, err) sigPost := nipost.Post{ - Nonce: rand.Uint32(), - Pow: rand.Uint64(), - Indices: types.RandomBytes(32), - NumUnits: rand.Uint32(), + Nonce: rand.Uint32(), + Pow: rand.Uint64(), + Indices: types.RandomBytes(32), + NumUnits: rand.Uint32(), + Challenge: shared.ZeroChallenge, } - err = nipost.AddInitialPost(srcDB, sig.NodeID(), sigPost) + err = nipost.AddPost(srcDB, sig.NodeID(), sigPost) require.NoError(t, err) sigPoet1 := nipost.PoETRegistration{ @@ -416,7 +420,7 @@ func Test_MergeDBs_Successful_Empty_Dir(t *testing.T) { require.NoError(t, err) require.Equal(t, sigCh, ch) - post, err := nipost.InitialPost(dstDB, sig.NodeID()) + post, err := nipost.GetPost(dstDB, sig.NodeID()) require.NoError(t, err) require.Equal(t, sigPost, *post) diff --git a/config/config.go b/config/config.go index 673d497bb7..230d9a9a8a 100644 --- a/config/config.go +++ b/config/config.go @@ -47,19 +47,20 @@ func init() { // Config defines the top level configuration for a spacemesh node. type Config struct { BaseConfig `mapstructure:"main"` - Preset string `mapstructure:"preset"` - Genesis GenesisConfig `mapstructure:"genesis"` - PublicMetrics PublicMetrics `mapstructure:"public-metrics"` - Tortoise tortoise.Config `mapstructure:"tortoise"` - P2P p2p.Config `mapstructure:"p2p"` - API grpcserver.Config `mapstructure:"api"` - HARE3 hare3.Config `mapstructure:"hare3"` - HareEligibility eligibility.Config `mapstructure:"hare-eligibility"` - Certificate blocks.CertConfig `mapstructure:"certificate"` - Beacon beacon.Config `mapstructure:"beacon"` - TIME timeConfig.TimeConfig `mapstructure:"time"` - VM vm.Config `mapstructure:"vm"` - POST activation.PostConfig `mapstructure:"post"` + Preset string `mapstructure:"preset"` + Genesis GenesisConfig `mapstructure:"genesis"` + PublicMetrics PublicMetrics `mapstructure:"public-metrics"` + Tortoise tortoise.Config `mapstructure:"tortoise"` + P2P p2p.Config `mapstructure:"p2p"` + API grpcserver.Config `mapstructure:"api"` + HARE3 hare3.Config `mapstructure:"hare3"` + HareEligibility eligibility.Config `mapstructure:"hare-eligibility"` + Certificate blocks.CertConfig `mapstructure:"certificate"` + Beacon beacon.Config `mapstructure:"beacon"` + TIME timeConfig.TimeConfig `mapstructure:"time"` + VM vm.Config `mapstructure:"vm"` + Certifier activation.CertifierConfig `mapstructure:"certifier"` + POST activation.PostConfig `mapstructure:"post"` POSTService activation.PostSupervisorConfig POET activation.PoetConfig `mapstructure:"poet"` SMESHING SmeshingConfig `mapstructure:"smeshing"` @@ -205,6 +206,7 @@ func DefaultConfig() Config { Recovery: checkpoint.DefaultConfig(), Cache: datastore.DefaultConfig(), ActiveSet: miner.DefaultActiveSetPreparation(), + Certifier: activation.DefaultCertifierConfig(), } } diff --git a/config/presets/testnet.go b/config/presets/testnet.go index 78396f8c8d..892d924aaf 100644 --- a/config/presets/testnet.go +++ b/config/presets/testnet.go @@ -163,5 +163,6 @@ func testnet() config.Config { RetryInterval: time.Minute, Tries: 5, }, + Certifier: activation.DefaultCertifierConfig(), } } diff --git a/go.mod b/go.mod index e08817fd52..7f1acaaef9 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/spacemeshos/fixed v0.1.1 github.com/spacemeshos/go-scale v1.2.0 github.com/spacemeshos/merkle-tree v0.2.3 - github.com/spacemeshos/poet v0.10.2 + github.com/spacemeshos/poet v0.10.3 github.com/spacemeshos/post v0.12.6 github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.0 diff --git a/go.sum b/go.sum index 0e3f3df769..9414948049 100644 --- a/go.sum +++ b/go.sum @@ -611,8 +611,8 @@ github.com/spacemeshos/go-scale v1.2.0 h1:ZlA2L1ILym2gmyJUwUdLTiyP1ZIG0U4xE9nFVF github.com/spacemeshos/go-scale v1.2.0/go.mod h1:HV6e3/X5h9u2aFpYKJxt7PY/fBuLBegEKWgeZJ+/5jE= github.com/spacemeshos/merkle-tree v0.2.3 h1:zGEgOR9nxAzJr0EWjD39QFngwFEOxfxMloEJZtAysas= github.com/spacemeshos/merkle-tree v0.2.3/go.mod h1:VomOcQ5pCBXz7goiWMP5hReyqOfDXGSKbrH2GB9Htww= -github.com/spacemeshos/poet v0.10.2 h1:FVb0xgCFcjZyIGBQ92SlOZVx4KCmlCRRL4JSHL6LMGU= -github.com/spacemeshos/poet v0.10.2/go.mod h1:73ROEXGladw3RbvhAk0sIGi/ttfpo+ASUBRvnBK55N8= +github.com/spacemeshos/poet v0.10.3 h1:ZDPqihukDphdM+Jr/3xgn7vXadheaRMa6wF70Zsv4fg= +github.com/spacemeshos/poet v0.10.3/go.mod h1:TPZ/aX+YIgIqs/bvYTcJIwUWEUzvZw6jueFPxdhCGpY= github.com/spacemeshos/post v0.12.6 h1:BtKK4n8qa7d0APtQZ2KBx9fQpx1B5LSt2OD7XIgE8g4= github.com/spacemeshos/post v0.12.6/go.mod h1:NEstvZ4BKHuiGTcb+H+cQsZiNSh0G7GOLjZv6jjnHxM= github.com/spacemeshos/sha256-simd v0.1.0 h1:G7Mfu5RYdQiuE+wu4ZyJ7I0TI74uqLhFnKblEnSpjYI= diff --git a/node/node.go b/node/node.go index ae0db0d4c3..4eb86e4b43 100644 --- a/node/node.go +++ b/node/node.go @@ -1000,15 +1000,39 @@ func (app *App) initServices(ctx context.Context) error { if err != nil { return fmt.Errorf("init post grpc service: %w", err) } + + nipostLogger := app.addLogger(NipostBuilderLogger, lg).Zap() + client := activation.NewCertifierClient( + app.db, + app.localDB, + nipostLogger, + activation.WithCertifierClientConfig(app.Config.Certifier.Client), + ) + certifier := activation.NewCertifier(app.localDB, nipostLogger, client) + + poetClients := make([]activation.PoetClient, 0, len(app.Config.PoetServers)) + for _, server := range app.Config.PoetServers { + client, err := activation.NewPoetClient( + poetDb, + server, + app.Config.POET, + lg.Zap().Named("poet"), + activation.WithCertifier(certifier), + ) + if err != nil { + app.log.Panic("failed to create poet client: %v", err) + } + poetClients = append(poetClients, client) + } + nipostBuilder, err := activation.NewNIPostBuilder( app.localDB, - poetDb, grpcPostService.(*grpcserver.PostService), - app.Config.PoetServers, - app.addLogger(NipostBuilderLogger, lg).Zap(), + nipostLogger, app.Config.POET, app.clock, activation.NipostbuilderWithPostStates(postStates), + activation.WithPoetClients(poetClients...), ) if err != nil { return fmt.Errorf("create nipost builder: %w", err) @@ -1036,6 +1060,7 @@ func (app *App) initServices(ctx context.Context) error { activation.WithValidator(app.validator), activation.WithPostValidityDelay(app.Config.PostValidDelay), activation.WithPostStates(postStates), + activation.WithPoets(poetClients...), ) if len(app.signers) > 1 || app.signers[0].Name() != supervisedIDKeyFileName { // in a remote setup we register eagerly so the atxBuilder can warn about missing connections asap. @@ -2101,6 +2126,7 @@ func (app *App) startSynchronous(ctx context.Context) (err error) { func (app *App) preserveAfterRecovery(ctx context.Context) { if app.preserve == nil { + app.log.Info("no need to preserve data after recovery") return } for i, poetProof := range app.preserve.Proofs { diff --git a/sql/localsql/certifier/db.go b/sql/localsql/certifier/db.go new file mode 100644 index 0000000000..cf74c50c99 --- /dev/null +++ b/sql/localsql/certifier/db.go @@ -0,0 +1,55 @@ +package certifier + +import ( + "fmt" + + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/sql" +) + +type PoetCert struct { + Data []byte + Signature []byte +} + +func AddCertificate(db sql.Executor, nodeID types.NodeID, cert PoetCert, cerifierID []byte) error { + enc := func(stmt *sql.Statement) { + stmt.BindBytes(1, nodeID.Bytes()) + stmt.BindBytes(2, cerifierID) + stmt.BindBytes(3, cert.Data) + stmt.BindBytes(4, cert.Signature) + } + if _, err := db.Exec(` + REPLACE INTO poet_certificates (node_id, certifier_id, certificate, signature) + VALUES (?1, ?2, ?3, ?4);`, enc, nil, + ); err != nil { + return fmt.Errorf("storing poet certificate for (%s; %x): %w", nodeID.ShortString(), cerifierID, err) + } + return nil +} + +func Certificate(db sql.Executor, nodeID types.NodeID, certifierID []byte) (*PoetCert, error) { + enc := func(stmt *sql.Statement) { + stmt.BindBytes(1, nodeID.Bytes()) + stmt.BindBytes(2, certifierID) + } + var cert PoetCert + dec := func(stmt *sql.Statement) bool { + cert.Data = make([]byte, stmt.ColumnLen(0)) + cert.Signature = make([]byte, stmt.ColumnLen(1)) + stmt.ColumnBytes(0, cert.Data) + stmt.ColumnBytes(1, cert.Signature) + return true + } + rows, err := db.Exec(` + select certificate, signature + from poet_certificates where node_id = ?1 and certifier_id = ?2 limit 1;`, enc, dec, + ) + switch { + case err != nil: + return nil, fmt.Errorf("getting poet certificate for (%s; %s): %w", nodeID.ShortString(), certifierID, err) + case rows == 0: + return nil, sql.ErrNotFound + } + return &cert, nil +} diff --git a/sql/localsql/certifier/db_test.go b/sql/localsql/certifier/db_test.go new file mode 100644 index 0000000000..d0611db432 --- /dev/null +++ b/sql/localsql/certifier/db_test.go @@ -0,0 +1,50 @@ +package certifier_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/sql/localsql" + "github.com/spacemeshos/go-spacemesh/sql/localsql/certifier" +) + +func TestAddingCertificates(t *testing.T) { + db := localsql.InMemory() + nodeId := types.RandomNodeID() + + expCert := certifier.PoetCert{Data: []byte("data"), Signature: []byte("sig")} + + require.NoError(t, certifier.AddCertificate(db, nodeId, expCert, []byte("certifier-0"))) + cert, err := certifier.Certificate(db, nodeId, []byte("certifier-0")) + require.NoError(t, err) + require.Equal(t, &expCert, cert) + + expCert2 := certifier.PoetCert{Data: []byte("data2"), Signature: []byte("sig2")} + require.NoError(t, certifier.AddCertificate(db, nodeId, expCert2, []byte("certifier-1"))) + + cert, err = certifier.Certificate(db, nodeId, []byte("certifier-1")) + require.NoError(t, err) + require.Equal(t, &expCert2, cert) + cert, err = certifier.Certificate(db, nodeId, []byte("certifier-0")) + require.NoError(t, err) + require.Equal(t, &expCert, cert) +} + +func TestOverwritingCertificates(t *testing.T) { + db := localsql.InMemory() + nodeId := types.RandomNodeID() + + expCert := certifier.PoetCert{Data: []byte("data"), Signature: []byte("sig")} + require.NoError(t, certifier.AddCertificate(db, nodeId, expCert, []byte("certifier-0"))) + cert, err := certifier.Certificate(db, nodeId, []byte("certifier-0")) + require.NoError(t, err) + require.Equal(t, &expCert, cert) + + expCert2 := certifier.PoetCert{Data: []byte("data2"), Signature: []byte("sig2")} + require.NoError(t, certifier.AddCertificate(db, nodeId, expCert2, []byte("certifier-0"))) + cert, err = certifier.Certificate(db, nodeId, []byte("certifier-0")) + require.NoError(t, err) + require.Equal(t, &expCert2, cert) +} diff --git a/sql/localsql/nipost/initial_post.go b/sql/localsql/nipost/initial_post.go deleted file mode 100644 index a4c84ab6d4..0000000000 --- a/sql/localsql/nipost/initial_post.go +++ /dev/null @@ -1,78 +0,0 @@ -package nipost - -import ( - "fmt" - - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/sql" -) - -type Post struct { - Nonce uint32 - Indices []byte - Pow uint64 - - NumUnits uint32 - CommitmentATX types.ATXID - VRFNonce types.VRFPostIndex -} - -func AddInitialPost(db sql.Executor, nodeID types.NodeID, post Post) error { - enc := func(stmt *sql.Statement) { - stmt.BindBytes(1, nodeID.Bytes()) - stmt.BindInt64(2, int64(post.Nonce)) - stmt.BindBytes(3, post.Indices) - stmt.BindInt64(4, int64(post.Pow)) - stmt.BindInt64(5, int64(post.NumUnits)) - stmt.BindBytes(6, post.CommitmentATX.Bytes()) - stmt.BindInt64(7, int64(post.VRFNonce)) - } - if _, err := db.Exec(` - insert into initial_post ( - id, post_nonce, post_indices, post_pow, num_units, commit_atx, vrf_nonce - ) values (?1, ?2, ?3, ?4, ?5, ?6, ?7);`, enc, nil, - ); err != nil { - return fmt.Errorf("insert initial post for %s: %w", nodeID.ShortString(), err) - } - return nil -} - -func RemoveInitialPost(db sql.Executor, nodeID types.NodeID) error { - enc := func(stmt *sql.Statement) { - stmt.BindBytes(1, nodeID.Bytes()) - } - if _, err := db.Exec(`delete from initial_post where id = ?1;`, enc, nil); err != nil { - return fmt.Errorf("remove initial post for %s: %w", nodeID, err) - } - return nil -} - -func InitialPost(db sql.Executor, nodeID types.NodeID) (*Post, error) { - var post *Post - enc := func(stmt *sql.Statement) { - stmt.BindBytes(1, nodeID.Bytes()) - } - dec := func(stmt *sql.Statement) bool { - post = &Post{ - Nonce: uint32(stmt.ColumnInt64(0)), - Indices: make([]byte, stmt.ColumnLen(1)), - Pow: uint64(stmt.ColumnInt64(2)), - - NumUnits: uint32(stmt.ColumnInt64(3)), - VRFNonce: types.VRFPostIndex(stmt.ColumnInt64(5)), - } - stmt.ColumnBytes(1, post.Indices) - stmt.ColumnBytes(4, post.CommitmentATX[:]) - return true - } - if _, err := db.Exec(` - select post_nonce, post_indices, post_pow, num_units, commit_atx, vrf_nonce - from initial_post where id = ?1 limit 1;`, enc, dec, - ); err != nil { - return nil, fmt.Errorf("get initial post from node id %s: %w", nodeID.ShortString(), err) - } - if post == nil { - return nil, fmt.Errorf("get initial post from node id %s: %w", nodeID.ShortString(), sql.ErrNotFound) - } - return post, nil -} diff --git a/sql/localsql/nipost/post.go b/sql/localsql/nipost/post.go new file mode 100644 index 0000000000..e09adde451 --- /dev/null +++ b/sql/localsql/nipost/post.go @@ -0,0 +1,84 @@ +package nipost + +import ( + "fmt" + + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/sql" +) + +type Post struct { + Nonce uint32 + Indices []byte + Pow uint64 + Challenge []byte + + NumUnits uint32 + CommitmentATX types.ATXID + VRFNonce types.VRFPostIndex +} + +func AddPost(db sql.Executor, nodeID types.NodeID, post Post) error { + enc := func(stmt *sql.Statement) { + stmt.BindBytes(1, nodeID.Bytes()) + stmt.BindInt64(2, int64(post.Nonce)) + stmt.BindBytes(3, post.Indices) + stmt.BindInt64(4, int64(post.Pow)) + stmt.BindBytes(5, post.Challenge) + stmt.BindInt64(6, int64(post.NumUnits)) + stmt.BindBytes(7, post.CommitmentATX.Bytes()) + stmt.BindInt64(8, int64(post.VRFNonce)) + } + if _, err := db.Exec(` + INSERT into post ( + id, post_nonce, post_indices, post_pow, challenge, num_units, commit_atx, vrf_nonce + ) values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8);`, enc, nil, + ); err != nil { + return fmt.Errorf("inserting post for %s: %w", nodeID.ShortString(), err) + } + return nil +} + +func RemovePost(db sql.Executor, nodeID types.NodeID) error { + enc := func(stmt *sql.Statement) { + stmt.BindBytes(1, nodeID.Bytes()) + } + if _, err := db.Exec(` + delete from post where id = ?1;`, enc, nil, + ); err != nil { + return fmt.Errorf("delete post for %s: %w", nodeID, err) + } + return nil +} + +func GetPost(db sql.Executor, nodeID types.NodeID) (*Post, error) { + var post *Post + enc := func(stmt *sql.Statement) { + stmt.BindBytes(1, nodeID.Bytes()) + } + dec := func(stmt *sql.Statement) bool { + post = &Post{ + Nonce: uint32(stmt.ColumnInt64(0)), + Indices: make([]byte, stmt.ColumnLen(1)), + Pow: uint64(stmt.ColumnInt64(2)), + Challenge: make([]byte, stmt.ColumnLen(3)), + + NumUnits: uint32(stmt.ColumnInt64(4)), + VRFNonce: types.VRFPostIndex(stmt.ColumnInt64(6)), + } + stmt.ColumnBytes(1, post.Indices) + stmt.ColumnBytes(3, post.Challenge) + stmt.ColumnBytes(5, post.CommitmentATX[:]) + return true + } + if _, err := db.Exec(` + select post_nonce, post_indices, post_pow, challenge, num_units, commit_atx, vrf_nonce + from post where id = ?1 limit 1;`, enc, dec, + ); err != nil { + return nil, fmt.Errorf("getting post for node id %s: %w", nodeID.ShortString(), err) + } + if post == nil { + return nil, fmt.Errorf("getting post for node id %s: %w", nodeID.ShortString(), sql.ErrNotFound) + } + return post, nil +} diff --git a/sql/localsql/nipost/initial_post_test.go b/sql/localsql/nipost/post_test.go similarity index 53% rename from sql/localsql/nipost/initial_post_test.go rename to sql/localsql/nipost/post_test.go index 50337970ad..b72184d437 100644 --- a/sql/localsql/nipost/initial_post_test.go +++ b/sql/localsql/nipost/post_test.go @@ -3,72 +3,68 @@ package nipost import ( "testing" + "github.com/spacemeshos/post/shared" "github.com/stretchr/testify/require" "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sql/localsql" ) -func Test_AddInitialPost(t *testing.T) { +func Test_AddPost(t *testing.T) { db := localsql.InMemory() nodeID := types.RandomNodeID() post := Post{ - Nonce: 1, - Indices: []byte{1, 2, 3}, - Pow: 1, + Nonce: 1, + Indices: []byte{1, 2, 3}, + Pow: 1, + Challenge: shared.Challenge([]byte{4, 5, 6}), NumUnits: 2, CommitmentATX: types.RandomATXID(), VRFNonce: 3, } - err := AddInitialPost(db, nodeID, post) + err := AddPost(db, nodeID, post) require.NoError(t, err) - got, err := InitialPost(db, nodeID) + got, err := GetPost(db, nodeID) require.NoError(t, err) require.NotNil(t, got) require.Equal(t, post, *got) - - err = RemoveInitialPost(db, nodeID) - require.NoError(t, err) - - got, err = InitialPost(db, nodeID) - require.ErrorIs(t, err, sql.ErrNotFound) - require.Nil(t, got) } -func Test_AddInitialPost_NoDuplicates(t *testing.T) { +func Test_AddPost_NoDuplicates(t *testing.T) { db := localsql.InMemory() nodeID := types.RandomNodeID() post := Post{ - Nonce: 1, - Indices: []byte{1, 2, 3}, - Pow: 1, + Nonce: 1, + Indices: []byte{1, 2, 3}, + Pow: 1, + Challenge: shared.ZeroChallenge, NumUnits: 2, CommitmentATX: types.RandomATXID(), VRFNonce: 3, } - err := AddInitialPost(db, nodeID, post) + err := AddPost(db, nodeID, post) require.NoError(t, err) - // fail to add new initial post for same node + // fail to add new post for same node post2 := Post{ - Nonce: 2, - Indices: []byte{1, 2, 3}, - Pow: 1, + Nonce: 2, + Indices: []byte{1, 2, 3}, + Pow: 1, + Challenge: shared.ZeroChallenge, NumUnits: 4, CommitmentATX: types.RandomATXID(), VRFNonce: 5, } - err = AddInitialPost(db, nodeID, post2) + err = AddPost(db, nodeID, post2) require.Error(t, err) // succeed to add initial post for different node - err = AddInitialPost(db, types.RandomNodeID(), post2) + err = AddPost(db, types.RandomNodeID(), post2) require.NoError(t, err) } diff --git a/sql/migrations/local/0008_next.sql b/sql/migrations/local/0008_next.sql new file mode 100644 index 0000000000..26d0bd1822 --- /dev/null +++ b/sql/migrations/local/0008_next.sql @@ -0,0 +1,13 @@ +ALTER TABLE initial_post ADD COLUMN challenge BLOB NOT NULL DEFAULT x'0000000000000000000000000000000000000000000000000000000000000000'; + +ALTER TABLE initial_post RENAME TO post; + +CREATE TABLE poet_certificates +( + node_id BLOB NOT NULL, + certifier_id BLOB NOT NULL, + certificate BLOB NOT NULL, + signature BLOB NOT NULL +); + +CREATE UNIQUE INDEX idx_poet_certificates ON poet_certificates (node_id, certifier_id); diff --git a/systest/Makefile b/systest/Makefile index c3a96138d9..489de35f9d 100644 --- a/systest/Makefile +++ b/systest/Makefile @@ -5,7 +5,8 @@ tmpfile := $(shell mktemp /tmp/systest-XXX) test_name ?= TestSmeshing org ?= spacemeshos image_name ?= $(org)/systest:$(version_info) -poet_image ?= $(org)/poet:v0.10.2 +certifier_image ?= spacemeshos/certifier-service:v0.7.8 +poet_image ?= $(org)/poet:v0.10.3 post_service_image ?= $(org)/post-service:v0.7.8 post_init_image ?= $(org)/postcli:v0.12.5 smesher_image ?= $(org)/go-spacemesh-dev:$(version_info) @@ -61,6 +62,7 @@ template: @echo node-selector=$(node_selector) >> $(tmpfile) @echo image=$(smesher_image) >> $(tmpfile) @echo bs-image=$(bs_image) >> $(tmpfile) + @echo certifier-image=$(certifier_image) >> $(tmpfile) @echo poet-image=$(poet_image) >> $(tmpfile) @echo poet-size=$(poet_size) >> $(tmpfile) @echo post-service-image=$(post_service_image) >> $(tmpfile) diff --git a/systest/cluster/cluster.go b/systest/cluster/cluster.go index 2b3e992b26..54f54f58c6 100644 --- a/systest/cluster/cluster.go +++ b/systest/cluster/cluster.go @@ -2,6 +2,7 @@ package cluster import ( "context" + "encoding/base64" "encoding/hex" "errors" "fmt" @@ -32,12 +33,14 @@ var errNotInitialized = errors.New("cluster: not initialized") const ( initBalance = 100000000000000000 defaultExtraData = "systest" + certifierApp = "certifier" poetApp = "poet" bootnodeApp = "boot" smesherApp = "smesher" postServiceApp = "postservice" bootstrapperApp = "bootstrapper" bootstrapperPort = 80 + certifierPort = 80 poetPort = 80 poetFlags = "poetflags" @@ -50,6 +53,10 @@ func MakePoetEndpoint(ith int) string { return fmt.Sprintf("http://%s:%d", createPoetIdentifier(ith), poetPort) } +func MakePoetMetricsEndpoint(testNamespace string, ith int) string { + return fmt.Sprintf("http://%s.%s:%d/metrics", createPoetIdentifier(ith), testNamespace, prometheusScrapePort) +} + func MakePoetGlobalEndpoint(testNamespace string, ith int) string { return fmt.Sprintf("http://%s.%s:%d", createPoetIdentifier(ith), testNamespace, poetPort) } @@ -170,10 +177,20 @@ func Default(cctx *testcontext.Context, opts ...Opt) (*Cluster, error) { if err := cl.AddBootstrappers(cctx); err != nil { return nil, err } + pubkey, privkey, err := ed25519.GenerateKey(nil) + if err != nil { + return nil, fmt.Errorf("generating keys for certifier: %w", err) + } + if err := cl.AddCertifier(cctx, base64.StdEncoding.EncodeToString(privkey.Seed())); err != nil { + return nil, err + } + cl.addPoetFlag(PoetCertifierURL("http://certifier-0")) + cl.addPoetFlag(PoetCertifierPubkey(base64.StdEncoding.EncodeToString(pubkey))) + if err := cl.AddPoets(cctx); err != nil { return nil, err } - err := cl.AddSmeshers(cctx, smeshers, WithSmeshers(keys[cctx.BootnodeSize:cctx.BootnodeSize+smeshers])) + err = cl.AddSmeshers(cctx, smeshers, WithSmeshers(keys[cctx.BootnodeSize:cctx.BootnodeSize+smeshers])) if err != nil { return nil, err } @@ -223,6 +240,7 @@ type Cluster struct { bootnodes int smeshers int clients []*NodeClient + certifiers []*NodeClient poets []*NodeClient bootstrappers []*NodeClient postServices []*NodeClient @@ -285,6 +303,16 @@ func (c *Cluster) persistConfigs(ctx *testcontext.Context) error { if err != nil { return fmt.Errorf("apply cfgmap %v/%v: %w", ctx.Namespace, spacemeshConfigMapName, err) } + _, err = ctx.Client.CoreV1().ConfigMaps(ctx.Namespace).Apply( + ctx, + corev1.ConfigMap(certifierConfigMapName, ctx.Namespace).WithData(map[string]string{ + attachedCertifierConfig: certifierConfig.Get(ctx.Parameters), + }), + apimetav1.ApplyOptions{FieldManager: "test"}, + ) + if err != nil { + return fmt.Errorf("apply cfgmap %v/%v: %w", ctx.Namespace, poetConfigMapName, err) + } _, err = ctx.Client.CoreV1().ConfigMaps(ctx.Namespace).Apply( ctx, corev1.ConfigMap(poetConfigMapName, ctx.Namespace).WithData(map[string]string{ @@ -454,13 +482,29 @@ func (c *Cluster) firstFreePoetId() int { } } +// AddCertifier spawns a single certifier with the first available id. +// Id is of form "certifier-N", where N ∈ [0, ∞). +func (c *Cluster) AddCertifier(cctx *testcontext.Context, privkey string) error { + if err := c.persist(cctx); err != nil { + return err + } + id := fmt.Sprintf("certifier-%d", len(c.certifiers)) + cctx.Log.Debugw("deploying poet", "id", id) + pod, err := deployCertifier(cctx, id, privkey) + if err != nil { + return err + } + c.certifiers = append(c.certifiers, pod) + return nil +} + // AddPoet spawns a single poet with the first available id. // Id is of form "poet-N", where N ∈ [0, ∞). -func (c *Cluster) AddPoet(cctx *testcontext.Context) error { +func (c *Cluster) AddPoet(cctx *testcontext.Context, flags ...DeploymentFlag) error { if err := c.persist(cctx); err != nil { return err } - flags := maps.Values(c.poetFlags) + flags = append(maps.Values(c.poetFlags), flags...) id := createPoetIdentifier(c.firstFreePoetId()) cctx.Log.Debugw("deploying poet", "id", id) @@ -656,6 +700,11 @@ func (c *Cluster) Bootnodes() int { return c.bootnodes } +// Smeshers returns number of the smeshers in the cluster. +func (c *Cluster) Smeshers() int { + return c.smeshers +} + // Total returns total number of clients. func (c *Cluster) Total() int { return len(c.clients) diff --git a/systest/cluster/nodes.go b/systest/cluster/nodes.go index 9c981f5e7a..6875499118 100644 --- a/systest/cluster/nodes.go +++ b/systest/cluster/nodes.go @@ -40,6 +40,11 @@ import ( ) var ( + certifierConfig = parameters.String( + "certifier", + "configuration for certifier service", + fastnet.CertifierConfig, + ) poetConfig = parameters.String( "poet", "configuration for poet service", @@ -109,9 +114,11 @@ func toResources(value string) (*apiv1.ResourceRequirements, error) { const ( configDir = "/etc/config/" - attachedPoetConfig = "poet.conf" - attachedSmesherConfig = "smesher.json" + attachedCertifierConfig = "certifier.yaml" + attachedPoetConfig = "poet.conf" + attachedSmesherConfig = "smesher.json" + certifierConfigMapName = "certifier" poetConfigMapName = "poet" spacemeshConfigMapName = "spacemesh" @@ -305,9 +312,64 @@ func (n *PrivNodeClient) NewStream( return stream, err } +func deployCertifierD(ctx *testcontext.Context, id, privkey string) (*NodeClient, error) { + args := []string{ + "-c" + configDir + attachedCertifierConfig, + } + + ctx.Log.Debugw("deploying certifier service pod", "id", id, "args", args, "image", ctx.CertifierImage) + + labels := nodeLabels(certifierApp, id) + + deployment := appsv1.Deployment(id, ctx.Namespace). + WithLabels(labels). + WithSpec(appsv1.DeploymentSpec(). + WithSelector(metav1.LabelSelector().WithMatchLabels(labels)). + WithReplicas(1). + WithTemplate(corev1.PodTemplateSpec(). + WithLabels(labels). + WithSpec(corev1.PodSpec(). + WithNodeSelector(ctx.NodeSelector). + WithVolumes(corev1.Volume(). + WithName("config"). + WithConfigMap(corev1.ConfigMapVolumeSource().WithName(certifierConfigMapName)), + ). + WithContainers(corev1.Container(). + WithName("certifier"). + WithImage(ctx.CertifierImage). + WithArgs(args...). + WithEnv(corev1.EnvVar().WithName("CERTIFIER_SIGNING_KEY").WithValue(privkey)). + WithPorts( + corev1.ContainerPort().WithProtocol("TCP").WithContainerPort(certifierPort), + ). + WithVolumeMounts( + corev1.VolumeMount().WithName("config").WithMountPath(configDir), + ). + WithResources(corev1.ResourceRequirements(). + WithRequests(poetResources.Get(ctx.Parameters).Requests). + WithLimits(poetResources.Get(ctx.Parameters).Limits), + ), + ), + ))) + + _, err := ctx.Client.AppsV1(). + Deployments(ctx.Namespace). + Apply(ctx, deployment, apimetav1.ApplyOptions{FieldManager: "test"}) + if err != nil { + return nil, fmt.Errorf("creating certifier: %w", err) + } + return &NodeClient{ + session: ctx, + Node: Node{ + Name: id, + }, + }, nil +} + func deployPoetD(ctx *testcontext.Context, id string, flags ...DeploymentFlag) (*NodeClient, error) { args := []string{ "-c=" + configDir + attachedPoetConfig, + "--metrics-port=" + strconv.Itoa(prometheusScrapePort), } for _, flag := range flags { args = append(args, flag.Flag()) @@ -326,6 +388,12 @@ func deployPoetD(ctx *testcontext.Context, id string, flags ...DeploymentFlag) ( WithReplicas(1). WithTemplate(corev1.PodTemplateSpec(). WithLabels(labels). + WithAnnotations( + map[string]string{ + "prometheus.io/port": strconv.Itoa(prometheusScrapePort), + "prometheus.io/scrape": "true", + }, + ). WithSpec(corev1.PodSpec(). WithNodeSelector(ctx.NodeSelector). WithVolumes(corev1.Volume(). @@ -338,6 +406,7 @@ func deployPoetD(ctx *testcontext.Context, id string, flags ...DeploymentFlag) ( WithArgs(args...). WithPorts( corev1.ContainerPort().WithName("rest").WithProtocol("TCP").WithContainerPort(poetPort), + corev1.ContainerPort().WithName("prometheus").WithContainerPort(prometheusScrapePort), ). WithVolumeMounts( corev1.VolumeMount().WithName("config").WithMountPath(configDir), @@ -407,6 +476,21 @@ func deployNodeSvc(ctx *testcontext.Context, id string) error { return nil } +func deployCertifierSvc(ctx *testcontext.Context, id string) (*apiv1.Service, error) { + ctx.Log.Debugw("deploying certifier service", "id", id) + labels := nodeLabels(certifierApp, id) + svc := corev1.Service(id, ctx.Namespace). + WithLabels(labels). + WithSpec(corev1.ServiceSpec(). + WithSelector(labels). + WithPorts( + corev1.ServicePort().WithName("rest").WithPort(certifierPort).WithProtocol("TCP"), + ), + ) + + return ctx.Client.CoreV1().Services(ctx.Namespace).Apply(ctx, svc, apimetav1.ApplyOptions{FieldManager: "test"}) +} + func deployPoetSvc(ctx *testcontext.Context, id string) (*apiv1.Service, error) { ctx.Log.Debugw("deploying poet service", "id", id) labels := nodeLabels(poetApp, id) @@ -416,6 +500,7 @@ func deployPoetSvc(ctx *testcontext.Context, id string) (*apiv1.Service, error) WithSelector(labels). WithPorts( corev1.ServicePort().WithName("rest").WithPort(poetPort).WithProtocol("TCP"), + corev1.ServicePort().WithName("prometheus").WithPort(prometheusScrapePort), ), ) @@ -442,6 +527,21 @@ func decodePoetIdentifier(id string) int { return ord } +// deployCertifier creates a certifier Deployment and exposes it via a Service. +// The key is passed to the certifier Pod. +func deployCertifier(ctx *testcontext.Context, id, privkey string) (*NodeClient, error) { + if _, err := deployCertifierSvc(ctx, id); err != nil { + return nil, fmt.Errorf("deploying certifier service: %w", err) + } + + node, err := deployCertifierD(ctx, id, privkey) + if err != nil { + return nil, err + } + + return node, nil +} + // deployPoet creates a poet Pod and exposes it via a Service. // Flags are passed to the poet Pod as arguments. func deployPoet(ctx *testcontext.Context, id string, flags ...DeploymentFlag) (*NodeClient, error) { @@ -1133,6 +1233,14 @@ func DurationFlag(flag string, d time.Duration) DeploymentFlag { return DeploymentFlag{Name: flag, Value: d.String()} } +func PoetCertifierURL(url string) DeploymentFlag { + return DeploymentFlag{Name: "--certifier-url", Value: url} +} + +func PoetCertifierPubkey(key string) DeploymentFlag { + return DeploymentFlag{Name: "--certifier-pubkey", Value: key} +} + // PoetRestListen socket pair with http api. func PoetRestListen(port int) DeploymentFlag { return DeploymentFlag{Name: "--restlisten", Value: fmt.Sprintf("0.0.0.0:%d", port)} diff --git a/systest/parameters/bignet/certifier.yaml b/systest/parameters/bignet/certifier.yaml new file mode 100644 index 0000000000..fb92d02979 --- /dev/null +++ b/systest/parameters/bignet/certifier.yaml @@ -0,0 +1,15 @@ +listen: "0.0.0.0:80" +metrics: "0.0.0.0:2112" +# Same as fastnet +post_cfg: + k1: 12 + k2: 4 + pow_difficulty: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" +init_cfg: + min_num_units: 2 + max_num_units: 4 + labels_per_unit: 128 + scrypt: + n: 2 + r: 1 + p: 1 diff --git a/systest/parameters/bignet/smesher.json b/systest/parameters/bignet/smesher.json index 1f724bb099..e64bb0fcb3 100644 --- a/systest/parameters/bignet/smesher.json +++ b/systest/parameters/bignet/smesher.json @@ -37,5 +37,10 @@ "post-k1": 26, "post-k2": 37, "post-k3": 37 + }, + "certifier": { + "client": { + "max-retries": 10 + } } } \ No newline at end of file diff --git a/systest/parameters/fastnet/certifier.yaml b/systest/parameters/fastnet/certifier.yaml new file mode 100644 index 0000000000..fb92d02979 --- /dev/null +++ b/systest/parameters/fastnet/certifier.yaml @@ -0,0 +1,15 @@ +listen: "0.0.0.0:80" +metrics: "0.0.0.0:2112" +# Same as fastnet +post_cfg: + k1: 12 + k2: 4 + pow_difficulty: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" +init_cfg: + min_num_units: 2 + max_num_units: 4 + labels_per_unit: 128 + scrypt: + n: 2 + r: 1 + p: 1 diff --git a/systest/parameters/fastnet/embed.go b/systest/parameters/fastnet/embed.go index dc8a450bb3..37cbc60ab8 100644 --- a/systest/parameters/fastnet/embed.go +++ b/systest/parameters/fastnet/embed.go @@ -9,3 +9,6 @@ var SmesherConfig string //go:embed "poet.conf" var PoetConfig string + +//go:embed "certifier.yaml" +var CertifierConfig string diff --git a/systest/parameters/fastnet/poet.conf b/systest/parameters/fastnet/poet.conf index 558fd5a254..27adb95f75 100644 --- a/systest/parameters/fastnet/poet.conf +++ b/systest/parameters/fastnet/poet.conf @@ -5,4 +5,4 @@ cycle-gap="30s" jsonlog="true" debuglog="true" -pow-difficulty=4 +pow-difficulty=4 \ No newline at end of file diff --git a/systest/parameters/fastnet/smesher.json b/systest/parameters/fastnet/smesher.json index 1e2a398736..08e290a574 100644 --- a/systest/parameters/fastnet/smesher.json +++ b/systest/parameters/fastnet/smesher.json @@ -26,10 +26,15 @@ "smeshing-opts-verifying-min-workers": 1000000 } }, + "certifier": { + "client": { + "max-retries": 10 + } + }, "logging": { "txHandler": "debug", "grpc": "debug", "sync": "debug", "fetcher": "debug" } -} \ No newline at end of file +} diff --git a/systest/parameters/longfast/certifier.yaml b/systest/parameters/longfast/certifier.yaml new file mode 100644 index 0000000000..fb92d02979 --- /dev/null +++ b/systest/parameters/longfast/certifier.yaml @@ -0,0 +1,15 @@ +listen: "0.0.0.0:80" +metrics: "0.0.0.0:2112" +# Same as fastnet +post_cfg: + k1: 12 + k2: 4 + pow_difficulty: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" +init_cfg: + min_num_units: 2 + max_num_units: 4 + labels_per_unit: 128 + scrypt: + n: 2 + r: 1 + p: 1 diff --git a/systest/parameters/longfast/smesher.json b/systest/parameters/longfast/smesher.json index 0d48e14016..ed4bca5dbe 100644 --- a/systest/parameters/longfast/smesher.json +++ b/systest/parameters/longfast/smesher.json @@ -33,5 +33,10 @@ "smeshing-verifying-opts": { "smeshing-opts-verifying-min-workers": 1000000 } + }, + "certifier": { + "client": { + "max-retries": 10 + } } -} \ No newline at end of file +} diff --git a/systest/testcontext/context.go b/systest/testcontext/context.go index 4d7e047975..7abda3d478 100644 --- a/systest/testcontext/context.go +++ b/systest/testcontext/context.go @@ -72,6 +72,11 @@ var ( "bootstrapper image", "spacemeshos/spacemesh-dev-bs:2beaf443f", ) + certifierImage = parameters.String( + "certifier-image", + "certifier service image", + "spacemeshos/certifier-service:latest", + ) poetImage = parameters.String( "poet-image", "poet server image", @@ -167,6 +172,7 @@ type Context struct { Namespace string Image string BootstrapperImage string + CertifierImage string PoetImage string PostServiceImage string PostInitImage string @@ -352,6 +358,7 @@ func New(t *testing.T, opts ...Opt) *Context { BootstrapperSize: bsSize.Get(p), Image: imageFlag.Get(p), BootstrapperImage: bsImage.Get(p), + CertifierImage: certifierImage.Get(p), PoetImage: poetImage.Get(p), PostServiceImage: postServiceImage.Get(p), PostInitImage: postInitImage.Get(p), diff --git a/systest/tests/distributed_post_verification_test.go b/systest/tests/distributed_post_verification_test.go index 615a4cc3b0..4ec6bd5cf8 100644 --- a/systest/tests/distributed_post_verification_test.go +++ b/systest/tests/distributed_post_verification_test.go @@ -34,6 +34,7 @@ import ( "github.com/spacemeshos/go-spacemesh/signing" "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sql/localsql" + "github.com/spacemeshos/go-spacemesh/sql/localsql/nipost" "github.com/spacemeshos/go-spacemesh/systest/cluster" "github.com/spacemeshos/go-spacemesh/systest/testcontext" "github.com/spacemeshos/go-spacemesh/timesync" @@ -67,9 +68,6 @@ func TestPostMalfeasanceProof(t *testing.T) { cfg.POET.RequestTimeout = time.Minute cfg.POET.MaxRequestRetries = 10 - cfg.PoetServers = []types.PoetServer{ - {Address: cluster.MakePoetGlobalEndpoint(ctx.Namespace, 0)}, - } var bootnodes []*cluster.NodeClient for i := 0; i < cl.Bootnodes(); i++ { @@ -158,14 +156,27 @@ func TestPostMalfeasanceProof(t *testing.T) { require.NoError(t, grpcPrivateServer.Start()) t.Cleanup(func() { assert.NoError(t, grpcPrivateServer.Close()) }) + db := sql.InMemory() + localDb := localsql.InMemory() + certClient := activation.NewCertifierClient(db, localDb, logger.Named("certifier")) + certifier := activation.NewCertifier(localDb, logger, certClient) + poetClient, err := activation.NewPoetClient( + activation.NewPoetDb(db, log.NewNop()), + types.PoetServer{ + Address: cluster.MakePoetGlobalEndpoint(ctx.Namespace, 0), + }, cfg.POET, + logger, + activation.WithCertifier(certifier), + ) + require.NoError(t, err) + nipostBuilder, err := activation.NewNIPostBuilder( - localsql.InMemory(), - activation.NewPoetDb(sql.InMemory(), log.NewNop()), + localDb, grpcPostService, - cfg.PoetServers, logger.Named("nipostBuilder"), cfg.POET, clock, + activation.WithPoetClients(poetClient), ) require.NoError(t, err) @@ -182,6 +193,17 @@ func TestPostMalfeasanceProof(t *testing.T) { post, postInfo, err := client.Proof(ctx, shared.ZeroChallenge) require.NoError(t, err) + err = nipost.AddPost(localDb, signer.NodeID(), nipost.Post{ + Nonce: post.Nonce, + Indices: post.Indices, + Pow: post.Pow, + Challenge: shared.ZeroChallenge, + NumUnits: postInfo.NumUnits, + CommitmentATX: postInfo.CommitmentATX, + VRFNonce: *postInfo.Nonce, + }) + require.NoError(t, err) + challenge = &wire.NIPostChallengeV1{ PrevATXID: types.EmptyATXID, PublishEpoch: 2, @@ -195,6 +217,7 @@ func TestPostMalfeasanceProof(t *testing.T) { } break } + nipost, err := nipostBuilder.BuildNIPost(ctx, signer, challenge.PublishEpoch, challenge.Hash()) require.NoError(t, err) diff --git a/systest/tests/metrics.go b/systest/tests/metrics.go new file mode 100644 index 0000000000..2d1ed131d1 --- /dev/null +++ b/systest/tests/metrics.go @@ -0,0 +1,65 @@ +package tests + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/prometheus/common/expfmt" +) + +var errMetricNotFound = errors.New("metric not found") + +func fetchCounterMetric( + ctx context.Context, + url string, + metricName string, + labelFilters map[string]string, +) (float64, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, fmt.Errorf("failed to create HTTP request: %v", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, fmt.Errorf("failed to fetch metrics: %v", err) + } + defer resp.Body.Close() + + parser := expfmt.TextParser{} + metricFamilies, err := parser.TextToMetricFamilies(resp.Body) + if err != nil { + return 0, fmt.Errorf("failed to parse metric families: %v", err) + } + + metricFamily, ok := metricFamilies[metricName] + if !ok { + return 0, fmt.Errorf("metric not found: %s", metricName) + } + + for _, metric := range metricFamily.Metric { + if metric.Counter != nil { + // Check if the metric has the specified labels + labels := make(map[string]string) + for _, lp := range metric.Label { + labels[lp.GetName()] = lp.GetValue() + } + + matches := true + for key, value := range labelFilters { + if labels[key] != value { + matches = false + break + } + } + + if matches { + return metric.GetCounter().GetValue(), nil + } + } + } + + return 0, errMetricNotFound +} diff --git a/systest/tests/poets_test.go b/systest/tests/poets_test.go index 67f80dca90..b35e64c045 100644 --- a/systest/tests/poets_test.go +++ b/systest/tests/poets_test.go @@ -1,6 +1,8 @@ package tests import ( + "crypto/ed25519" + "encoding/base64" "encoding/hex" "fmt" "math" @@ -121,6 +123,7 @@ func testPoetDies(t *testing.T, tctx *testcontext.Context, cl *cluster.Cluster) } func TestNodesUsingDifferentPoets(t *testing.T) { + t.Parallel() tctx := testcontext.New(t, testcontext.Labels("sanity")) if tctx.PoetSize < 2 { t.Skip("Skipping test for using different poets - test configured with less then 2 poets") @@ -203,3 +206,70 @@ func TestNodesUsingDifferentPoets(t *testing.T) { ) } } + +// Test verifying that nodes can register in both poets +// - supporting PoW only +// - supporting certificates +// TODO: When PoW support is removed, convert this test to verify only the cert path. +// https://github.com/spacemeshos/go-spacemesh/issues/5212 +func TestRegisteringInPoetWithPowAndCert(t *testing.T) { + t.Parallel() + tctx := testcontext.New(t, testcontext.Labels("sanity")) + tctx.PoetSize = 2 + + cl := cluster.New(tctx, cluster.WithKeys(10)) + require.NoError(t, cl.AddBootnodes(tctx, 2)) + require.NoError(t, cl.AddBootstrappers(tctx)) + + pubkey, privkey, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + require.NoError(t, cl.AddCertifier(tctx, base64.StdEncoding.EncodeToString(privkey.Seed()))) + // First poet supports PoW only (legacy) + require.NoError(t, cl.AddPoet(tctx)) + // Second poet supports certs + require.NoError( + t, + cl.AddPoet( + tctx, + cluster.PoetCertifierURL("http://certifier-0"), + cluster.PoetCertifierPubkey(base64.StdEncoding.EncodeToString(pubkey)), + ), + ) + require.NoError(t, cl.AddSmeshers(tctx, tctx.ClusterSize-2)) + require.NoError(t, cl.WaitAll(tctx)) + + epoch := 2 + layersPerEpoch := testcontext.LayersPerEpoch.Get(tctx.Parameters) + last := uint32(layersPerEpoch * epoch) + tctx.Log.Debugw("waiting for epoch", "epoch", epoch, "layer", last) + + eg, ctx := errgroup.WithContext(tctx) + for i := 0; i < cl.Total(); i++ { + client := cl.Client(i) + tctx.Log.Debugw("watching", "client", client.Name) + watchProposals(ctx, eg, client, tctx.Log.Desugar(), func(proposal *pb.Proposal) (bool, error) { + return proposal.Layer.Number < last, nil + }) + } + + require.NoError(t, eg.Wait()) + + // Check that smeshers are registered in both poets + valid := map[string]string{"result": "valid"} + invalid := map[string]string{"result": "invalid"} + + metricsEndpoint := cluster.MakePoetMetricsEndpoint(tctx.Namespace, 0) + powRegs, err := fetchCounterMetric(tctx, metricsEndpoint, "poet_registration_with_pow_total", valid) + require.NoError(t, err) + require.GreaterOrEqual(t, powRegs, float64(cl.Smeshers()*epoch)) + powRegsInvalid, err := fetchCounterMetric(tctx, metricsEndpoint, "poet_registration_with_pow_total", invalid) + require.ErrorIs(t, err, errMetricNotFound, "metric for invalid PoW registrations value: %v", powRegsInvalid) + + metricsEndpoint = cluster.MakePoetMetricsEndpoint(tctx.Namespace, 1) + certRegs, err := fetchCounterMetric(tctx, metricsEndpoint, "poet_registration_with_cert_total", valid) + require.NoError(t, err) + require.GreaterOrEqual(t, certRegs, float64(cl.Smeshers()*epoch)) + + certRegsInvalid, err := fetchCounterMetric(tctx, metricsEndpoint, "poet_registration_with_cert_total", invalid) + require.ErrorIs(t, err, errMetricNotFound, "metric for invalid cert registrations value: %v", certRegsInvalid) +}