diff --git a/libs/utils/close.go b/libs/utils/close.go new file mode 100644 index 0000000000..c9a2ddaa30 --- /dev/null +++ b/libs/utils/close.go @@ -0,0 +1,15 @@ +package utils + +import ( + "io" + + logging "github.com/ipfs/go-log/v2" +) + +// CloseAndLog closes the closer and logs any error that occurs. The function is handy wrapping +// to group closing and logging in one call for defer statements. +func CloseAndLog(log logging.StandardLogger, name string, closer io.Closer) { + if err := closer.Close(); err != nil { + log.Warnf("closing %s: %s", name, err) + } +} diff --git a/share/getters/utils.go b/libs/utils/ctx.go similarity index 53% rename from share/getters/utils.go rename to libs/utils/ctx.go index 2260183b4f..5a13232a16 100644 --- a/share/getters/utils.go +++ b/libs/utils/ctx.go @@ -1,24 +1,22 @@ -package getters +package utils import ( "context" - "errors" "time" - - logging "github.com/ipfs/go-log/v2" - "go.opentelemetry.io/otel" ) -var ( - tracer = otel.Tracer("share/getters") - log = logging.Logger("share/getters") +// ResetContextOnError returns a fresh context if the given context has an error. +func ResetContextOnError(ctx context.Context) context.Context { + if ctx.Err() != nil { + ctx = context.Background() + } - errOperationNotSupported = errors.New("operation is not supported") -) + return ctx +} -// ctxWithSplitTimeout will split timeout stored in context by splitFactor and return the result if +// CtxWithSplitTimeout will split timeout stored in context by splitFactor and return the result if // it is greater than minTimeout. minTimeout == 0 will be ignored, splitFactor <= 0 will be ignored -func ctxWithSplitTimeout( +func CtxWithSplitTimeout( ctx context.Context, splitFactor int, minTimeout time.Duration, @@ -42,16 +40,3 @@ func ctxWithSplitTimeout( } return context.WithTimeout(ctx, splitTimeout) } - -// ErrorContains reports whether any error in err's tree matches any error in targets tree. -func ErrorContains(err, target error) bool { - if errors.Is(err, target) || target == nil { - return true - } - - target = errors.Unwrap(target) - if target == nil { - return false - } - return ErrorContains(err, target) -} diff --git a/share/getters/utils_test.go b/libs/utils/ctx_test.go similarity index 62% rename from share/getters/utils_test.go rename to libs/utils/ctx_test.go index ce94d3ac04..c1fccfa48a 100644 --- a/share/getters/utils_test.go +++ b/libs/utils/ctx_test.go @@ -1,119 +1,13 @@ -package getters +package utils import ( "context" - "errors" - "fmt" "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func Test_ErrorContains(t *testing.T) { - err1 := errors.New("1") - err2 := errors.New("2") - - w1 := func(err error) error { - return fmt.Errorf("wrap1: %w", err) - } - w2 := func(err error) error { - return fmt.Errorf("wrap1: %w", err) - } - - type args struct { - err error - target error - } - tests := []struct { - name string - args args - want bool - }{ - { - "nil err", - args{ - err: nil, - target: err1, - }, - false, - }, - { - "nil target", - args{ - err: err1, - target: nil, - }, - true, - }, - { - "errors.Is true", - args{ - err: w1(err1), - target: err1, - }, - true, - }, - { - "errors.Is false", - args{ - err: w1(err1), - target: err2, - }, - false, - }, - { - "same wrap but different base error", - args{ - err: w1(err1), - target: w1(err2), - }, - false, - }, - { - "both wrapped true", - args{ - err: w1(err1), - target: w2(err1), - }, - true, - }, - { - "both wrapped false", - args{ - err: w1(err1), - target: w2(err2), - }, - false, - }, - { - "multierr first in slice", - args{ - err: errors.Join(w1(err1), w2(err2)), - target: w2(err1), - }, - true, - }, - { - "multierr second in slice", - args{ - err: errors.Join(w1(err1), w2(err2)), - target: w1(err2), - }, - true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, - tt.want, - ErrorContains(tt.args.err, tt.args.target), - "ErrorContains(%v, %v)", tt.args.err, tt.args.target) - }) - } -} - func Test_ctxWithSplitTimeout(t *testing.T) { type args struct { ctxTimeout time.Duration @@ -216,7 +110,7 @@ func Test_ctxWithSplitTimeout(t *testing.T) { ctx, cancel = context.WithTimeout(ctx, tt.args.ctxTimeout) } t.Cleanup(cancel) - got, _ := ctxWithSplitTimeout(ctx, sf, tt.args.minTimeout) + got, _ := CtxWithSplitTimeout(ctx, sf, tt.args.minTimeout) dl, ok := got.Deadline() // in case no deadline is found in ctx or not expected to be found, check both cases apply at the // same time diff --git a/libs/utils/resetctx.go b/libs/utils/resetctx.go deleted file mode 100644 index a108cc27b4..0000000000 --- a/libs/utils/resetctx.go +++ /dev/null @@ -1,14 +0,0 @@ -package utils - -import ( - "context" -) - -// ResetContextOnError returns a fresh context if the given context has an error. -func ResetContextOnError(ctx context.Context) context.Context { - if ctx.Err() != nil { - ctx = context.Background() - } - - return ctx -} diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index 9c3bf7e646..36e8a77e81 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -11,6 +11,7 @@ import ( "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrex_getter" ) func newShareModule(getter share.Getter, avail share.Availability) Module { @@ -42,7 +43,7 @@ func ensureEmptyEDSInBS(ctx context.Context, bServ blockservice.BlockService) er } func lightGetter( - shrexGetter *getters.ShrexGetter, + shrexGetter *shrex_getter.Getter, ipldGetter *getters.IPLDGetter, cfg Config, ) share.Getter { @@ -54,13 +55,13 @@ func lightGetter( return getters.NewCascadeGetter(cascade) } -// ShrexGetter is added to bridge nodes for the case that a shard is removed +// Getter is added to bridge nodes for the case that a shard is removed // after detected shard corruption. This ensures the block is fetched and stored // by shrex the next time the data is retrieved (meaning shard recovery is // manual after corruption is detected). func bridgeGetter( storeGetter *getters.StoreGetter, - shrexGetter *getters.ShrexGetter, + shrexGetter *shrex_getter.Getter, cfg Config, ) share.Getter { var cascade []share.Getter @@ -73,7 +74,7 @@ func bridgeGetter( func fullGetter( storeGetter *getters.StoreGetter, - shrexGetter *getters.ShrexGetter, + shrexGetter *shrex_getter.Getter, ipldGetter *getters.IPLDGetter, cfg Config, ) share.Getter { diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index 5d0dbbd096..7e8d7bf3d8 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -19,6 +19,7 @@ import ( "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrex_getter" ) func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option { @@ -92,8 +93,8 @@ func shrexComponents(tp node.Type, cfg *Config) fx.Option { edsClient *shrexeds.Client, ndClient *shrexnd.Client, managers map[string]*peers.Manager, - ) *getters.ShrexGetter { - return getters.NewShrexGetter( + ) *shrex_getter.Getter { + return shrex_getter.NewGetter( edsClient, ndClient, managers[fullNodesTag], @@ -101,10 +102,10 @@ func shrexComponents(tp node.Type, cfg *Config) fx.Option { lightprune.Window, ) }, - fx.OnStart(func(ctx context.Context, getter *getters.ShrexGetter) error { + fx.OnStart(func(ctx context.Context, getter *shrex_getter.Getter) error { return getter.Start(ctx) }), - fx.OnStop(func(ctx context.Context, getter *getters.ShrexGetter) error { + fx.OnStop(func(ctx context.Context, getter *shrex_getter.Getter) error { return getter.Stop(ctx) }), )), diff --git a/nodebuilder/share/opts.go b/nodebuilder/share/opts.go index 9c122b7b0f..2856f30e11 100644 --- a/nodebuilder/share/opts.go +++ b/nodebuilder/share/opts.go @@ -4,11 +4,11 @@ import ( "errors" "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/getters" disc "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrex_getter" ) // WithPeerManagerMetrics is a utility function to turn on peer manager metrics and that is @@ -49,7 +49,7 @@ func WithShrexServerMetrics(edsServer *shrexeds.Server, ndServer *shrexnd.Server return ndServer.WithMetrics() } -func WithShrexGetterMetrics(sg *getters.ShrexGetter) error { +func WithShrexGetterMetrics(sg *shrex_getter.Getter) error { return sg.WithMetrics() } diff --git a/nodebuilder/tests/nd_test.go b/nodebuilder/tests/nd_test.go index 338aa6d0c1..a76f595980 100644 --- a/nodebuilder/tests/nd_test.go +++ b/nodebuilder/tests/nd_test.go @@ -20,6 +20,7 @@ import ( "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrex_getter" ) func TestShrexNDFromLights(t *testing.T) { @@ -200,7 +201,7 @@ func replaceShareGetter() fx.Option { host host.Host, store *eds.Store, storeGetter *getters.StoreGetter, - shrexGetter *getters.ShrexGetter, + shrexGetter *shrex_getter.Getter, network p2p.Network, ) share.Getter { cascade := make([]share.Getter, 0, 2) diff --git a/nodebuilder/tests/prune_test.go b/nodebuilder/tests/prune_test.go index 15b939f465..0496a16321 100644 --- a/nodebuilder/tests/prune_test.go +++ b/nodebuilder/tests/prune_test.go @@ -19,10 +19,10 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" "github.com/celestiaorg/celestia-node/pruner" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/getters" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrex_getter" ) // TestArchivalBlobSync tests whether a LN is able to sync historical blobs from @@ -71,15 +71,15 @@ func TestArchivalBlobSync(t *testing.T) { edsClient *shrexeds.Client, ndClient *shrexnd.Client, managers map[string]*peers.Manager, - ) *getters.ShrexGetter { - return getters.NewShrexGetter( + ) *shrex_getter.Getter { + return shrex_getter.NewGetter( edsClient, ndClient, managers["full"], managers["archival"], testAvailWindow, ) - }, new(getters.ShrexGetter)), + }, new(shrex_getter.Getter)), ) // stop the archival BN to force LN to have to discover diff --git a/share/getters/cascade.go b/share/getters/cascade.go index 3875127580..574b70153c 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -4,6 +4,8 @@ import ( "context" "errors" + logging "github.com/ipfs/go-log/v2" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -15,6 +17,11 @@ import ( "github.com/celestiaorg/celestia-node/share/eds/byzantine" ) +var ( + tracer = otel.Tracer("share/getters") + log = logging.Logger("share/getters") +) + var _ share.Getter = (*CascadeGetter)(nil) // CascadeGetter implements custom share.Getter that composes multiple Getter implementations in @@ -125,14 +132,14 @@ func cascadeGetters[V any]( // we split the timeout between left getters // once async cascadegetter is implemented, we can remove this - getCtx, cancel := ctxWithSplitTimeout(ctx, len(getters)-i, 0) + getCtx, cancel := utils.CtxWithSplitTimeout(ctx, len(getters)-i, 0) val, getErr := get(getCtx, getter) cancel() if getErr == nil { return val, nil } - if errors.Is(getErr, errOperationNotSupported) { + if errors.Is(getErr, ErrOperationNotSupported) { continue } diff --git a/share/new_eds/nd.go b/share/new_eds/nd.go index 45b700f482..36de7a0234 100644 --- a/share/new_eds/nd.go +++ b/share/new_eds/nd.go @@ -8,20 +8,20 @@ import ( "github.com/celestiaorg/celestia-node/share/shwap" ) -// NamespacedData extracts shares for a specific namespace from an EDS, considering +// NamespaceData extracts shares for a specific namespace from an EDS, considering // each row independently. It uses root to determine which rows to extract data from, // avoiding the need to recalculate the row roots for each row. -func NamespacedData( +func NamespaceData( ctx context.Context, eds Accessor, namespace share.Namespace, -) (shwap.NamespacedData, error) { +) (shwap.NamespaceData, error) { roots, err := eds.AxisRoots(ctx) if err != nil { return nil, fmt.Errorf("failed to get AxisRoots: %w", err) } rowIdxs := share.RowsWithNamespace(roots, namespace) - rows := make(shwap.NamespacedData, len(rowIdxs)) + rows := make(shwap.NamespaceData, len(rowIdxs)) for i, idx := range rowIdxs { rows[i], err = eds.RowNamespaceData(ctx, namespace, idx) if err != nil { diff --git a/share/new_eds/nd_test.go b/share/new_eds/nd_test.go index 5d3765e03f..f3fc2e4bb4 100644 --- a/share/new_eds/nd_test.go +++ b/share/new_eds/nd_test.go @@ -11,7 +11,7 @@ import ( "github.com/celestiaorg/celestia-node/share/sharetest" ) -func TestNamespacedData(t *testing.T) { +func TestNamespaceData(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) @@ -21,7 +21,7 @@ func TestNamespacedData(t *testing.T) { for amount := 1; amount < sharesAmount; amount++ { eds, root := edstest.RandEDSWithNamespace(t, namespace, amount, odsSize) rsmt2d := &Rsmt2D{ExtendedDataSquare: eds} - nd, err := NamespacedData(ctx, rsmt2d, namespace) + nd, err := NamespaceData(ctx, rsmt2d, namespace) require.NoError(t, err) require.True(t, len(nd) > 0) require.Len(t, nd.Flatten(), amount) diff --git a/share/p2p/shrexeds/pb/extended_data_square.pb.go b/share/p2p/shrexeds/pb/extended_data_square.pb.go deleted file mode 100644 index ed1a96ae3b..0000000000 --- a/share/p2p/shrexeds/pb/extended_data_square.pb.go +++ /dev/null @@ -1,509 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: share/p2p/shrexeds/pb/extended_data_square.proto - -package extended_data_square - -import ( - fmt "fmt" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -type Status int32 - -const ( - Status_INVALID Status = 0 - Status_OK Status = 1 - Status_NOT_FOUND Status = 2 - Status_INTERNAL Status = 3 -) - -var Status_name = map[int32]string{ - 0: "INVALID", - 1: "OK", - 2: "NOT_FOUND", - 3: "INTERNAL", -} - -var Status_value = map[string]int32{ - "INVALID": 0, - "OK": 1, - "NOT_FOUND": 2, - "INTERNAL": 3, -} - -func (x Status) String() string { - return proto.EnumName(Status_name, int32(x)) -} - -func (Status) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_49d42aa96098056e, []int{0} -} - -type EDSRequest struct { - Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` -} - -func (m *EDSRequest) Reset() { *m = EDSRequest{} } -func (m *EDSRequest) String() string { return proto.CompactTextString(m) } -func (*EDSRequest) ProtoMessage() {} -func (*EDSRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_49d42aa96098056e, []int{0} -} -func (m *EDSRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *EDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_EDSRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *EDSRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_EDSRequest.Merge(m, src) -} -func (m *EDSRequest) XXX_Size() int { - return m.Size() -} -func (m *EDSRequest) XXX_DiscardUnknown() { - xxx_messageInfo_EDSRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_EDSRequest proto.InternalMessageInfo - -func (m *EDSRequest) GetHash() []byte { - if m != nil { - return m.Hash - } - return nil -} - -type EDSResponse struct { - Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=Status" json:"status,omitempty"` -} - -func (m *EDSResponse) Reset() { *m = EDSResponse{} } -func (m *EDSResponse) String() string { return proto.CompactTextString(m) } -func (*EDSResponse) ProtoMessage() {} -func (*EDSResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_49d42aa96098056e, []int{1} -} -func (m *EDSResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *EDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_EDSResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *EDSResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_EDSResponse.Merge(m, src) -} -func (m *EDSResponse) XXX_Size() int { - return m.Size() -} -func (m *EDSResponse) XXX_DiscardUnknown() { - xxx_messageInfo_EDSResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_EDSResponse proto.InternalMessageInfo - -func (m *EDSResponse) GetStatus() Status { - if m != nil { - return m.Status - } - return Status_INVALID -} - -func init() { - proto.RegisterEnum("Status", Status_name, Status_value) - proto.RegisterType((*EDSRequest)(nil), "EDSRequest") - proto.RegisterType((*EDSResponse)(nil), "EDSResponse") -} - -func init() { - proto.RegisterFile("share/p2p/shrexeds/pb/extended_data_square.proto", fileDescriptor_49d42aa96098056e) -} - -var fileDescriptor_49d42aa96098056e = []byte{ - // 227 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x32, 0x28, 0xce, 0x48, 0x2c, - 0x4a, 0xd5, 0x2f, 0x30, 0x2a, 0xd0, 0x2f, 0xce, 0x28, 0x4a, 0xad, 0x48, 0x4d, 0x29, 0xd6, 0x2f, - 0x48, 0xd2, 0x4f, 0xad, 0x28, 0x49, 0xcd, 0x4b, 0x49, 0x4d, 0x89, 0x4f, 0x49, 0x2c, 0x49, 0x8c, - 0x2f, 0x2e, 0x2c, 0x4d, 0x2c, 0x4a, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x57, 0x52, 0xe0, 0xe2, - 0x72, 0x75, 0x09, 0x0e, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x12, 0xe2, 0x62, 0xc9, 0x48, - 0x2c, 0xce, 0x90, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x02, 0xb3, 0x95, 0xf4, 0xb8, 0xb8, 0xc1, - 0x2a, 0x8a, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0xe4, 0xb9, 0xd8, 0x8a, 0x4b, 0x12, 0x4b, 0x4a, - 0x8b, 0xc1, 0x8a, 0xf8, 0x8c, 0xd8, 0xf5, 0x82, 0xc1, 0xdc, 0x20, 0xa8, 0xb0, 0x96, 0x15, 0x17, - 0x1b, 0x44, 0x44, 0x88, 0x9b, 0x8b, 0xdd, 0xd3, 0x2f, 0xcc, 0xd1, 0xc7, 0xd3, 0x45, 0x80, 0x41, - 0x88, 0x8d, 0x8b, 0xc9, 0xdf, 0x5b, 0x80, 0x51, 0x88, 0x97, 0x8b, 0xd3, 0xcf, 0x3f, 0x24, 0xde, - 0xcd, 0x3f, 0xd4, 0xcf, 0x45, 0x80, 0x49, 0x88, 0x87, 0x8b, 0xc3, 0xd3, 0x2f, 0xc4, 0x35, 0xc8, - 0xcf, 0xd1, 0x47, 0x80, 0xd9, 0x49, 0xe2, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4, 0x18, 0x1f, - 0x3c, 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, 0x1b, 0x8f, 0xe5, 0x18, - 0x92, 0xd8, 0xc0, 0xce, 0x35, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0x7b, 0x1d, 0xd4, 0xa7, 0xe2, - 0x00, 0x00, 0x00, -} - -func (m *EDSRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *EDSRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *EDSRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Hash) > 0 { - i -= len(m.Hash) - copy(dAtA[i:], m.Hash) - i = encodeVarintExtendedDataSquare(dAtA, i, uint64(len(m.Hash))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *EDSResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *EDSResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *EDSResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Status != 0 { - i = encodeVarintExtendedDataSquare(dAtA, i, uint64(m.Status)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func encodeVarintExtendedDataSquare(dAtA []byte, offset int, v uint64) int { - offset -= sovExtendedDataSquare(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *EDSRequest) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Hash) - if l > 0 { - n += 1 + l + sovExtendedDataSquare(uint64(l)) - } - return n -} - -func (m *EDSResponse) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Status != 0 { - n += 1 + sovExtendedDataSquare(uint64(m.Status)) - } - return n -} - -func sovExtendedDataSquare(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozExtendedDataSquare(x uint64) (n int) { - return sovExtendedDataSquare(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *EDSRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowExtendedDataSquare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: EDSRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: EDSRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowExtendedDataSquare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthExtendedDataSquare - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthExtendedDataSquare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Hash = append(m.Hash[:0], dAtA[iNdEx:postIndex]...) - if m.Hash == nil { - m.Hash = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipExtendedDataSquare(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthExtendedDataSquare - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *EDSResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowExtendedDataSquare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: EDSResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: EDSResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) - } - m.Status = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowExtendedDataSquare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Status |= Status(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipExtendedDataSquare(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthExtendedDataSquare - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipExtendedDataSquare(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - depth := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowExtendedDataSquare - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowExtendedDataSquare - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - case 1: - iNdEx += 8 - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowExtendedDataSquare - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthExtendedDataSquare - } - iNdEx += length - case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupExtendedDataSquare - } - depth-- - case 5: - iNdEx += 4 - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - if iNdEx < 0 { - return 0, ErrInvalidLengthExtendedDataSquare - } - if depth == 0 { - return iNdEx, nil - } - } - return 0, io.ErrUnexpectedEOF -} - -var ( - ErrInvalidLengthExtendedDataSquare = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowExtendedDataSquare = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupExtendedDataSquare = fmt.Errorf("proto: unexpected end of group") -) diff --git a/share/p2p/shrexeds/server.go b/share/p2p/shrexeds/server.go deleted file mode 100644 index 15d67d2111..0000000000 --- a/share/p2p/shrexeds/server.go +++ /dev/null @@ -1,202 +0,0 @@ -package shrexeds - -import ( - "context" - "errors" - "fmt" - "io" - "time" - - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/protocol" - "go.uber.org/zap" - - "github.com/celestiaorg/go-libp2p-messenger/serde" - - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/p2p" - p2p_pb "github.com/celestiaorg/celestia-node/share/p2p/shrexeds/pb" -) - -// Server is responsible for serving ODSs for blocksync over the ShrEx/EDS protocol. -type Server struct { - ctx context.Context - cancel context.CancelFunc - - host host.Host - protocolID protocol.ID - - store *eds.Store - - params *Parameters - middleware *p2p.Middleware - metrics *p2p.Metrics -} - -// NewServer creates a new ShrEx/EDS server. -func NewServer(params *Parameters, host host.Host, store *eds.Store) (*Server, error) { - if err := params.Validate(); err != nil { - return nil, fmt.Errorf("shrex-eds: server creation failed: %w", err) - } - - return &Server{ - host: host, - store: store, - protocolID: p2p.ProtocolID(params.NetworkID(), protocolString), - params: params, - middleware: p2p.NewMiddleware(params.ConcurrencyLimit), - }, nil -} - -func (s *Server) Start(context.Context) error { - s.ctx, s.cancel = context.WithCancel(context.Background()) - handler := s.handleStream - withRateLimit := s.middleware.RateLimitHandler(handler) - withRecovery := p2p.RecoveryMiddleware(withRateLimit) - s.host.SetStreamHandler(s.protocolID, withRecovery) - return nil -} - -func (s *Server) Stop(context.Context) error { - defer s.cancel() - s.host.RemoveStreamHandler(s.protocolID) - return nil -} - -func (s *Server) observeRateLimitedRequests() { - numRateLimited := s.middleware.DrainCounter() - if numRateLimited > 0 { - s.metrics.ObserveRequests(context.Background(), numRateLimited, p2p.StatusRateLimited) - } -} - -func (s *Server) handleStream(stream network.Stream) { - logger := log.With("peer", stream.Conn().RemotePeer().String()) - logger.Debug("server: handling eds request") - - s.observeRateLimitedRequests() - - // read request from stream to get the dataHash for store lookup - req, err := s.readRequest(logger, stream) - if err != nil { - logger.Warnw("server: reading request from stream", "err", err) - stream.Reset() //nolint:errcheck - return - } - - // ensure the requested dataHash is a valid root - hash := share.DataHash(req.Hash) - err = hash.Validate() - if err != nil { - logger.Warnw("server: invalid request", "err", err) - stream.Reset() //nolint:errcheck - return - } - logger = logger.With("hash", hash.String()) - - ctx, cancel := context.WithTimeout(s.ctx, s.params.HandleRequestTimeout) - defer cancel() - - // determine whether the EDS is available in our store - // we do not close the reader, so that other requests will not need to re-open the file. - // closing is handled by the LRU cache. - edsReader, err := s.store.GetCAR(ctx, hash) - var status p2p_pb.Status - switch { - case err == nil: - defer func() { - if err := edsReader.Close(); err != nil { - log.Warnw("closing car reader", "err", err) - } - }() - status = p2p_pb.Status_OK - case errors.Is(err, eds.ErrNotFound): - logger.Warnw("server: request hash not found") - s.metrics.ObserveRequests(ctx, 1, p2p.StatusNotFound) - status = p2p_pb.Status_NOT_FOUND - case err != nil: - logger.Errorw("server: get CAR", "err", err) - status = p2p_pb.Status_INTERNAL - } - - // inform the client of our status - err = s.writeStatus(logger, status, stream) - if err != nil { - logger.Warnw("server: writing status to stream", "err", err) - stream.Reset() //nolint:errcheck - return - } - // if we cannot serve the EDS, we are already done - if status != p2p_pb.Status_OK { - err = stream.Close() - if err != nil { - logger.Debugw("server: closing stream", "err", err) - } - return - } - - // start streaming the ODS to the client - err = s.writeODS(logger, edsReader, stream) - if err != nil { - logger.Warnw("server: writing ods to stream", "err", err) - stream.Reset() //nolint:errcheck - return - } - - s.metrics.ObserveRequests(ctx, 1, p2p.StatusSuccess) - err = stream.Close() - if err != nil { - logger.Debugw("server: closing stream", "err", err) - } -} - -func (s *Server) readRequest(logger *zap.SugaredLogger, stream network.Stream) (*p2p_pb.EDSRequest, error) { - err := stream.SetReadDeadline(time.Now().Add(s.params.ServerReadTimeout)) - if err != nil { - logger.Debugw("server: set read deadline", "err", err) - } - - req := new(p2p_pb.EDSRequest) - _, err = serde.Read(stream, req) - if err != nil { - return nil, err - } - err = stream.CloseRead() - if err != nil { - logger.Debugw("server: closing read", "err", err) - } - - return req, nil -} - -func (s *Server) writeStatus(logger *zap.SugaredLogger, status p2p_pb.Status, stream network.Stream) error { - err := stream.SetWriteDeadline(time.Now().Add(s.params.ServerWriteTimeout)) - if err != nil { - logger.Debugw("server: set write deadline", "err", err) - } - - resp := &p2p_pb.EDSResponse{Status: status} - _, err = serde.Write(stream, resp) - return err -} - -func (s *Server) writeODS(logger *zap.SugaredLogger, edsReader io.Reader, stream network.Stream) error { - err := stream.SetWriteDeadline(time.Now().Add(s.params.ServerWriteTimeout)) - if err != nil { - logger.Debugw("server: set read deadline", "err", err) - } - - odsReader, err := eds.ODSReader(edsReader) - if err != nil { - return fmt.Errorf("creating ODS reader: %w", err) - } - buf := make([]byte, s.params.BufferSize) - _, err = io.CopyBuffer(stream, odsReader, buf) - if err != nil { - return fmt.Errorf("writing ODS bytes: %w", err) - } - - return nil -} diff --git a/share/p2p/shrexnd/pb/share.pb.go b/share/p2p/shrexnd/pb/share.pb.go deleted file mode 100644 index 7e3c11416f..0000000000 --- a/share/p2p/shrexnd/pb/share.pb.go +++ /dev/null @@ -1,801 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: share/p2p/shrexnd/pb/share.proto - -package share_p2p_shrex_nd - -import ( - fmt "fmt" - pb "github.com/celestiaorg/nmt/pb" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -type StatusCode int32 - -const ( - StatusCode_INVALID StatusCode = 0 - StatusCode_OK StatusCode = 1 - StatusCode_NOT_FOUND StatusCode = 2 - StatusCode_INTERNAL StatusCode = 3 -) - -var StatusCode_name = map[int32]string{ - 0: "INVALID", - 1: "OK", - 2: "NOT_FOUND", - 3: "INTERNAL", -} - -var StatusCode_value = map[string]int32{ - "INVALID": 0, - "OK": 1, - "NOT_FOUND": 2, - "INTERNAL": 3, -} - -func (x StatusCode) String() string { - return proto.EnumName(StatusCode_name, int32(x)) -} - -func (StatusCode) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_ed9f13149b0de397, []int{0} -} - -type GetSharesByNamespaceRequest struct { - RootHash []byte `protobuf:"bytes,1,opt,name=root_hash,json=rootHash,proto3" json:"root_hash,omitempty"` - Namespace []byte `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` -} - -func (m *GetSharesByNamespaceRequest) Reset() { *m = GetSharesByNamespaceRequest{} } -func (m *GetSharesByNamespaceRequest) String() string { return proto.CompactTextString(m) } -func (*GetSharesByNamespaceRequest) ProtoMessage() {} -func (*GetSharesByNamespaceRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ed9f13149b0de397, []int{0} -} -func (m *GetSharesByNamespaceRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *GetSharesByNamespaceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_GetSharesByNamespaceRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *GetSharesByNamespaceRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetSharesByNamespaceRequest.Merge(m, src) -} -func (m *GetSharesByNamespaceRequest) XXX_Size() int { - return m.Size() -} -func (m *GetSharesByNamespaceRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetSharesByNamespaceRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_GetSharesByNamespaceRequest proto.InternalMessageInfo - -func (m *GetSharesByNamespaceRequest) GetRootHash() []byte { - if m != nil { - return m.RootHash - } - return nil -} - -func (m *GetSharesByNamespaceRequest) GetNamespace() []byte { - if m != nil { - return m.Namespace - } - return nil -} - -type GetSharesByNamespaceStatusResponse struct { - Status StatusCode `protobuf:"varint,1,opt,name=status,proto3,enum=share.p2p.shrex.nd.StatusCode" json:"status,omitempty"` -} - -func (m *GetSharesByNamespaceStatusResponse) Reset() { *m = GetSharesByNamespaceStatusResponse{} } -func (m *GetSharesByNamespaceStatusResponse) String() string { return proto.CompactTextString(m) } -func (*GetSharesByNamespaceStatusResponse) ProtoMessage() {} -func (*GetSharesByNamespaceStatusResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ed9f13149b0de397, []int{1} -} -func (m *GetSharesByNamespaceStatusResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *GetSharesByNamespaceStatusResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_GetSharesByNamespaceStatusResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *GetSharesByNamespaceStatusResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetSharesByNamespaceStatusResponse.Merge(m, src) -} -func (m *GetSharesByNamespaceStatusResponse) XXX_Size() int { - return m.Size() -} -func (m *GetSharesByNamespaceStatusResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetSharesByNamespaceStatusResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_GetSharesByNamespaceStatusResponse proto.InternalMessageInfo - -func (m *GetSharesByNamespaceStatusResponse) GetStatus() StatusCode { - if m != nil { - return m.Status - } - return StatusCode_INVALID -} - -type NamespaceRowResponse struct { - Shares [][]byte `protobuf:"bytes,1,rep,name=shares,proto3" json:"shares,omitempty"` - Proof *pb.Proof `protobuf:"bytes,2,opt,name=proof,proto3" json:"proof,omitempty"` -} - -func (m *NamespaceRowResponse) Reset() { *m = NamespaceRowResponse{} } -func (m *NamespaceRowResponse) String() string { return proto.CompactTextString(m) } -func (*NamespaceRowResponse) ProtoMessage() {} -func (*NamespaceRowResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ed9f13149b0de397, []int{2} -} -func (m *NamespaceRowResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NamespaceRowResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NamespaceRowResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *NamespaceRowResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NamespaceRowResponse.Merge(m, src) -} -func (m *NamespaceRowResponse) XXX_Size() int { - return m.Size() -} -func (m *NamespaceRowResponse) XXX_DiscardUnknown() { - xxx_messageInfo_NamespaceRowResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_NamespaceRowResponse proto.InternalMessageInfo - -func (m *NamespaceRowResponse) GetShares() [][]byte { - if m != nil { - return m.Shares - } - return nil -} - -func (m *NamespaceRowResponse) GetProof() *pb.Proof { - if m != nil { - return m.Proof - } - return nil -} - -func init() { - proto.RegisterEnum("share.p2p.shrex.nd.StatusCode", StatusCode_name, StatusCode_value) - proto.RegisterType((*GetSharesByNamespaceRequest)(nil), "share.p2p.shrex.nd.GetSharesByNamespaceRequest") - proto.RegisterType((*GetSharesByNamespaceStatusResponse)(nil), "share.p2p.shrex.nd.GetSharesByNamespaceStatusResponse") - proto.RegisterType((*NamespaceRowResponse)(nil), "share.p2p.shrex.nd.NamespaceRowResponse") -} - -func init() { proto.RegisterFile("share/p2p/shrexnd/pb/share.proto", fileDescriptor_ed9f13149b0de397) } - -var fileDescriptor_ed9f13149b0de397 = []byte{ - // 326 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0x4f, 0x4b, 0xf3, 0x40, - 0x10, 0xc6, 0x93, 0x96, 0x37, 0x6f, 0x3b, 0xad, 0x35, 0x2c, 0x22, 0xc5, 0xca, 0x52, 0x02, 0x42, - 0xf1, 0xb0, 0x81, 0x08, 0x1e, 0x85, 0xd6, 0xfa, 0xa7, 0x58, 0x52, 0xd9, 0xb6, 0xe2, 0x41, 0x28, - 0x1b, 0xbb, 0x92, 0x8b, 0xd9, 0x35, 0xbb, 0x45, 0xfd, 0x16, 0x7e, 0x2c, 0x8f, 0x3d, 0x7a, 0x94, - 0xf6, 0x8b, 0x48, 0xb6, 0xd1, 0x1c, 0xf4, 0xb6, 0xf3, 0xcc, 0x33, 0xbf, 0x7d, 0x66, 0xa0, 0xad, - 0x62, 0x96, 0x72, 0x5f, 0x06, 0xd2, 0x57, 0x71, 0xca, 0x5f, 0x92, 0xb9, 0x2f, 0x23, 0xdf, 0x88, - 0x44, 0xa6, 0x42, 0x0b, 0x84, 0xf2, 0x22, 0x90, 0xc4, 0x38, 0x48, 0x32, 0xdf, 0x6b, 0xc8, 0xc8, - 0x97, 0xa9, 0x10, 0x0f, 0x1b, 0x8f, 0x77, 0x0b, 0xad, 0x0b, 0xae, 0xc7, 0x99, 0x51, 0xf5, 0x5e, - 0x43, 0xf6, 0xc8, 0x95, 0x64, 0xf7, 0x9c, 0xf2, 0xa7, 0x05, 0x57, 0x1a, 0xb5, 0xa0, 0x9a, 0x0a, - 0xa1, 0x67, 0x31, 0x53, 0x71, 0xd3, 0x6e, 0xdb, 0x9d, 0x3a, 0xad, 0x64, 0xc2, 0x25, 0x53, 0x31, - 0xda, 0x87, 0x6a, 0xf2, 0x3d, 0xd0, 0x2c, 0x99, 0x66, 0x21, 0x78, 0x77, 0xe0, 0xfd, 0x45, 0x1e, - 0x6b, 0xa6, 0x17, 0x8a, 0x72, 0x25, 0x45, 0xa2, 0x38, 0x3a, 0x06, 0x47, 0x19, 0xc5, 0xd0, 0x1b, - 0x01, 0x26, 0xbf, 0x43, 0x93, 0xcd, 0xcc, 0xa9, 0x98, 0x73, 0x9a, 0xbb, 0xbd, 0x29, 0xec, 0x14, - 0x61, 0xc5, 0xf3, 0x0f, 0x6f, 0x17, 0x1c, 0x03, 0xc8, 0x78, 0xe5, 0x4e, 0x9d, 0xe6, 0x15, 0x3a, - 0x80, 0x7f, 0x66, 0x6d, 0x93, 0xb3, 0x16, 0x6c, 0x93, 0xfc, 0x08, 0x11, 0xb9, 0xce, 0x1e, 0x74, - 0xd3, 0x3d, 0x3c, 0x01, 0x28, 0x3e, 0x43, 0x35, 0xf8, 0x3f, 0x08, 0x6f, 0xba, 0xc3, 0x41, 0xdf, - 0xb5, 0x90, 0x03, 0xa5, 0xd1, 0x95, 0x6b, 0xa3, 0x2d, 0xa8, 0x86, 0xa3, 0xc9, 0xec, 0x7c, 0x34, - 0x0d, 0xfb, 0x6e, 0x09, 0xd5, 0xa1, 0x32, 0x08, 0x27, 0x67, 0x34, 0xec, 0x0e, 0xdd, 0x72, 0xaf, - 0xf9, 0xbe, 0xc2, 0xf6, 0x72, 0x85, 0xed, 0xcf, 0x15, 0xb6, 0xdf, 0xd6, 0xd8, 0x5a, 0xae, 0xb1, - 0xf5, 0xb1, 0xc6, 0x56, 0xe4, 0x98, 0x7b, 0x1f, 0x7d, 0x05, 0x00, 0x00, 0xff, 0xff, 0x1a, 0x53, - 0xb4, 0x86, 0xb7, 0x01, 0x00, 0x00, -} - -func (m *GetSharesByNamespaceRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *GetSharesByNamespaceRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *GetSharesByNamespaceRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Namespace) > 0 { - i -= len(m.Namespace) - copy(dAtA[i:], m.Namespace) - i = encodeVarintShare(dAtA, i, uint64(len(m.Namespace))) - i-- - dAtA[i] = 0x12 - } - if len(m.RootHash) > 0 { - i -= len(m.RootHash) - copy(dAtA[i:], m.RootHash) - i = encodeVarintShare(dAtA, i, uint64(len(m.RootHash))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *GetSharesByNamespaceStatusResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *GetSharesByNamespaceStatusResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *GetSharesByNamespaceStatusResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Status != 0 { - i = encodeVarintShare(dAtA, i, uint64(m.Status)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *NamespaceRowResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *NamespaceRowResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *NamespaceRowResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Proof != nil { - { - size, err := m.Proof.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintShare(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - if len(m.Shares) > 0 { - for iNdEx := len(m.Shares) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Shares[iNdEx]) - copy(dAtA[i:], m.Shares[iNdEx]) - i = encodeVarintShare(dAtA, i, uint64(len(m.Shares[iNdEx]))) - i-- - dAtA[i] = 0xa - } - } - return len(dAtA) - i, nil -} - -func encodeVarintShare(dAtA []byte, offset int, v uint64) int { - offset -= sovShare(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *GetSharesByNamespaceRequest) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.RootHash) - if l > 0 { - n += 1 + l + sovShare(uint64(l)) - } - l = len(m.Namespace) - if l > 0 { - n += 1 + l + sovShare(uint64(l)) - } - return n -} - -func (m *GetSharesByNamespaceStatusResponse) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Status != 0 { - n += 1 + sovShare(uint64(m.Status)) - } - return n -} - -func (m *NamespaceRowResponse) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if len(m.Shares) > 0 { - for _, b := range m.Shares { - l = len(b) - n += 1 + l + sovShare(uint64(l)) - } - } - if m.Proof != nil { - l = m.Proof.Size() - n += 1 + l + sovShare(uint64(l)) - } - return n -} - -func sovShare(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozShare(x uint64) (n int) { - return sovShare(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *GetSharesByNamespaceRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: GetSharesByNamespaceRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: GetSharesByNamespaceRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field RootHash", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthShare - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthShare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.RootHash = append(m.RootHash[:0], dAtA[iNdEx:postIndex]...) - if m.RootHash == nil { - m.RootHash = []byte{} - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthShare - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthShare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Namespace = append(m.Namespace[:0], dAtA[iNdEx:postIndex]...) - if m.Namespace == nil { - m.Namespace = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipShare(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthShare - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *GetSharesByNamespaceStatusResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: GetSharesByNamespaceStatusResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: GetSharesByNamespaceStatusResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) - } - m.Status = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Status |= StatusCode(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipShare(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthShare - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *NamespaceRowResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: NamespaceRowResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: NamespaceRowResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Shares", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthShare - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthShare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Shares = append(m.Shares, make([]byte, postIndex-iNdEx)) - copy(m.Shares[len(m.Shares)-1], dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Proof", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthShare - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthShare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Proof == nil { - m.Proof = &pb.Proof{} - } - if err := m.Proof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipShare(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthShare - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipShare(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - depth := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowShare - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowShare - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - case 1: - iNdEx += 8 - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowShare - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthShare - } - iNdEx += length - case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupShare - } - depth-- - case 5: - iNdEx += 4 - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - if iNdEx < 0 { - return 0, ErrInvalidLengthShare - } - if depth == 0 { - return iNdEx, nil - } - } - return 0, io.ErrUnexpectedEOF -} - -var ( - ErrInvalidLengthShare = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowShare = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupShare = fmt.Errorf("proto: unexpected end of group") -) diff --git a/share/p2p/shrexnd/pb/share.proto b/share/p2p/shrexnd/pb/share.proto deleted file mode 100644 index a5bdbfa071..0000000000 --- a/share/p2p/shrexnd/pb/share.proto +++ /dev/null @@ -1,25 +0,0 @@ -syntax = "proto3"; - -package share.p2p.shrex.nd; -import "pb/proof.proto"; - -message GetSharesByNamespaceRequest{ - bytes root_hash = 1; - bytes namespace = 2; -} - -message GetSharesByNamespaceStatusResponse{ - StatusCode status = 1; -} - -enum StatusCode { - INVALID = 0; - OK = 1; - NOT_FOUND = 2; - INTERNAL = 3; -}; - -message NamespaceRowResponse { - repeated bytes shares = 1; - proof.pb.Proof proof = 2; -} diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go deleted file mode 100644 index 123d581494..0000000000 --- a/share/p2p/shrexnd/server.go +++ /dev/null @@ -1,257 +0,0 @@ -package shrexnd - -import ( - "context" - "crypto/sha256" - "errors" - "fmt" - "time" - - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/protocol" - "go.uber.org/zap" - - "github.com/celestiaorg/go-libp2p-messenger/serde" - nmt_pb "github.com/celestiaorg/nmt/pb" - - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/p2p" - pb "github.com/celestiaorg/celestia-node/share/p2p/shrexnd/pb" -) - -// Server implements server side of shrex/nd protocol to serve namespaced share to remote -// peers. -type Server struct { - cancel context.CancelFunc - - host host.Host - protocolID protocol.ID - - handler network.StreamHandler - store *eds.Store - - params *Parameters - middleware *p2p.Middleware - metrics *p2p.Metrics -} - -// NewServer creates new Server -func NewServer(params *Parameters, host host.Host, store *eds.Store) (*Server, error) { - if err := params.Validate(); err != nil { - return nil, fmt.Errorf("shrex-nd: server creation failed: %w", err) - } - - srv := &Server{ - store: store, - host: host, - params: params, - protocolID: p2p.ProtocolID(params.NetworkID(), protocolString), - middleware: p2p.NewMiddleware(params.ConcurrencyLimit), - } - - ctx, cancel := context.WithCancel(context.Background()) - srv.cancel = cancel - - handler := srv.streamHandler(ctx) - withRateLimit := srv.middleware.RateLimitHandler(handler) - withRecovery := p2p.RecoveryMiddleware(withRateLimit) - srv.handler = withRecovery - return srv, nil -} - -// Start starts the server -func (srv *Server) Start(context.Context) error { - srv.host.SetStreamHandler(srv.protocolID, srv.handler) - return nil -} - -// Stop stops the server -func (srv *Server) Stop(context.Context) error { - srv.cancel() - srv.host.RemoveStreamHandler(srv.protocolID) - return nil -} - -func (srv *Server) streamHandler(ctx context.Context) network.StreamHandler { - return func(s network.Stream) { - err := srv.handleNamespacedData(ctx, s) - if err != nil { - s.Reset() //nolint:errcheck - return - } - if err = s.Close(); err != nil { - log.Debugw("server: closing stream", "err", err) - } - } -} - -// SetHandler sets server handler -func (srv *Server) SetHandler(handler network.StreamHandler) { - srv.handler = handler -} - -func (srv *Server) observeRateLimitedRequests() { - numRateLimited := srv.middleware.DrainCounter() - if numRateLimited > 0 { - srv.metrics.ObserveRequests(context.Background(), numRateLimited, p2p.StatusRateLimited) - } -} - -func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stream) error { - logger := log.With("source", "server", "peer", stream.Conn().RemotePeer().String()) - logger.Debug("handling nd request") - - srv.observeRateLimitedRequests() - req, err := srv.readRequest(logger, stream) - if err != nil { - logger.Warnw("read request", "err", err) - srv.metrics.ObserveRequests(ctx, 1, p2p.StatusBadRequest) - return err - } - - logger = logger.With("namespace", share.Namespace(req.Namespace).String(), - "hash", share.DataHash(req.RootHash).String()) - - ctx, cancel := context.WithTimeout(ctx, srv.params.HandleRequestTimeout) - defer cancel() - - shares, status, err := srv.getNamespaceData(ctx, req.RootHash, req.Namespace) - if err != nil { - // server should respond with status regardless if there was an error getting data - sendErr := srv.respondStatus(ctx, logger, stream, status) - if sendErr != nil { - logger.Errorw("sending response", "err", sendErr) - srv.metrics.ObserveRequests(ctx, 1, p2p.StatusSendRespErr) - } - logger.Errorw("handling request", "err", err) - return errors.Join(err, sendErr) - } - - err = srv.respondStatus(ctx, logger, stream, status) - if err != nil { - logger.Errorw("sending response", "err", err) - srv.metrics.ObserveRequests(ctx, 1, p2p.StatusSendRespErr) - return err - } - - err = srv.sendNamespacedShares(shares, stream) - if err != nil { - logger.Errorw("send nd data", "err", err) - srv.metrics.ObserveRequests(ctx, 1, p2p.StatusSendRespErr) - return err - } - return nil -} - -func (srv *Server) readRequest( - logger *zap.SugaredLogger, - stream network.Stream, -) (*pb.GetSharesByNamespaceRequest, error) { - err := stream.SetReadDeadline(time.Now().Add(srv.params.ServerReadTimeout)) - if err != nil { - logger.Debugw("setting read deadline", "err", err) - } - - var req pb.GetSharesByNamespaceRequest - _, err = serde.Read(stream, &req) - if err != nil { - return nil, fmt.Errorf("reading request: %w", err) - } - - logger.Debugw("new request") - err = stream.CloseRead() - if err != nil { - logger.Debugw("closing read side of the stream", "err", err) - } - - err = validateRequest(req) - if err != nil { - return nil, fmt.Errorf("invalid request: %w", err) - } - return &req, nil -} - -func (srv *Server) getNamespaceData(ctx context.Context, - hash share.DataHash, namespace share.Namespace, -) (share.NamespacedShares, pb.StatusCode, error) { - roots, err := srv.store.GetDAH(ctx, hash) - if err != nil { - if errors.Is(err, eds.ErrNotFound) { - return nil, pb.StatusCode_NOT_FOUND, nil - } - return nil, pb.StatusCode_INTERNAL, fmt.Errorf("retrieving DAH: %w", err) - } - - shares, err := eds.RetrieveNamespaceFromStore(ctx, srv.store, roots, namespace) - if err != nil { - return nil, pb.StatusCode_INTERNAL, fmt.Errorf("retrieving shares: %w", err) - } - - return shares, pb.StatusCode_OK, nil -} - -func (srv *Server) respondStatus( - ctx context.Context, - logger *zap.SugaredLogger, - stream network.Stream, - status pb.StatusCode, -) error { - srv.observeStatus(ctx, status) - - err := stream.SetWriteDeadline(time.Now().Add(srv.params.ServerWriteTimeout)) - if err != nil { - logger.Debugw("setting write deadline", "err", err) - } - - _, err = serde.Write(stream, &pb.GetSharesByNamespaceStatusResponse{Status: status}) - if err != nil { - return fmt.Errorf("writing response: %w", err) - } - - return nil -} - -// sendNamespacedShares encodes shares into proto messages and sends it to client -func (srv *Server) sendNamespacedShares(shares share.NamespacedShares, stream network.Stream) error { - for _, row := range shares { - row := &pb.NamespaceRowResponse{ - Shares: row.Shares, - Proof: &nmt_pb.Proof{ - Start: int64(row.Proof.Start()), - End: int64(row.Proof.End()), - Nodes: row.Proof.Nodes(), - LeafHash: row.Proof.LeafHash(), - IsMaxNamespaceIgnored: row.Proof.IsMaxNamespaceIDIgnored(), - }, - } - _, err := serde.Write(stream, row) - if err != nil { - return fmt.Errorf("writing nd data to stream: %w", err) - } - } - return nil -} - -func (srv *Server) observeStatus(ctx context.Context, status pb.StatusCode) { - switch { - case status == pb.StatusCode_OK: - srv.metrics.ObserveRequests(ctx, 1, p2p.StatusSuccess) - case status == pb.StatusCode_NOT_FOUND: - srv.metrics.ObserveRequests(ctx, 1, p2p.StatusNotFound) - case status == pb.StatusCode_INTERNAL: - srv.metrics.ObserveRequests(ctx, 1, p2p.StatusInternalErr) - } -} - -// validateRequest checks correctness of the request -func validateRequest(req pb.GetSharesByNamespaceRequest) error { - if err := share.Namespace(req.Namespace).ValidateForData(); err != nil { - return err - } - if len(req.RootHash) != sha256.Size { - return fmt.Errorf("incorrect root hash length: %v", len(req.RootHash)) - } - return nil -} diff --git a/share/shwap/eds_id.go b/share/shwap/eds_id.go index 011a9be412..dab4aa7ba0 100644 --- a/share/shwap/eds_id.go +++ b/share/shwap/eds_id.go @@ -7,13 +7,16 @@ import ( "io" ) -// EdsIDSize defines the byte size of the EdsID. -const EdsIDSize = 8 +const ( + // EdsIDSize defines the byte size of the EdsID. + EdsIDSize = 8 + // EDSName is the name identifier for the Extended Data Square. + EDSName = "eds_v0" +) // ErrOutOfBounds is returned whenever an index is out of bounds. var ( ErrInvalidShwapID = errors.New("invalid shwap ID") - ErrOutOfBounds = fmt.Errorf("index out of bounds: %w", ErrInvalidShwapID) ) // EdsID represents a unique identifier for a row, using the height of the block diff --git a/share/shwap/getter.go b/share/shwap/getter.go new file mode 100644 index 0000000000..370a2d696a --- /dev/null +++ b/share/shwap/getter.go @@ -0,0 +1,42 @@ +package shwap + +import ( + "context" + "errors" + "fmt" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share" +) + +var ( + // ErrOperationNotSupported is used to indicate that the operation is not supported by the + // implementation of the getter interface. + ErrOperationNotSupported = errors.New("operation is not supported") + // ErrNotFound is used to indicate that requested data could not be found. + ErrNotFound = errors.New("data not found") + // ErrOutOfBounds is used to indicate that a passed row or column index is out of bounds of the + // square size. + ErrOutOfBounds = fmt.Errorf("index out of bounds: %w", ErrInvalidShwapID) +) + +// Getter interface provides a set of accessors for shares by the Root. +// Automatically verifies integrity of shares(exceptions possible depending on the implementation). +// +//go:generate mockgen -destination=mocks/getter.go -package=mocks . Getter +type Getter interface { + // GetShare gets a Share by coordinates in EDS. + GetShare(ctx context.Context, header *header.ExtendedHeader, row, col int) (share.Share, error) + + // GetEDS gets the full EDS identified by the given extended header. + GetEDS(context.Context, *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) + + // GetSharesByNamespace gets all shares from an EDS within the given namespace. + // Shares are returned in a row-by-row order if the namespace spans multiple rows. + // Inclusion of returned data could be verified using Verify method on NamespacedShares. + // If no shares are found for target namespace non-inclusion could be also verified by calling + // Verify method. + GetSharesByNamespace(context.Context, *header.ExtendedHeader, share.Namespace) (NamespaceData, error) +} diff --git a/share/shwap/namespace_data.go b/share/shwap/namespace_data.go index 62b157e783..5b3e7a7e9c 100644 --- a/share/shwap/namespace_data.go +++ b/share/shwap/namespace_data.go @@ -11,12 +11,15 @@ import ( "github.com/celestiaorg/celestia-node/share/shwap/pb" ) -// NamespacedData stores collections of RowNamespaceData, each representing shares and their proofs +// NamespaceDataName is the name identifier for the namespace data container. +const NamespaceDataName = "nd_v0" + +// NamespaceData stores collections of RowNamespaceData, each representing shares and their proofs // within a namespace. -type NamespacedData []RowNamespaceData +type NamespaceData []RowNamespaceData // Flatten combines all shares from all rows within the namespace into a single slice. -func (nd NamespacedData) Flatten() []share.Share { +func (nd NamespaceData) Flatten() []share.Share { var shares []share.Share for _, row := range nd { shares = append(shares, row.Shares...) @@ -24,8 +27,8 @@ func (nd NamespacedData) Flatten() []share.Share { return shares } -// Validate checks the integrity of the NamespacedData against a provided root and namespace. -func (nd NamespacedData) Validate(root *share.AxisRoots, namespace share.Namespace) error { +// Validate checks the integrity of the NamespaceData against a provided root and namespace. +func (nd NamespaceData) Validate(root *share.AxisRoots, namespace share.Namespace) error { rowIdxs := share.RowsWithNamespace(root, namespace) if len(rowIdxs) != len(nd) { return fmt.Errorf("expected %d rows, found %d rows", len(rowIdxs), len(nd)) @@ -39,21 +42,37 @@ func (nd NamespacedData) Validate(root *share.AxisRoots, namespace share.Namespa return nil } -// ReadFrom reads the binary form of NamespacedData from the provided reader. -func (nd *NamespacedData) ReadFrom(reader io.Reader) error { +// ReadFrom reads the binary form of NamespaceData from the provided reader. +func (nd *NamespaceData) ReadFrom(reader io.Reader) (int64, error) { var data []RowNamespaceData + var n int64 for { var pbRow pb.RowNamespaceData - _, err := serde.Read(reader, &pbRow) + nn, err := serde.Read(reader, &pbRow) + n += int64(nn) if err != nil { if errors.Is(err, io.EOF) { // all rows have been read *nd = data - return nil + return n, nil } - return err + return n, err } row := RowNamespaceDataFromProto(&pbRow) data = append(data, row) } } + +// WriteTo writes the binary form of NamespaceData to the provided writer. +func (nd NamespaceData) WriteTo(writer io.Writer) (int64, error) { + var n int64 + for _, row := range nd { + pbRow := row.ToProto() + nn, err := serde.Write(writer, pbRow) + n += int64(nn) + if err != nil { + return n, err + } + } + return n, nil +} diff --git a/share/p2p/discovery/backoff.go b/share/shwap/p2p/discovery/backoff.go similarity index 100% rename from share/p2p/discovery/backoff.go rename to share/shwap/p2p/discovery/backoff.go diff --git a/share/p2p/discovery/backoff_test.go b/share/shwap/p2p/discovery/backoff_test.go similarity index 100% rename from share/p2p/discovery/backoff_test.go rename to share/shwap/p2p/discovery/backoff_test.go diff --git a/share/p2p/discovery/discovery.go b/share/shwap/p2p/discovery/discovery.go similarity index 100% rename from share/p2p/discovery/discovery.go rename to share/shwap/p2p/discovery/discovery.go diff --git a/share/p2p/discovery/discovery_test.go b/share/shwap/p2p/discovery/discovery_test.go similarity index 100% rename from share/p2p/discovery/discovery_test.go rename to share/shwap/p2p/discovery/discovery_test.go diff --git a/share/p2p/discovery/metrics.go b/share/shwap/p2p/discovery/metrics.go similarity index 100% rename from share/p2p/discovery/metrics.go rename to share/shwap/p2p/discovery/metrics.go diff --git a/share/p2p/discovery/options.go b/share/shwap/p2p/discovery/options.go similarity index 100% rename from share/p2p/discovery/options.go rename to share/shwap/p2p/discovery/options.go diff --git a/share/p2p/discovery/set.go b/share/shwap/p2p/discovery/set.go similarity index 100% rename from share/p2p/discovery/set.go rename to share/shwap/p2p/discovery/set.go diff --git a/share/p2p/discovery/set_test.go b/share/shwap/p2p/discovery/set_test.go similarity index 100% rename from share/p2p/discovery/set_test.go rename to share/shwap/p2p/discovery/set_test.go diff --git a/share/p2p/doc.go b/share/shwap/p2p/shrex/doc.go similarity index 88% rename from share/p2p/doc.go rename to share/shwap/p2p/shrex/doc.go index 991ddf94db..9654532842 100644 --- a/share/p2p/doc.go +++ b/share/shwap/p2p/shrex/doc.go @@ -1,4 +1,4 @@ -// Package p2p provides p2p functionality that powers the share exchange protocols used by celestia-node. +// Package shrex provides functionality that powers the share exchange protocols used by celestia-node. // The available protocols are: // // - shrexsub : a floodsub-based pubsub protocol that is used to broadcast/subscribe to the event @@ -15,4 +15,4 @@ // and is primarily used by `getters.ShrexGetter` in share/getters/shrex.go. // // Find out more about each protocol in their respective sub-packages. -package p2p +package shrex diff --git a/share/shwap/p2p/shrex/error.go b/share/shwap/p2p/shrex/error.go new file mode 100644 index 0000000000..d32c3c85b2 --- /dev/null +++ b/share/shwap/p2p/shrex/error.go @@ -0,0 +1,16 @@ +package shrex + +import "errors" + +// ErrorContains reports whether any error in err's tree matches any error in targets tree. +func ErrorContains(err, target error) bool { + if errors.Is(err, target) || target == nil { + return true + } + + target = errors.Unwrap(target) + if target == nil { + return false + } + return ErrorContains(err, target) +} diff --git a/share/shwap/p2p/shrex/error_test.go b/share/shwap/p2p/shrex/error_test.go new file mode 100644 index 0000000000..d9b343931c --- /dev/null +++ b/share/shwap/p2p/shrex/error_test.go @@ -0,0 +1,112 @@ +package shrex + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_ErrorContains(t *testing.T) { + err1 := errors.New("1") + err2 := errors.New("2") + + w1 := func(err error) error { + return fmt.Errorf("wrap1: %w", err) + } + w2 := func(err error) error { + return fmt.Errorf("wrap1: %w", err) + } + + type args struct { + err error + target error + } + tests := []struct { + name string + args args + want bool + }{ + { + "nil err", + args{ + err: nil, + target: err1, + }, + false, + }, + { + "nil target", + args{ + err: err1, + target: nil, + }, + true, + }, + { + "errors.Is true", + args{ + err: w1(err1), + target: err1, + }, + true, + }, + { + "errors.Is false", + args{ + err: w1(err1), + target: err2, + }, + false, + }, + { + "same wrap but different base error", + args{ + err: w1(err1), + target: w1(err2), + }, + false, + }, + { + "both wrapped true", + args{ + err: w1(err1), + target: w2(err1), + }, + true, + }, + { + "both wrapped false", + args{ + err: w1(err1), + target: w2(err2), + }, + false, + }, + { + "multierr first in slice", + args{ + err: errors.Join(w1(err1), w2(err2)), + target: w2(err1), + }, + true, + }, + { + "multierr second in slice", + args{ + err: errors.Join(w1(err1), w2(err2)), + target: w1(err2), + }, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, + tt.want, + ErrorContains(tt.args.err, tt.args.target), + "ErrorContains(%v, %v)", tt.args.err, tt.args.target) + }) + } +} diff --git a/share/p2p/errors.go b/share/shwap/p2p/shrex/errors.go similarity index 98% rename from share/p2p/errors.go rename to share/shwap/p2p/shrex/errors.go index cb7b596f47..79ff0ed2b2 100644 --- a/share/p2p/errors.go +++ b/share/shwap/p2p/shrex/errors.go @@ -1,4 +1,4 @@ -package p2p +package shrex import ( "errors" diff --git a/share/p2p/metrics.go b/share/shwap/p2p/shrex/metrics.go similarity index 99% rename from share/p2p/metrics.go rename to share/shwap/p2p/shrex/metrics.go index 55aefda81d..9d5c605139 100644 --- a/share/p2p/metrics.go +++ b/share/shwap/p2p/shrex/metrics.go @@ -1,4 +1,4 @@ -package p2p +package shrex import ( "context" diff --git a/share/p2p/middleware.go b/share/shwap/p2p/shrex/middleware.go similarity index 98% rename from share/p2p/middleware.go rename to share/shwap/p2p/shrex/middleware.go index df0a690af7..c53a996eec 100644 --- a/share/p2p/middleware.go +++ b/share/shwap/p2p/shrex/middleware.go @@ -1,4 +1,4 @@ -package p2p +package shrex import ( "sync/atomic" diff --git a/share/p2p/params.go b/share/shwap/p2p/shrex/params.go similarity index 94% rename from share/p2p/params.go rename to share/shwap/p2p/shrex/params.go index 6636e38fc5..a5c0d5e3e2 100644 --- a/share/p2p/params.go +++ b/share/shwap/p2p/shrex/params.go @@ -1,4 +1,4 @@ -package p2p +package shrex import ( "fmt" @@ -7,6 +7,9 @@ import ( "github.com/libp2p/go-libp2p/core/protocol" ) +// protocolString is the protocol string for the shrex protocol. +const ProtocolString = "/shrex/v0.1.0/" + // Parameters is the set of parameters that must be configured for the shrex/eds protocol. type Parameters struct { // ServerReadTimeout sets the timeout for reading messages from the stream. diff --git a/share/shwap/p2p/shrex/pb/shrex.pb.go b/share/shwap/p2p/shrex/pb/shrex.pb.go new file mode 100644 index 0000000000..88322b1985 --- /dev/null +++ b/share/shwap/p2p/shrex/pb/shrex.pb.go @@ -0,0 +1,336 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: share/shwap/p2p/shrex/pb/shrex.proto + +package pb + +import ( + fmt "fmt" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type Status int32 + +const ( + Status_INVALID Status = 0 + Status_OK Status = 1 + Status_NOT_FOUND Status = 2 + Status_INTERNAL Status = 3 +) + +var Status_name = map[int32]string{ + 0: "INVALID", + 1: "OK", + 2: "NOT_FOUND", + 3: "INTERNAL", +} + +var Status_value = map[string]int32{ + "INVALID": 0, + "OK": 1, + "NOT_FOUND": 2, + "INTERNAL": 3, +} + +func (x Status) String() string { + return proto.EnumName(Status_name, int32(x)) +} + +func (Status) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_68dfd78ade756110, []int{0} +} + +type Response struct { + Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=Status" json:"status,omitempty"` +} + +func (m *Response) Reset() { *m = Response{} } +func (m *Response) String() string { return proto.CompactTextString(m) } +func (*Response) ProtoMessage() {} +func (*Response) Descriptor() ([]byte, []int) { + return fileDescriptor_68dfd78ade756110, []int{0} +} +func (m *Response) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Response.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Response) XXX_Merge(src proto.Message) { + xxx_messageInfo_Response.Merge(m, src) +} +func (m *Response) XXX_Size() int { + return m.Size() +} +func (m *Response) XXX_DiscardUnknown() { + xxx_messageInfo_Response.DiscardUnknown(m) +} + +var xxx_messageInfo_Response proto.InternalMessageInfo + +func (m *Response) GetStatus() Status { + if m != nil { + return m.Status + } + return Status_INVALID +} + +func init() { + proto.RegisterEnum("Status", Status_name, Status_value) + proto.RegisterType((*Response)(nil), "Response") +} + +func init() { + proto.RegisterFile("share/shwap/p2p/shrex/pb/shrex.proto", fileDescriptor_68dfd78ade756110) +} + +var fileDescriptor_68dfd78ade756110 = []byte{ + // 218 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x29, 0xce, 0x48, 0x2c, + 0x4a, 0xd5, 0x2f, 0xce, 0x28, 0x4f, 0x2c, 0xd0, 0x2f, 0x30, 0x2a, 0xd0, 0x2f, 0xce, 0x28, 0x4a, + 0xad, 0xd0, 0x2f, 0x48, 0x82, 0x30, 0xf4, 0x0a, 0x8a, 0xf2, 0x4b, 0xf2, 0x95, 0xb4, 0xb9, 0x38, + 0x82, 0x52, 0x8b, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0xe4, 0xb9, 0xd8, 0x8a, 0x4b, 0x12, 0x4b, + 0x4a, 0x8b, 0x25, 0x18, 0x15, 0x18, 0x35, 0xf8, 0x8c, 0xd8, 0xf5, 0x82, 0xc1, 0xdc, 0x20, 0xa8, + 0xb0, 0x96, 0x15, 0x17, 0x1b, 0x44, 0x44, 0x88, 0x9b, 0x8b, 0xdd, 0xd3, 0x2f, 0xcc, 0xd1, 0xc7, + 0xd3, 0x45, 0x80, 0x41, 0x88, 0x8d, 0x8b, 0xc9, 0xdf, 0x5b, 0x80, 0x51, 0x88, 0x97, 0x8b, 0xd3, + 0xcf, 0x3f, 0x24, 0xde, 0xcd, 0x3f, 0xd4, 0xcf, 0x45, 0x80, 0x49, 0x88, 0x87, 0x8b, 0xc3, 0xd3, + 0x2f, 0xc4, 0x35, 0xc8, 0xcf, 0xd1, 0x47, 0x80, 0xd9, 0x29, 0xfc, 0xc4, 0x23, 0x39, 0xc6, 0x0b, + 0x8f, 0xe4, 0x18, 0x1f, 0x3c, 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, + 0x1b, 0x8f, 0xe5, 0x18, 0xa2, 0x6c, 0xd3, 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, + 0xf5, 0x93, 0x53, 0x73, 0x52, 0x8b, 0x4b, 0x32, 0x13, 0xf3, 0x8b, 0xd2, 0xe1, 0x6c, 0xdd, 0xbc, + 0xfc, 0x14, 0x90, 0x3f, 0xb0, 0xfb, 0x26, 0x89, 0x0d, 0xec, 0x11, 0x63, 0x40, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xd8, 0xb0, 0x4b, 0x58, 0xf0, 0x00, 0x00, 0x00, +} + +func (m *Response) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Response) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Response) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Status != 0 { + i = encodeVarintShrex(dAtA, i, uint64(m.Status)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func encodeVarintShrex(dAtA []byte, offset int, v uint64) int { + offset -= sovShrex(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *Response) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Status != 0 { + n += 1 + sovShrex(uint64(m.Status)) + } + return n +} + +func sovShrex(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozShrex(x uint64) (n int) { + return sovShrex(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Response) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShrex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Response: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Response: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + m.Status = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShrex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Status |= Status(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipShrex(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShrex + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipShrex(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowShrex + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowShrex + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowShrex + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthShrex + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupShrex + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthShrex + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthShrex = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowShrex = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupShrex = fmt.Errorf("proto: unexpected end of group") +) diff --git a/share/p2p/shrexeds/pb/extended_data_square.proto b/share/shwap/p2p/shrex/pb/shrex.proto similarity index 62% rename from share/p2p/shrexeds/pb/extended_data_square.proto rename to share/shwap/p2p/shrex/pb/shrex.proto index 63750962e9..7b3a57227d 100644 --- a/share/p2p/shrexeds/pb/extended_data_square.proto +++ b/share/shwap/p2p/shrex/pb/shrex.proto @@ -1,8 +1,6 @@ syntax = "proto3"; -message EDSRequest { - bytes hash = 1; // identifies the requested EDS. -} +option go_package = "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/pb"; enum Status { INVALID = 0; @@ -11,6 +9,6 @@ enum Status { INTERNAL = 3; // internal server error } -message EDSResponse { +message Response { Status status = 1; } diff --git a/share/p2p/peers/doc.go b/share/shwap/p2p/shrex/peers/doc.go similarity index 100% rename from share/p2p/peers/doc.go rename to share/shwap/p2p/shrex/peers/doc.go diff --git a/share/p2p/peers/manager.go b/share/shwap/p2p/shrex/peers/manager.go similarity index 99% rename from share/p2p/peers/manager.go rename to share/shwap/p2p/shrex/peers/manager.go index 0d2f6ac42b..857c5ee937 100644 --- a/share/p2p/peers/manager.go +++ b/share/shwap/p2p/shrex/peers/manager.go @@ -21,7 +21,7 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexsub" ) const ( diff --git a/share/p2p/peers/manager_test.go b/share/shwap/p2p/shrex/peers/manager_test.go similarity index 99% rename from share/p2p/peers/manager_test.go rename to share/shwap/p2p/shrex/peers/manager_test.go index 2a465dc59a..390d368b02 100644 --- a/share/p2p/peers/manager_test.go +++ b/share/shwap/p2p/shrex/peers/manager_test.go @@ -22,8 +22,8 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/p2p/discovery" - "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/discovery" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexsub" ) func TestManager(t *testing.T) { @@ -76,6 +76,7 @@ func TestManager(t *testing.T) { t.Cleanup(cancel) // create headerSub mock + h := testHeader() headerSub := newSubLock(h, nil) diff --git a/share/p2p/peers/metrics.go b/share/shwap/p2p/shrex/peers/metrics.go similarity index 99% rename from share/p2p/peers/metrics.go rename to share/shwap/p2p/shrex/peers/metrics.go index b28b263127..d401d6a4fc 100644 --- a/share/p2p/peers/metrics.go +++ b/share/shwap/p2p/shrex/peers/metrics.go @@ -13,7 +13,7 @@ import ( "go.opentelemetry.io/otel/metric" "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexsub" ) const ( diff --git a/share/p2p/peers/options.go b/share/shwap/p2p/shrex/peers/options.go similarity index 97% rename from share/p2p/peers/options.go rename to share/shwap/p2p/shrex/peers/options.go index 2970dd2465..e268550853 100644 --- a/share/p2p/peers/options.go +++ b/share/shwap/p2p/shrex/peers/options.go @@ -7,7 +7,7 @@ import ( libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexsub" ) type Parameters struct { diff --git a/share/p2p/peers/pool.go b/share/shwap/p2p/shrex/peers/pool.go similarity index 100% rename from share/p2p/peers/pool.go rename to share/shwap/p2p/shrex/peers/pool.go diff --git a/share/p2p/peers/pool_test.go b/share/shwap/p2p/shrex/peers/pool_test.go similarity index 100% rename from share/p2p/peers/pool_test.go rename to share/shwap/p2p/shrex/peers/pool_test.go diff --git a/share/p2p/peers/timedqueue.go b/share/shwap/p2p/shrex/peers/timedqueue.go similarity index 100% rename from share/p2p/peers/timedqueue.go rename to share/shwap/p2p/shrex/peers/timedqueue.go diff --git a/share/p2p/peers/timedqueue_test.go b/share/shwap/p2p/shrex/peers/timedqueue_test.go similarity index 100% rename from share/p2p/peers/timedqueue_test.go rename to share/shwap/p2p/shrex/peers/timedqueue_test.go diff --git a/share/p2p/recovery.go b/share/shwap/p2p/shrex/recovery.go similarity index 96% rename from share/p2p/recovery.go rename to share/shwap/p2p/shrex/recovery.go index b214969399..67bcb98d73 100644 --- a/share/p2p/recovery.go +++ b/share/shwap/p2p/shrex/recovery.go @@ -1,4 +1,4 @@ -package p2p +package shrex import ( "fmt" diff --git a/share/getters/shrex.go b/share/shwap/p2p/shrex/shrex_getter/shrex.go similarity index 75% rename from share/getters/shrex.go rename to share/shwap/p2p/shrex/shrex_getter/shrex.go index 09601d60de..92f76bef4a 100644 --- a/share/getters/shrex.go +++ b/share/shwap/p2p/shrex/shrex_getter/shrex.go @@ -1,4 +1,4 @@ -package getters +package shrex_getter import ( "context" @@ -6,6 +6,7 @@ import ( "fmt" "time" + logging "github.com/ipfs/go-log/v2" libpeer "github.com/libp2p/go-libp2p/core/peer" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -18,14 +19,20 @@ import ( "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/pruner" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/ipld" - "github.com/celestiaorg/celestia-node/share/p2p" - "github.com/celestiaorg/celestia-node/share/p2p/peers" - "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" - "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/peers" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexeds" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexnd" ) -var _ share.Getter = (*ShrexGetter)(nil) +var ( + tracer = otel.Tracer("shrex/getter") + meter = otel.Meter("shrex/getter") + log = logging.Logger("shrex/getter") +) + +var _ shwap.Getter = (*Getter)(nil) const ( // defaultMinRequestTimeout value is set according to observed time taken by healthy peer to @@ -34,8 +41,6 @@ const ( defaultMinAttemptsCount = 3 ) -var meter = otel.Meter("shrex/getter") - type metrics struct { edsAttempts metric.Int64Histogram ndAttempts metric.Int64Histogram @@ -61,7 +66,7 @@ func (m *metrics) recordNDAttempt(ctx context.Context, attemptCount int, success attribute.Bool("success", success))) } -func (sg *ShrexGetter) WithMetrics() error { +func (sg *Getter) WithMetrics() error { edsAttemptHistogram, err := meter.Int64Histogram( "getters_shrex_eds_attempts_per_request", metric.WithDescription("Number of attempts per shrex/eds request"), @@ -85,8 +90,8 @@ func (sg *ShrexGetter) WithMetrics() error { return nil } -// ShrexGetter is a share.Getter that uses the shrex/eds and shrex/nd protocol to retrieve shares. -type ShrexGetter struct { +// Getter is a share.Getter that uses the shrex/eds and shrex/nd protocol to retrieve shares. +type Getter struct { edsClient *shrexeds.Client ndClient *shrexnd.Client @@ -104,14 +109,14 @@ type ShrexGetter struct { metrics *metrics } -func NewShrexGetter( +func NewGetter( edsClient *shrexeds.Client, ndClient *shrexnd.Client, fullPeerManager *peers.Manager, archivalManager *peers.Manager, availWindow pruner.AvailabilityWindow, -) *ShrexGetter { - s := &ShrexGetter{ +) *Getter { + s := &Getter{ edsClient: edsClient, ndClient: ndClient, fullPeerManager: fullPeerManager, @@ -124,7 +129,7 @@ func NewShrexGetter( return s } -func (sg *ShrexGetter) Start(ctx context.Context) error { +func (sg *Getter) Start(ctx context.Context) error { err := sg.fullPeerManager.Start(ctx) if err != nil { return err @@ -132,7 +137,7 @@ func (sg *ShrexGetter) Start(ctx context.Context) error { return sg.archivalPeerManager.Start(ctx) } -func (sg *ShrexGetter) Stop(ctx context.Context) error { +func (sg *Getter) Stop(ctx context.Context) error { err := sg.fullPeerManager.Stop(ctx) if err != nil { return err @@ -140,11 +145,11 @@ func (sg *ShrexGetter) Stop(ctx context.Context) error { return sg.archivalPeerManager.Stop(ctx) } -func (sg *ShrexGetter) GetShare(context.Context, *header.ExtendedHeader, int, int) (share.Share, error) { - return nil, fmt.Errorf("getter/shrex: GetShare %w", errOperationNotSupported) +func (sg *Getter) GetShare(context.Context, *header.ExtendedHeader, int, int) (share.Share, error) { + return nil, fmt.Errorf("getter/shrex: GetShare %w", shwap.ErrOperationNotSupported) } -func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { +func (sg *Getter) GetEDS(ctx context.Context, header *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { var err error ctx, span := tracer.Start(ctx, "shrex/get-eds") defer func() { @@ -176,8 +181,8 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader } reqStart := time.Now() - reqCtx, cancel := ctxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) - eds, getErr := sg.edsClient.RequestEDS(reqCtx, header.DAH.Hash(), peer) + reqCtx, cancel := utils.CtxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) + eds, getErr := sg.edsClient.RequestEDS(reqCtx, header.DAH, header.Height(), peer) cancel() switch { case getErr == nil: @@ -187,16 +192,16 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader case errors.Is(getErr, context.DeadlineExceeded), errors.Is(getErr, context.Canceled): setStatus(peers.ResultCooldownPeer) - case errors.Is(getErr, p2p.ErrNotFound): - getErr = share.ErrNotFound + case errors.Is(getErr, shrex.ErrNotFound): + getErr = shwap.ErrNotFound setStatus(peers.ResultCooldownPeer) - case errors.Is(getErr, p2p.ErrInvalidResponse): + case errors.Is(getErr, shrex.ErrInvalidResponse): setStatus(peers.ResultBlacklistPeer) default: setStatus(peers.ResultCooldownPeer) } - if !ErrorContains(err, getErr) { + if !shrex.ErrorContains(err, getErr) { err = errors.Join(err, getErr) } log.Debugw("eds: request failed", @@ -208,11 +213,11 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader } } -func (sg *ShrexGetter) GetSharesByNamespace( +func (sg *Getter) GetSharesByNamespace( ctx context.Context, header *header.ExtendedHeader, namespace share.Namespace, -) (share.NamespacedShares, error) { +) (shwap.NamespaceData, error) { if err := namespace.ValidateForData(); err != nil { return nil, err } @@ -229,9 +234,9 @@ func (sg *ShrexGetter) GetSharesByNamespace( // verify that the namespace could exist inside the roots before starting network requests dah := header.DAH - roots := ipld.FilterRootByNamespace(dah, namespace) - if len(roots) == 0 { - return []share.NamespacedRow{}, nil + rowIdxs := share.RowsWithNamespace(dah, namespace) + if len(rowIdxs) == 0 { + return shwap.NamespaceData{}, nil } for { @@ -254,13 +259,13 @@ func (sg *ShrexGetter) GetSharesByNamespace( } reqStart := time.Now() - reqCtx, cancel := ctxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) - nd, getErr := sg.ndClient.RequestND(reqCtx, dah, namespace, peer) + reqCtx, cancel := utils.CtxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) + nd, getErr := sg.ndClient.RequestND(reqCtx, header.Height(), namespace, peer) cancel() switch { case getErr == nil: // both inclusion and non-inclusion cases needs verification - if verErr := nd.Verify(dah, namespace); verErr != nil { + if verErr := nd.Validate(dah, namespace); verErr != nil { getErr = verErr setStatus(peers.ResultBlacklistPeer) break @@ -271,16 +276,16 @@ func (sg *ShrexGetter) GetSharesByNamespace( case errors.Is(getErr, context.DeadlineExceeded), errors.Is(getErr, context.Canceled): setStatus(peers.ResultCooldownPeer) - case errors.Is(getErr, p2p.ErrNotFound): - getErr = share.ErrNotFound + case errors.Is(getErr, shrex.ErrNotFound): + getErr = shwap.ErrNotFound setStatus(peers.ResultCooldownPeer) - case errors.Is(getErr, p2p.ErrInvalidResponse): + case errors.Is(getErr, shrex.ErrInvalidResponse): setStatus(peers.ResultBlacklistPeer) default: setStatus(peers.ResultCooldownPeer) } - if !ErrorContains(err, getErr) { + if !shrex.ErrorContains(err, getErr) { err = errors.Join(err, getErr) } log.Debugw("nd: request failed", @@ -293,7 +298,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( } } -func (sg *ShrexGetter) getPeer( +func (sg *Getter) getPeer( ctx context.Context, header *header.ExtendedHeader, ) (libpeer.ID, peers.DoneFunc, error) { diff --git a/share/getters/shrex_test.go b/share/shwap/p2p/shrex/shrex_getter/shrex_test.go similarity index 82% rename from share/getters/shrex_test.go rename to share/shwap/p2p/shrex/shrex_getter/shrex_test.go index a74fd8b890..5f1e1169a0 100644 --- a/share/getters/shrex_test.go +++ b/share/shwap/p2p/shrex/shrex_getter/shrex_test.go @@ -1,9 +1,10 @@ -package getters +package shrex_getter import ( "context" "encoding/binary" "errors" + "sync/atomic" "testing" "time" @@ -24,14 +25,14 @@ import ( "github.com/celestiaorg/celestia-node/pruner/full" "github.com/celestiaorg/celestia-node/pruner/light" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/ipld" - "github.com/celestiaorg/celestia-node/share/p2p/peers" - "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" - "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" - "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/peers" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexeds" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexnd" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexsub" + "github.com/celestiaorg/celestia-node/store" ) func TestShrexGetter(t *testing.T) { @@ -46,8 +47,6 @@ func TestShrexGetter(t *testing.T) { // launch eds store and put test data into it edsStore, err := newStore(t) require.NoError(t, err) - err = edsStore.Start(ctx) - require.NoError(t, err) ndClient, _ := newNDClientServer(ctx, t, edsStore, srvHost, clHost) edsClient, _ := newEDSClientServer(ctx, t, edsStore, srvHost, clHost) @@ -60,9 +59,12 @@ func TestShrexGetter(t *testing.T) { archivalPeerManager, err := testManager(ctx, clHost, sub) require.NoError(t, err) - getter := NewShrexGetter(edsClient, ndClient, fullPeerManager, archivalPeerManager, light.Window) + getter := NewGetter(edsClient, ndClient, fullPeerManager, archivalPeerManager, light.Window) require.NoError(t, getter.Start(ctx)) + height := atomic.Uint64{} + height.Add(1) + t.Run("ND_Available, total data size > 1mb", func(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second*10) t.Cleanup(cancel) @@ -70,17 +72,21 @@ func TestShrexGetter(t *testing.T) { // generate test data size := 64 namespace := sharetest.RandV0Namespace() + height := height.Add(1) randEDS, roots := edstest.RandEDSWithNamespace(t, namespace, size*size, size) eh := headertest.RandExtendedHeaderWithRoot(t, roots) - require.NoError(t, edsStore.Put(ctx, roots.Hash(), randEDS)) + eh.RawHeader.Height = int64(height) + + err = edsStore.Put(ctx, roots, height, randEDS) + require.NoError(t, err) fullPeerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: roots.Hash(), - Height: 1, + Height: height, }) got, err := getter.GetSharesByNamespace(ctx, eh, namespace) require.NoError(t, err) - require.NoError(t, got.Verify(roots, namespace)) + require.NoError(t, got.Validate(roots, namespace)) }) t.Run("ND_err_not_found", func(t *testing.T) { @@ -88,15 +94,18 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data + height := height.Add(1) _, roots, namespace := generateTestEDS(t) eh := headertest.RandExtendedHeaderWithRoot(t, roots) + eh.RawHeader.Height = int64(height) + fullPeerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: roots.Hash(), - Height: 1, + Height: height, }) _, err := getter.GetSharesByNamespace(ctx, eh, namespace) - require.ErrorIs(t, err, share.ErrNotFound) + require.ErrorIs(t, err, shwap.ErrNotFound) }) t.Run("ND_namespace_not_included", func(t *testing.T) { @@ -104,37 +113,41 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data + height := height.Add(1) eds, roots, maxNamespace := generateTestEDS(t) eh := headertest.RandExtendedHeaderWithRoot(t, roots) - require.NoError(t, edsStore.Put(ctx, roots.Hash(), eds)) + eh.RawHeader.Height = int64(height) + + err = edsStore.Put(ctx, roots, height, eds) + require.NoError(t, err) fullPeerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: roots.Hash(), - Height: 1, + Height: height, }) // namespace inside root range nID, err := addToNamespace(maxNamespace, -1) require.NoError(t, err) // check for namespace to be between max and min namespace in root - require.Len(t, ipld.FilterRootByNamespace(roots, nID), 1) + require.Len(t, share.RowsWithNamespace(roots, nID), 1) emptyShares, err := getter.GetSharesByNamespace(ctx, eh, nID) require.NoError(t, err) // no shares should be returned require.Nil(t, emptyShares.Flatten()) - require.Nil(t, emptyShares.Verify(roots, nID)) + require.Nil(t, emptyShares.Validate(roots, nID)) // namespace outside root range nID, err = addToNamespace(maxNamespace, 1) require.NoError(t, err) // check for namespace to be not in root - require.Len(t, ipld.FilterRootByNamespace(roots, nID), 0) + require.Len(t, share.RowsWithNamespace(roots, nID), 0) emptyShares, err = getter.GetSharesByNamespace(ctx, eh, nID) require.NoError(t, err) // no shares should be returned require.Nil(t, emptyShares.Flatten()) - require.Nil(t, emptyShares.Verify(roots, nID)) + require.Nil(t, emptyShares.Validate(roots, nID)) }) t.Run("ND_namespace_not_in_dah", func(t *testing.T) { @@ -143,23 +156,27 @@ func TestShrexGetter(t *testing.T) { // generate test data eds, roots, maxNamespace := generateTestEDS(t) + height := height.Add(1) eh := headertest.RandExtendedHeaderWithRoot(t, roots) - require.NoError(t, edsStore.Put(ctx, roots.Hash(), eds)) + eh.RawHeader.Height = int64(height) + + err = edsStore.Put(ctx, roots, height, eds) + require.NoError(t, err) fullPeerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: roots.Hash(), - Height: 1, + Height: height, }) namespace, err := addToNamespace(maxNamespace, 1) require.NoError(t, err) // check for namespace to be not in root - require.Len(t, ipld.FilterRootByNamespace(roots, namespace), 0) + require.Len(t, share.RowsWithNamespace(roots, namespace), 0) emptyShares, err := getter.GetSharesByNamespace(ctx, eh, namespace) require.NoError(t, err) // no shares should be returned require.Empty(t, emptyShares.Flatten()) - require.Nil(t, emptyShares.Verify(roots, namespace)) + require.Nil(t, emptyShares.Validate(roots, namespace)) }) t.Run("EDS_Available", func(t *testing.T) { @@ -168,11 +185,15 @@ func TestShrexGetter(t *testing.T) { // generate test data randEDS, roots, _ := generateTestEDS(t) + height := height.Add(1) eh := headertest.RandExtendedHeaderWithRoot(t, roots) - require.NoError(t, edsStore.Put(ctx, roots.Hash(), randEDS)) + eh.RawHeader.Height = int64(height) + + err = edsStore.Put(ctx, roots, height, randEDS) + require.NoError(t, err) fullPeerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: roots.Hash(), - Height: 1, + Height: height, }) got, err := getter.GetEDS(ctx, eh) @@ -185,10 +206,13 @@ func TestShrexGetter(t *testing.T) { // generate test data _, roots, _ := generateTestEDS(t) + height := height.Add(1) eh := headertest.RandExtendedHeaderWithRoot(t, roots) + eh.RawHeader.Height = int64(height) + fullPeerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: roots.Hash(), - Height: 1, + Height: height, }) cancel() @@ -202,14 +226,17 @@ func TestShrexGetter(t *testing.T) { // generate test data _, roots, _ := generateTestEDS(t) + height := height.Add(1) eh := headertest.RandExtendedHeaderWithRoot(t, roots) + eh.RawHeader.Height = int64(height) + fullPeerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: roots.Hash(), - Height: 1, + Height: height, }) _, err := getter.GetEDS(ctx, eh) - require.ErrorIs(t, err, share.ErrNotFound) + require.ErrorIs(t, err, shwap.ErrNotFound) }) // tests getPeer's ability to route requests based on whether @@ -226,7 +253,9 @@ func TestShrexGetter(t *testing.T) { getter.archivalPeerManager.UpdateNodePool(archivalPeer.ID(), true) getter.fullPeerManager.UpdateNodePool(fullPeer.ID(), true) + height := height.Add(1) eh := headertest.RandExtendedHeader(t) + eh.RawHeader.Height = int64(height) // historical data expects an archival peer eh.RawHeader.Time = time.Now().Add(-(time.Duration(full.Window) + time.Second)) @@ -242,11 +271,10 @@ func TestShrexGetter(t *testing.T) { }) } -func newStore(t *testing.T) (*eds.Store, error) { +func newStore(t *testing.T) (*store.Store, error) { t.Helper() - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - return eds.NewStore(eds.DefaultParameters(), t.TempDir(), ds) + return store.NewStore(store.DefaultParameters(), t.TempDir()) } func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.AxisRoots, share.Namespace) { @@ -280,7 +308,7 @@ func testManager( } func newNDClientServer( - ctx context.Context, t *testing.T, edsStore *eds.Store, srvHost, clHost host.Host, + ctx context.Context, t *testing.T, edsStore *store.Store, srvHost, clHost host.Host, ) (*shrexnd.Client, *shrexnd.Server) { params := shrexnd.DefaultParameters() @@ -300,7 +328,7 @@ func newNDClientServer( } func newEDSClientServer( - ctx context.Context, t *testing.T, edsStore *eds.Store, srvHost, clHost host.Host, + ctx context.Context, t *testing.T, edsStore *store.Store, srvHost, clHost host.Host, ) (*shrexeds.Client, *shrexeds.Server) { params := shrexeds.DefaultParameters() diff --git a/share/p2p/shrexeds/client.go b/share/shwap/p2p/shrex/shrexeds/client.go similarity index 56% rename from share/p2p/shrexeds/client.go rename to share/shwap/p2p/shrex/shrexeds/client.go index d56e0e20f5..200c169d92 100644 --- a/share/p2p/shrexeds/client.go +++ b/share/shwap/p2p/shrex/shrexeds/client.go @@ -1,6 +1,7 @@ package shrexeds import ( + "bytes" "context" "errors" "fmt" @@ -16,10 +17,12 @@ import ( "github.com/celestiaorg/go-libp2p-messenger/serde" "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/p2p" - pb "github.com/celestiaorg/celestia-node/share/p2p/shrexeds/pb" + eds "github.com/celestiaorg/celestia-node/share/new_eds" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" + shrexpb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/pb" ) // Client is responsible for requesting EDSs for blocksync over the ShrEx/EDS protocol. @@ -28,7 +31,7 @@ type Client struct { protocolID protocol.ID host host.Host - metrics *p2p.Metrics + metrics *shrex.Metrics } // NewClient creates a new ShrEx/EDS client. @@ -40,23 +43,27 @@ func NewClient(params *Parameters, host host.Host) (*Client, error) { return &Client{ params: params, host: host, - protocolID: p2p.ProtocolID(params.NetworkID(), protocolString), + protocolID: shrex.ProtocolID(params.NetworkID(), protocolString), }, nil } // RequestEDS requests the ODS from the given peers and returns the EDS upon success. func (c *Client) RequestEDS( ctx context.Context, - dataHash share.DataHash, + root *share.AxisRoots, + height uint64, peer peer.ID, ) (*rsmt2d.ExtendedDataSquare, error) { - eds, err := c.doRequest(ctx, dataHash, peer) + eds, err := c.doRequest(ctx, root, height, peer) if err == nil { return eds, nil } - log.Debugw("client: eds request to peer failed", "peer", peer.String(), "hash", dataHash.String(), "error", err) + log.Debugw("client: eds request to peer failed", + "height", height, + "peer", peer.String(), + "error", err) if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { - c.metrics.ObserveRequests(ctx, 1, p2p.StatusTimeout) + c.metrics.ObserveRequests(ctx, 1, shrex.StatusTimeout) return nil, err } // some net.Errors also mean the context deadline was exceeded, but yamux/mocknet do not @@ -64,14 +71,14 @@ func (c *Client) RequestEDS( var ne net.Error if errors.As(err, &ne) && ne.Timeout() { if deadline, _ := ctx.Deadline(); deadline.Before(time.Now()) { - c.metrics.ObserveRequests(ctx, 1, p2p.StatusTimeout) + c.metrics.ObserveRequests(ctx, 1, shrex.StatusTimeout) return nil, context.DeadlineExceeded } } - if !errors.Is(err, p2p.ErrNotFound) { + if !errors.Is(err, shrex.ErrNotFound) { log.Warnw("client: eds request to peer failed", "peer", peer.String(), - "hash", dataHash.String(), + "height", height, "err", err) } @@ -80,35 +87,39 @@ func (c *Client) RequestEDS( func (c *Client) doRequest( ctx context.Context, - dataHash share.DataHash, + root *share.AxisRoots, + height uint64, to peer.ID, ) (*rsmt2d.ExtendedDataSquare, error) { streamOpenCtx, cancel := context.WithTimeout(ctx, c.params.ServerReadTimeout) defer cancel() stream, err := c.host.NewStream(streamOpenCtx, to, c.protocolID) if err != nil { - return nil, fmt.Errorf("failed to open stream: %w", err) + return nil, fmt.Errorf("open stream: %w", err) } - defer stream.Close() + defer utils.CloseAndLog(log, "client", stream) c.setStreamDeadlines(ctx, stream) - - req := &pb.EDSRequest{Hash: dataHash} - // request ODS - log.Debugw("client: requesting ods", "hash", dataHash.String(), "peer", to.String()) - _, err = serde.Write(stream, req) + log.Debugw("client: requesting ods", + "height", height, + "peer", to.String()) + id, err := shwap.NewEdsID(height) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + _, err = id.WriteTo(stream) if err != nil { - stream.Reset() //nolint:errcheck - return nil, fmt.Errorf("failed to write request to stream: %w", err) + return nil, fmt.Errorf("write request to stream: %w", err) } + err = stream.CloseWrite() if err != nil { - log.Debugw("client: error closing write", "err", err) + log.Warnw("client: error closing write", "err", err) } // read and parse status from peer - resp := new(pb.EDSResponse) + resp := new(shrexpb.Response) err = stream.SetReadDeadline(time.Now().Add(c.params.ServerReadTimeout)) if err != nil { log.Debugw("client: failed to set read deadline for reading status", "err", err) @@ -117,36 +128,60 @@ func (c *Client) doRequest( if err != nil { // server closes the stream here if we are rate limited if errors.Is(err, io.EOF) { - c.metrics.ObserveRequests(ctx, 1, p2p.StatusRateLimited) - return nil, p2p.ErrNotFound + c.metrics.ObserveRequests(ctx, 1, shrex.StatusRateLimited) + return nil, shrex.ErrNotFound } - stream.Reset() //nolint:errcheck - return nil, fmt.Errorf("failed to read status from stream: %w", err) + return nil, fmt.Errorf("read status from stream: %w", err) } - switch resp.Status { - case pb.Status_OK: + case shrexpb.Status_OK: // reset stream deadlines to original values, since read deadline was changed during status read c.setStreamDeadlines(ctx, stream) // use header and ODS bytes to construct EDS and verify it against dataHash - eds, err := eds.ReadEDS(ctx, stream, dataHash) + eds, err := readEds(ctx, stream, root) if err != nil { - return nil, fmt.Errorf("failed to read eds from ods bytes: %w", err) + return nil, fmt.Errorf("read eds from stream: %w", err) } - c.metrics.ObserveRequests(ctx, 1, p2p.StatusSuccess) + c.metrics.ObserveRequests(ctx, 1, shrex.StatusSuccess) return eds, nil - case pb.Status_NOT_FOUND: - c.metrics.ObserveRequests(ctx, 1, p2p.StatusNotFound) - return nil, p2p.ErrNotFound - case pb.Status_INVALID: + case shrexpb.Status_NOT_FOUND: + c.metrics.ObserveRequests(ctx, 1, shrex.StatusNotFound) + return nil, shrex.ErrNotFound + case shrexpb.Status_INVALID: log.Debug("client: invalid request") fallthrough - case pb.Status_INTERNAL: + case shrexpb.Status_INTERNAL: fallthrough default: - c.metrics.ObserveRequests(ctx, 1, p2p.StatusInternalErr) - return nil, p2p.ErrInvalidResponse + c.metrics.ObserveRequests(ctx, 1, shrex.StatusInternalErr) + return nil, shrex.ErrInvalidResponse + } +} + +func readEds(ctx context.Context, stream network.Stream, root *share.AxisRoots) (*rsmt2d.ExtendedDataSquare, error) { + odsSize := len(root.RowRoots) / 2 + shares, err := eds.ReadShares(stream, share.Size, odsSize) + if err != nil { + return nil, fmt.Errorf("failed to read eds from ods bytes: %w", err) + } + + // verify that the EDS hash matches the expected hash + rsmt2d, err := eds.Rsmt2DFromShares(shares, odsSize) + if err != nil { + return nil, fmt.Errorf("failed to create rsmt2d from shares: %w", err) + } + datahash, err := rsmt2d.DataHash(ctx) + if err != nil { + return nil, fmt.Errorf("failed to calculate data hash: %w", err) + } + if !bytes.Equal(datahash, root.Hash()) { + return nil, fmt.Errorf( + "content integrity mismatch: imported root %s doesn't match expected root %s", + datahash, + root.Hash(), + ) } + return rsmt2d.ExtendedDataSquare, nil } func (c *Client) setStreamDeadlines(ctx context.Context, stream network.Stream) { diff --git a/share/p2p/shrexeds/doc.go b/share/shwap/p2p/shrex/shrexeds/doc.go similarity index 100% rename from share/p2p/shrexeds/doc.go rename to share/shwap/p2p/shrex/shrexeds/doc.go diff --git a/share/p2p/shrexeds/exchange_test.go b/share/shwap/p2p/shrex/shrexeds/exchange_test.go similarity index 68% rename from share/p2p/shrexeds/exchange_test.go rename to share/shwap/p2p/shrex/shrexeds/exchange_test.go index 450ad291f0..54fa585a70 100644 --- a/share/p2p/shrexeds/exchange_test.go +++ b/share/shwap/p2p/shrex/shrexeds/exchange_test.go @@ -6,8 +6,6 @@ import ( "testing" "time" - "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" libhost "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" @@ -15,9 +13,10 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/p2p" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" + "github.com/celestiaorg/celestia-node/store" ) func TestExchange_RequestEDS(t *testing.T) { @@ -25,10 +24,7 @@ func TestExchange_RequestEDS(t *testing.T) { t.Cleanup(cancel) store, client, server := makeExchange(t) - err := store.Start(ctx) - require.NoError(t, err) - - err = server.Start(ctx) + err := server.Start(ctx) require.NoError(t, err) // Testcase: EDS is immediately available @@ -36,10 +32,11 @@ func TestExchange_RequestEDS(t *testing.T) { eds := edstest.RandEDS(t, 4) roots, err := share.NewAxisRoots(eds) require.NoError(t, err) - err = store.Put(ctx, roots.Hash(), eds) + height := uint64(1) + err = store.Put(ctx, roots, height, eds) require.NoError(t, err) - requestedEDS, err := client.RequestEDS(ctx, roots.Hash(), server.host.ID()) + requestedEDS, err := client.RequestEDS(ctx, roots, height, server.host.ID()) assert.NoError(t, err) assert.Equal(t, eds.Flattened(), requestedEDS.Flattened()) }) @@ -49,17 +46,18 @@ func TestExchange_RequestEDS(t *testing.T) { eds := edstest.RandEDS(t, 4) roots, err := share.NewAxisRoots(eds) require.NoError(t, err) + height := uint64(666) lock := make(chan struct{}) go func() { <-lock - err = store.Put(ctx, roots.Hash(), eds) + err := store.Put(ctx, roots, height, eds) require.NoError(t, err) lock <- struct{}{} }() - requestedEDS, err := client.RequestEDS(ctx, roots.Hash(), server.host.ID()) - assert.ErrorIs(t, err, p2p.ErrNotFound) + requestedEDS, err := client.RequestEDS(ctx, roots, height, server.host.ID()) + assert.ErrorIs(t, err, shrex.ErrNotFound) assert.Nil(t, requestedEDS) // unlock write @@ -67,16 +65,17 @@ func TestExchange_RequestEDS(t *testing.T) { // wait for write to finish <-lock - requestedEDS, err = client.RequestEDS(ctx, roots.Hash(), server.host.ID()) + requestedEDS, err = client.RequestEDS(ctx, roots, height, server.host.ID()) assert.NoError(t, err) assert.Equal(t, eds.Flattened(), requestedEDS.Flattened()) }) // Testcase: Invalid request excludes peer from round-robin, stopping request t.Run("EDS_InvalidRequest", func(t *testing.T) { - dataHash := []byte("invalid") - requestedEDS, err := client.RequestEDS(ctx, dataHash, server.host.ID()) - assert.ErrorContains(t, err, "stream reset") + emptyRoot := share.EmptyEDSRoots() + height := uint64(0) + requestedEDS, err := client.RequestEDS(ctx, emptyRoot, height, server.host.ID()) + assert.ErrorIs(t, err, shwap.ErrInvalidShwapID) assert.Nil(t, requestedEDS) }) @@ -86,15 +85,15 @@ func TestExchange_RequestEDS(t *testing.T) { eds := edstest.RandEDS(t, 4) roots, err := share.NewAxisRoots(eds) require.NoError(t, err) - _, err = client.RequestEDS(timeoutCtx, roots.Hash(), server.host.ID()) - require.ErrorIs(t, err, p2p.ErrNotFound) + height := uint64(668) + _, err = client.RequestEDS(timeoutCtx, roots, height, server.host.ID()) + require.ErrorIs(t, err, shrex.ErrNotFound) }) // Testcase: Concurrency limit reached t.Run("EDS_concurrency_limit", func(t *testing.T) { - store, client, server := makeExchange(t) + _, client, server := makeExchange(t) - require.NoError(t, store.Start(ctx)) require.NoError(t, server.Start(ctx)) ctx, cancel := context.WithTimeout(ctx, time.Second) @@ -115,34 +114,25 @@ func TestExchange_RequestEDS(t *testing.T) { t.Fatal("timeout") } } - middleware := p2p.NewMiddleware(rateLimit) + middleware := shrex.NewMiddleware(rateLimit) server.host.SetStreamHandler(server.protocolID, middleware.RateLimitHandler(mockHandler)) // take server concurrency slots with blocked requests + emptyRoot := share.EmptyEDSRoots() for i := 0; i < rateLimit; i++ { go func(i int) { - client.RequestEDS(ctx, nil, server.host.ID()) //nolint:errcheck + client.RequestEDS(ctx, emptyRoot, 1, server.host.ID()) //nolint:errcheck }(i) } // wait until all server slots are taken wg.Wait() - _, err = client.RequestEDS(ctx, nil, server.host.ID()) - require.ErrorIs(t, err, p2p.ErrNotFound) + _, err = client.RequestEDS(ctx, emptyRoot, 1, server.host.ID()) + require.ErrorIs(t, err, shrex.ErrNotFound) }) } -func newStore(t *testing.T) *eds.Store { - t.Helper() - - storeCfg := eds.DefaultParameters() - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - store, err := eds.NewStore(storeCfg, t.TempDir(), ds) - require.NoError(t, err) - return store -} - func createMocknet(t *testing.T, amount int) []libhost.Host { t.Helper() @@ -152,9 +142,10 @@ func createMocknet(t *testing.T, amount int) []libhost.Host { return net.Hosts() } -func makeExchange(t *testing.T) (*eds.Store, *Client, *Server) { +func makeExchange(t *testing.T) (*store.Store, *Client, *Server) { t.Helper() - store := newStore(t) + store, err := store.NewStore(store.DefaultParameters(), t.TempDir()) + require.NoError(t, err) hosts := createMocknet(t, 2) client, err := NewClient(DefaultParameters(), hosts[0]) diff --git a/share/p2p/shrexeds/params.go b/share/shwap/p2p/shrex/shrexeds/params.go similarity index 73% rename from share/p2p/shrexeds/params.go rename to share/shwap/p2p/shrex/shrexeds/params.go index 795cb313ed..c5d76fecb2 100644 --- a/share/p2p/shrexeds/params.go +++ b/share/shwap/p2p/shrex/shrexeds/params.go @@ -5,16 +5,17 @@ import ( logging "github.com/ipfs/go-log/v2" - "github.com/celestiaorg/celestia-node/share/p2p" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" ) -const protocolString = "/shrex/eds/v0.0.1" +const protocolString = shrex.ProtocolString + shwap.EDSName var log = logging.Logger("shrex/eds") // Parameters is the set of parameters that must be configured for the shrex/eds protocol. type Parameters struct { - *p2p.Parameters + *shrex.Parameters // BufferSize defines the size of the buffer used for writing an ODS over the stream. BufferSize uint64 @@ -22,7 +23,7 @@ type Parameters struct { func DefaultParameters() *Parameters { return &Parameters{ - Parameters: p2p.DefaultParameters(), + Parameters: shrex.DefaultParameters(), BufferSize: 32 * 1024, } } @@ -36,7 +37,7 @@ func (p *Parameters) Validate() error { } func (c *Client) WithMetrics() error { - metrics, err := p2p.InitClientMetrics("eds") + metrics, err := shrex.InitClientMetrics("eds") if err != nil { return fmt.Errorf("shrex/eds: init Metrics: %w", err) } @@ -45,7 +46,7 @@ func (c *Client) WithMetrics() error { } func (s *Server) WithMetrics() error { - metrics, err := p2p.InitServerMetrics("eds") + metrics, err := shrex.InitServerMetrics("eds") if err != nil { return fmt.Errorf("shrex/eds: init Metrics: %w", err) } diff --git a/share/shwap/p2p/shrex/shrexeds/server.go b/share/shwap/p2p/shrex/shrexeds/server.go new file mode 100644 index 0000000000..799c8a09a5 --- /dev/null +++ b/share/shwap/p2p/shrex/shrexeds/server.go @@ -0,0 +1,190 @@ +package shrexeds + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/protocol" + "go.uber.org/zap" + + "github.com/celestiaorg/go-libp2p-messenger/serde" + + "github.com/celestiaorg/celestia-node/libs/utils" + eds "github.com/celestiaorg/celestia-node/share/new_eds" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" + shrexpb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/pb" + "github.com/celestiaorg/celestia-node/store" +) + +// Server is responsible for serving ODSs for blocksync over the ShrEx/EDS protocol. +type Server struct { + cancel context.CancelFunc + + host host.Host + protocolID protocol.ID + + store *store.Store + + params *Parameters + middleware *shrex.Middleware + metrics *shrex.Metrics +} + +// NewServer creates a new ShrEx/EDS server. +func NewServer(params *Parameters, host host.Host, store *store.Store) (*Server, error) { + if err := params.Validate(); err != nil { + return nil, fmt.Errorf("shrex-eds: server creation failed: %w", err) + } + + return &Server{ + host: host, + store: store, + protocolID: shrex.ProtocolID(params.NetworkID(), protocolString), + params: params, + middleware: shrex.NewMiddleware(params.ConcurrencyLimit), + }, nil +} + +func (s *Server) Start(context.Context) error { + ctx, cancel := context.WithCancel(context.Background()) + s.cancel = cancel + + s.host.SetStreamHandler(s.protocolID, s.middleware.RateLimitHandler(s.streamHandler(ctx))) + return nil +} + +func (s *Server) Stop(context.Context) error { + defer s.cancel() + s.host.RemoveStreamHandler(s.protocolID) + return nil +} + +func (s *Server) observeRateLimitedRequests() { + numRateLimited := s.middleware.DrainCounter() + if numRateLimited > 0 { + s.metrics.ObserveRequests(context.Background(), numRateLimited, shrex.StatusRateLimited) + } +} + +func (srv *Server) streamHandler(ctx context.Context) network.StreamHandler { + return func(s network.Stream) { + err := srv.handleEDS(ctx, s) + if err != nil { + s.Reset() //nolint:errcheck + return + } + srv.metrics.ObserveRequests(ctx, 1, shrex.StatusSuccess) + if err = s.Close(); err != nil { + log.Debugw("server: closing stream", "err", err) + } + } +} + +func (s *Server) handleEDS(ctx context.Context, stream network.Stream) error { + logger := log.With("peer", stream.Conn().RemotePeer().String()) + logger.Debug("server: handling eds request") + + // read request from stream to get the dataHash for store lookup + id, err := s.readRequest(logger, stream) + if err != nil { + logger.Warnw("server: reading request from stream", "err", err) + return err + } + + logger = logger.With("height", id.Height) + + ctx, cancel := context.WithTimeout(ctx, s.params.HandleRequestTimeout) + defer cancel() + + // determine whether the EDS is available in our store + // we do not close the reader, so that other requests will not need to re-open the file. + // closing is handled by the LRU cache. + file, err := s.store.GetByHeight(ctx, id.Height) + var status shrexpb.Status + switch { + case err == nil: + defer utils.CloseAndLog(logger, "file", file) + status = shrexpb.Status_OK + case errors.Is(err, store.ErrNotFound): + logger.Warnw("server: request height not found") + s.metrics.ObserveRequests(ctx, 1, shrex.StatusNotFound) + status = shrexpb.Status_NOT_FOUND + case err != nil: + logger.Errorw("server: get file", "err", err) + status = shrexpb.Status_INTERNAL + } + + // inform the client of our status + err = s.writeStatus(logger, status, stream) + if err != nil { + logger.Warnw("server: writing status to stream", "err", err) + return err + } + // if we cannot serve the EDS, we are already done + if status != shrexpb.Status_OK { + return nil + } + + // start streaming the ODS to the client + err = s.writeODS(logger, file, stream) + if err != nil { + logger.Warnw("server: writing ods to stream", "err", err) + return err + } + return nil +} + +func (s *Server) readRequest(logger *zap.SugaredLogger, stream network.Stream) (shwap.EdsID, error) { + err := stream.SetReadDeadline(time.Now().Add(s.params.ServerReadTimeout)) + if err != nil { + logger.Debugw("server: set read deadline", "err", err) + } + + edsIds := shwap.EdsID{} + _, err = edsIds.ReadFrom(stream) + if err != nil { + return shwap.EdsID{}, fmt.Errorf("reading request: %w", err) + } + err = stream.CloseRead() + if err != nil { + logger.Warnw("server: closing read", "err", err) + } + return edsIds, nil +} + +func (s *Server) writeStatus(logger *zap.SugaredLogger, status shrexpb.Status, stream network.Stream) error { + err := stream.SetWriteDeadline(time.Now().Add(s.params.ServerWriteTimeout)) + if err != nil { + logger.Debugw("server: set write deadline", "err", err) + } + + resp := &shrexpb.Response{Status: status} + _, err = serde.Write(stream, resp) + return err +} + +func (s *Server) writeODS(logger *zap.SugaredLogger, streamer eds.Streamer, stream network.Stream) error { + reader, err := streamer.Reader() + if err != nil { + return fmt.Errorf("getting ODS reader: %w", err) + } + err = stream.SetWriteDeadline(time.Now().Add(s.params.ServerWriteTimeout)) + if err != nil { + logger.Debugw("server: set read deadline", "err", err) + } + + buf := make([]byte, s.params.BufferSize) + n, err := io.CopyBuffer(stream, reader, buf) + if err != nil { + return fmt.Errorf("written: %v, writing ODS bytes: %w", n, err) + } + + logger.Debugw("server: wrote ODS", "bytes", n) + return nil +} diff --git a/share/p2p/shrexnd/client.go b/share/shwap/p2p/shrex/shrexnd/client.go similarity index 56% rename from share/p2p/shrexnd/client.go rename to share/shwap/p2p/shrex/shrexnd/client.go index ad4d3a37a7..a7029a348a 100644 --- a/share/p2p/shrexnd/client.go +++ b/share/shwap/p2p/shrex/shrexnd/client.go @@ -14,11 +14,12 @@ import ( "github.com/libp2p/go-libp2p/core/protocol" "github.com/celestiaorg/go-libp2p-messenger/serde" - "github.com/celestiaorg/nmt" + "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/p2p" - pb "github.com/celestiaorg/celestia-node/share/p2p/shrexnd/pb" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" + shrexpb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/pb" ) // Client implements client side of shrex/nd protocol to obtain namespaced shares data from remote @@ -28,7 +29,7 @@ type Client struct { protocolID protocol.ID host host.Host - metrics *p2p.Metrics + metrics *shrex.Metrics } // NewClient creates a new shrEx/nd client @@ -39,29 +40,29 @@ func NewClient(params *Parameters, host host.Host) (*Client, error) { return &Client{ host: host, - protocolID: p2p.ProtocolID(params.NetworkID(), protocolString), + protocolID: shrex.ProtocolID(params.NetworkID(), protocolString), params: params, }, nil } // RequestND requests namespaced data from the given peer. -// Returns NamespacedShares with unverified inclusion proofs against the share.AxisRoots. +// Returns NamespaceData with unverified inclusion proofs against the share.Root. func (c *Client) RequestND( ctx context.Context, - root *share.AxisRoots, + height uint64, namespace share.Namespace, peer peer.ID, -) (share.NamespacedShares, error) { +) (shwap.NamespaceData, error) { if err := namespace.ValidateForData(); err != nil { return nil, err } - shares, err := c.doRequest(ctx, root, namespace, peer) + shares, err := c.doRequest(ctx, height, namespace, peer) if err == nil { return shares, nil } if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { - c.metrics.ObserveRequests(ctx, 1, p2p.StatusTimeout) + c.metrics.ObserveRequests(ctx, 1, shrex.StatusTimeout) return nil, err } // some net.Errors also mean the context deadline was exceeded, but yamux/mocknet do not @@ -69,11 +70,11 @@ func (c *Client) RequestND( var ne net.Error if errors.As(err, &ne) && ne.Timeout() { if deadline, _ := ctx.Deadline(); deadline.Before(time.Now()) { - c.metrics.ObserveRequests(ctx, 1, p2p.StatusTimeout) + c.metrics.ObserveRequests(ctx, 1, shrex.StatusTimeout) return nil, context.DeadlineExceeded } } - if !errors.Is(err, p2p.ErrNotFound) && !errors.Is(err, p2p.ErrRateLimited) { + if !errors.Is(err, shrex.ErrNotFound) && errors.Is(err, shrex.ErrRateLimited) { log.Warnw("client-nd: peer returned err", "err", err) } return nil, err @@ -81,51 +82,60 @@ func (c *Client) RequestND( func (c *Client) doRequest( ctx context.Context, - root *share.AxisRoots, + height uint64, namespace share.Namespace, peerID peer.ID, -) (share.NamespacedShares, error) { - stream, err := c.host.NewStream(ctx, peerID, c.protocolID) +) (shwap.NamespaceData, error) { + streamOpenCtx, cancel := context.WithTimeout(ctx, c.params.ServerReadTimeout) + defer cancel() + stream, err := c.host.NewStream(streamOpenCtx, peerID, c.protocolID) if err != nil { return nil, err } - defer stream.Close() + defer utils.CloseAndLog(log, "client", stream) c.setStreamDeadlines(ctx, stream) - req := &pb.GetSharesByNamespaceRequest{ - RootHash: root.Hash(), - Namespace: namespace, + req, err := shwap.NewNamespaceDataID(height, namespace) + if err != nil { + return nil, fmt.Errorf("client-nd: creating request: %w", err) } - _, err = serde.Write(stream, req) + _, err = req.WriteTo(stream) if err != nil { - c.metrics.ObserveRequests(ctx, 1, p2p.StatusSendReqErr) - stream.Reset() //nolint:errcheck + c.metrics.ObserveRequests(ctx, 1, shrex.StatusSendReqErr) return nil, fmt.Errorf("client-nd: writing request: %w", err) } err = stream.CloseWrite() if err != nil { - log.Debugw("client-nd: closing write side of the stream", "err", err) + log.Warnw("client-nd: closing write side of the stream", "err", err) } if err := c.readStatus(ctx, stream); err != nil { + c.metrics.ObserveRequests(ctx, 1, shrex.StatusReadRespErr) return nil, err } - return c.readNamespacedShares(ctx, stream) + + nd := shwap.NamespaceData{} + _, err = nd.ReadFrom(stream) + if err != nil { + c.metrics.ObserveRequests(ctx, 1, shrex.StatusReadRespErr) + return nil, err + } + return nd, nil } func (c *Client) readStatus(ctx context.Context, stream network.Stream) error { - var resp pb.GetSharesByNamespaceStatusResponse + var resp shrexpb.Response _, err := serde.Read(stream, &resp) if err != nil { // server is overloaded and closed the stream if errors.Is(err, io.EOF) { - c.metrics.ObserveRequests(ctx, 1, p2p.StatusRateLimited) - return p2p.ErrRateLimited + c.metrics.ObserveRequests(ctx, 1, shrex.StatusRateLimited) + return shrex.ErrRateLimited } - c.metrics.ObserveRequests(ctx, 1, p2p.StatusReadRespErr) + c.metrics.ObserveRequests(ctx, 1, shrex.StatusReadRespErr) stream.Reset() //nolint:errcheck return fmt.Errorf("client-nd: reading status response: %w", err) } @@ -133,49 +143,6 @@ func (c *Client) readStatus(ctx context.Context, stream network.Stream) error { return c.convertStatusToErr(ctx, resp.Status) } -// readNamespacedShares converts proto Rows to share.NamespacedShares -func (c *Client) readNamespacedShares( - ctx context.Context, - stream network.Stream, -) (share.NamespacedShares, error) { - var shares share.NamespacedShares - for { - var row pb.NamespaceRowResponse - _, err := serde.Read(stream, &row) - if err != nil { - if errors.Is(err, io.EOF) { - // all data is received and steam is closed by server - return shares, nil - } - c.metrics.ObserveRequests(ctx, 1, p2p.StatusReadRespErr) - return nil, err - } - var proof nmt.Proof - if row.Proof != nil { - if len(row.Shares) != 0 { - proof = nmt.NewInclusionProof( - int(row.Proof.Start), - int(row.Proof.End), - row.Proof.Nodes, - row.Proof.IsMaxNamespaceIgnored, - ) - } else { - proof = nmt.NewAbsenceProof( - int(row.Proof.Start), - int(row.Proof.End), - row.Proof.Nodes, - row.Proof.LeafHash, - row.Proof.IsMaxNamespaceIgnored, - ) - } - } - shares = append(shares, share.NamespacedRow{ - Shares: row.Shares, - Proof: &proof, - }) - } -} - func (c *Client) setStreamDeadlines(ctx context.Context, stream network.Stream) { // set read/write deadline to use context deadline if it exists deadline, ok := ctx.Deadline() @@ -204,20 +171,20 @@ func (c *Client) setStreamDeadlines(ctx context.Context, stream network.Stream) } } -func (c *Client) convertStatusToErr(ctx context.Context, status pb.StatusCode) error { +func (c *Client) convertStatusToErr(ctx context.Context, status shrexpb.Status) error { switch status { - case pb.StatusCode_OK: - c.metrics.ObserveRequests(ctx, 1, p2p.StatusSuccess) + case shrexpb.Status_OK: + c.metrics.ObserveRequests(ctx, 1, shrex.StatusSuccess) return nil - case pb.StatusCode_NOT_FOUND: - c.metrics.ObserveRequests(ctx, 1, p2p.StatusNotFound) - return p2p.ErrNotFound - case pb.StatusCode_INVALID: + case shrexpb.Status_NOT_FOUND: + c.metrics.ObserveRequests(ctx, 1, shrex.StatusNotFound) + return shrex.ErrNotFound + case shrexpb.Status_INVALID: log.Warn("client-nd: invalid request") fallthrough - case pb.StatusCode_INTERNAL: + case shrexpb.Status_INTERNAL: fallthrough default: - return p2p.ErrInvalidResponse + return shrex.ErrInvalidResponse } } diff --git a/share/p2p/shrexnd/doc.go b/share/shwap/p2p/shrex/shrexnd/doc.go similarity index 100% rename from share/p2p/shrexnd/doc.go rename to share/shwap/p2p/shrex/shrexnd/doc.go diff --git a/share/p2p/shrexnd/exchange_test.go b/share/shwap/p2p/shrex/shrexnd/exchange_test.go similarity index 69% rename from share/p2p/shrexnd/exchange_test.go rename to share/shwap/p2p/shrex/shrexnd/exchange_test.go index e86f72e2fe..c49ebd867e 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/shwap/p2p/shrex/shrexnd/exchange_test.go @@ -3,38 +3,39 @@ package shrexnd import ( "context" "sync" + "sync/atomic" "testing" "time" - "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" libhost "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/p2p" "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" + "github.com/celestiaorg/celestia-node/store" ) func TestExchange_RequestND_NotFound(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) t.Cleanup(cancel) edsStore, client, server := makeExchange(t) - require.NoError(t, edsStore.Start(ctx)) require.NoError(t, server.Start(ctx)) + height := atomic.Uint64{} + height.Add(1) + t.Run("CAR_not_exist", func(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - root := share.AxisRoots{} namespace := sharetest.RandV0Namespace() - _, err := client.RequestND(ctx, &root, namespace, server.host.ID()) - require.ErrorIs(t, err, p2p.ErrNotFound) + height := height.Add(1) + _, err := client.RequestND(ctx, height, namespace, server.host.ID()) + require.ErrorIs(t, err, shrex.ErrNotFound) }) t.Run("ErrNamespaceNotFound", func(t *testing.T) { @@ -44,10 +45,13 @@ func TestExchange_RequestND_NotFound(t *testing.T) { eds := edstest.RandEDS(t, 4) roots, err := share.NewAxisRoots(eds) require.NoError(t, err) - require.NoError(t, edsStore.Put(ctx, roots.Hash(), eds)) + + height := height.Add(1) + err = edsStore.Put(ctx, roots, height, eds) + require.NoError(t, err) namespace := sharetest.RandV0Namespace() - emptyShares, err := client.RequestND(ctx, roots, namespace, server.host.ID()) + emptyShares, err := client.RequestND(ctx, height, namespace, server.host.ID()) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) }) @@ -83,34 +87,24 @@ func TestExchange_RequestND(t *testing.T) { t.Fatal("timeout") } } - middleware := p2p.NewMiddleware(rateLimit) + middleware := shrex.NewMiddleware(rateLimit) server.host.SetStreamHandler(server.protocolID, middleware.RateLimitHandler(mockHandler)) // take server concurrency slots with blocked requests for i := 0; i < rateLimit; i++ { go func(i int) { - client.RequestND(ctx, nil, sharetest.RandV0Namespace(), server.host.ID()) //nolint:errcheck + client.RequestND(ctx, 1, sharetest.RandV0Namespace(), server.host.ID()) //nolint:errcheck }(i) } // wait until all server slots are taken wg.Wait() - _, err = client.RequestND(ctx, nil, sharetest.RandV0Namespace(), server.host.ID()) - require.ErrorIs(t, err, p2p.ErrRateLimited) + _, err = client.RequestND(ctx, 1, sharetest.RandV0Namespace(), server.host.ID()) + require.ErrorIs(t, err, shrex.ErrRateLimited) }) } -func newStore(t *testing.T) *eds.Store { - t.Helper() - - storeCfg := eds.DefaultParameters() - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - store, err := eds.NewStore(storeCfg, t.TempDir(), ds) - require.NoError(t, err) - return store -} - func createMocknet(t *testing.T, amount int) []libhost.Host { t.Helper() @@ -120,15 +114,16 @@ func createMocknet(t *testing.T, amount int) []libhost.Host { return net.Hosts() } -func makeExchange(t *testing.T) (*eds.Store, *Client, *Server) { +func makeExchange(t *testing.T) (*store.Store, *Client, *Server) { t.Helper() - store := newStore(t) + s, err := store.NewStore(store.DefaultParameters(), t.TempDir()) + require.NoError(t, err) hosts := createMocknet(t, 2) client, err := NewClient(DefaultParameters(), hosts[0]) require.NoError(t, err) - server, err := NewServer(DefaultParameters(), hosts[1], store) + server, err := NewServer(DefaultParameters(), hosts[1], s) require.NoError(t, err) - return store, client, server + return s, client, server } diff --git a/share/p2p/shrexnd/params.go b/share/shwap/p2p/shrex/shrexnd/params.go similarity index 61% rename from share/p2p/shrexnd/params.go rename to share/shwap/p2p/shrex/shrexnd/params.go index 8489627a07..5544ae6b27 100644 --- a/share/p2p/shrexnd/params.go +++ b/share/shwap/p2p/shrex/shrexnd/params.go @@ -5,22 +5,23 @@ import ( logging "github.com/ipfs/go-log/v2" - "github.com/celestiaorg/celestia-node/share/p2p" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" ) -const protocolString = "/shrex/nd/v0.0.3" +const protocolString = shrex.ProtocolString + shwap.NamespaceDataName var log = logging.Logger("shrex/nd") // Parameters is the set of parameters that must be configured for the shrex/eds protocol. -type Parameters = p2p.Parameters +type Parameters = shrex.Parameters func DefaultParameters() *Parameters { - return p2p.DefaultParameters() + return shrex.DefaultParameters() } func (c *Client) WithMetrics() error { - metrics, err := p2p.InitClientMetrics("nd") + metrics, err := shrex.InitClientMetrics("nd") if err != nil { return fmt.Errorf("shrex/nd: init Metrics: %w", err) } @@ -29,7 +30,7 @@ func (c *Client) WithMetrics() error { } func (srv *Server) WithMetrics() error { - metrics, err := p2p.InitServerMetrics("nd") + metrics, err := shrex.InitServerMetrics("nd") if err != nil { return fmt.Errorf("shrex/nd: init Metrics: %w", err) } diff --git a/share/shwap/p2p/shrex/shrexnd/server.go b/share/shwap/p2p/shrex/shrexnd/server.go new file mode 100644 index 0000000000..39d6e291dc --- /dev/null +++ b/share/shwap/p2p/shrex/shrexnd/server.go @@ -0,0 +1,219 @@ +package shrexnd + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/protocol" + "go.uber.org/zap" + + "github.com/celestiaorg/go-libp2p-messenger/serde" + + "github.com/celestiaorg/celestia-node/libs/utils" + eds "github.com/celestiaorg/celestia-node/share/new_eds" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" + shrexpb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/pb" + "github.com/celestiaorg/celestia-node/store" +) + +// Server implements server side of shrex/nd protocol to serve namespaced share to remote +// peers. +type Server struct { + cancel context.CancelFunc + + host host.Host + protocolID protocol.ID + + handler network.StreamHandler + store *store.Store + + params *Parameters + middleware *shrex.Middleware + metrics *shrex.Metrics +} + +// NewServer creates new Server +func NewServer(params *Parameters, host host.Host, store *store.Store) (*Server, error) { + if err := params.Validate(); err != nil { + return nil, fmt.Errorf("shrex-nd: server creation failed: %w", err) + } + + srv := &Server{ + store: store, + host: host, + params: params, + protocolID: shrex.ProtocolID(params.NetworkID(), protocolString), + middleware: shrex.NewMiddleware(params.ConcurrencyLimit), + } + + ctx, cancel := context.WithCancel(context.Background()) + srv.cancel = cancel + + srv.handler = srv.middleware.RateLimitHandler(srv.streamHandler(ctx)) + return srv, nil +} + +// Start starts the server +func (srv *Server) Start(context.Context) error { + srv.host.SetStreamHandler(srv.protocolID, srv.handler) + return nil +} + +// Stop stops the server +func (srv *Server) Stop(context.Context) error { + srv.cancel() + srv.host.RemoveStreamHandler(srv.protocolID) + return nil +} + +func (srv *Server) streamHandler(ctx context.Context) network.StreamHandler { + return func(s network.Stream) { + err := srv.handleNamespaceData(ctx, s) + if err != nil { + s.Reset() //nolint:errcheck + return + } + if err = s.Close(); err != nil { + log.Debugw("server: closing stream", "err", err) + } + } +} + +// SetHandler sets server handler +func (srv *Server) SetHandler(handler network.StreamHandler) { + srv.handler = handler +} + +func (srv *Server) observeRateLimitedRequests() { + numRateLimited := srv.middleware.DrainCounter() + if numRateLimited > 0 { + srv.metrics.ObserveRequests(context.Background(), numRateLimited, shrex.StatusRateLimited) + } +} + +func (srv *Server) handleNamespaceData(ctx context.Context, stream network.Stream) error { + logger := log.With("source", "server", "peer", stream.Conn().RemotePeer().String()) + logger.Debug("handling nd request") + + srv.observeRateLimitedRequests() + ndid, err := srv.readRequest(logger, stream) + if err != nil { + logger.Warnw("read request", "err", err) + srv.metrics.ObserveRequests(ctx, 1, shrex.StatusBadRequest) + return err + } + + logger = logger.With( + "namespace", ndid.DataNamespace.String(), + "height", ndid.Height, + ) + logger.Debugw("new request") + + ctx, cancel := context.WithTimeout(ctx, srv.params.HandleRequestTimeout) + defer cancel() + + nd, status, err := srv.getNamespaceData(ctx, ndid) + if err != nil { + // server should respond with status regardless if there was an error getting data + sendErr := srv.respondStatus(ctx, logger, stream, status) + if sendErr != nil { + logger.Errorw("sending response", "err", sendErr) + srv.metrics.ObserveRequests(ctx, 1, shrex.StatusSendRespErr) + } + logger.Errorw("handling request", "err", err) + return errors.Join(err, sendErr) + } + + err = srv.respondStatus(ctx, logger, stream, status) + if err != nil { + logger.Errorw("sending response", "err", err) + srv.metrics.ObserveRequests(ctx, 1, shrex.StatusSendRespErr) + return err + } + + _, err = nd.WriteTo(stream) + if err != nil { + logger.Errorw("send nd data", "err", err) + srv.metrics.ObserveRequests(ctx, 1, shrex.StatusSendRespErr) + return err + } + return nil +} + +func (srv *Server) readRequest( + logger *zap.SugaredLogger, + stream network.Stream, +) (shwap.NamespaceDataID, error) { + err := stream.SetReadDeadline(time.Now().Add(srv.params.ServerReadTimeout)) + if err != nil { + logger.Debugw("setting read deadline", "err", err) + } + + ndid := shwap.NamespaceDataID{} + _, err = ndid.ReadFrom(stream) + if err != nil { + return shwap.NamespaceDataID{}, fmt.Errorf("reading request: %w", err) + } + + err = stream.CloseRead() + if err != nil { + logger.Warnw("closing read side of the stream", "err", err) + } + + return ndid, nil +} + +func (srv *Server) getNamespaceData(ctx context.Context, id shwap.NamespaceDataID) (shwap.NamespaceData, shrexpb.Status, error) { + file, err := srv.store.GetByHeight(ctx, id.Height) + if errors.Is(err, store.ErrNotFound) { + return nil, shrexpb.Status_NOT_FOUND, nil + } + if err != nil { + return nil, shrexpb.Status_INTERNAL, fmt.Errorf("retrieving DAH: %w", err) + } + defer utils.CloseAndLog(log, "file", file) + + nd, err := eds.NamespaceData(ctx, file, id.DataNamespace) + if err != nil { + return nil, shrexpb.Status_INVALID, fmt.Errorf("getting nd: %w", err) + } + + return nd, shrexpb.Status_OK, nil +} + +func (srv *Server) respondStatus( + ctx context.Context, + logger *zap.SugaredLogger, + stream network.Stream, + status shrexpb.Status, +) error { + srv.observeStatus(ctx, status) + + err := stream.SetWriteDeadline(time.Now().Add(srv.params.ServerWriteTimeout)) + if err != nil { + logger.Debugw("setting write deadline", "err", err) + } + + _, err = serde.Write(stream, &shrexpb.Response{Status: status}) + if err != nil { + return fmt.Errorf("writing response: %w", err) + } + + return nil +} + +func (srv *Server) observeStatus(ctx context.Context, status shrexpb.Status) { + switch { + case status == shrexpb.Status_OK: + srv.metrics.ObserveRequests(ctx, 1, shrex.StatusSuccess) + case status != shrexpb.Status_NOT_FOUND: + srv.metrics.ObserveRequests(ctx, 1, shrex.StatusNotFound) + case status == shrexpb.Status_INVALID: + srv.metrics.ObserveRequests(ctx, 1, shrex.StatusInternalErr) + } +} diff --git a/share/p2p/shrexsub/doc.go b/share/shwap/p2p/shrex/shrexsub/doc.go similarity index 100% rename from share/p2p/shrexsub/doc.go rename to share/shwap/p2p/shrex/shrexsub/doc.go diff --git a/share/p2p/shrexsub/pb/notification.pb.go b/share/shwap/p2p/shrex/shrexsub/pb/notification.pb.go similarity index 85% rename from share/p2p/shrexsub/pb/notification.pb.go rename to share/shwap/p2p/shrex/shrexsub/pb/notification.pb.go index e154dc62b7..c7cddbba5c 100644 --- a/share/p2p/shrexsub/pb/notification.pb.go +++ b/share/shwap/p2p/shrex/shrexsub/pb/notification.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: share/p2p/shrexsub/pb/notification.proto +// source: share/shwap/p2p/shrex/shrexsub/pb/notification.proto package share_p2p_shrex_sub @@ -31,7 +31,7 @@ func (m *RecentEDSNotification) Reset() { *m = RecentEDSNotification{} } func (m *RecentEDSNotification) String() string { return proto.CompactTextString(m) } func (*RecentEDSNotification) ProtoMessage() {} func (*RecentEDSNotification) Descriptor() ([]byte, []int) { - return fileDescriptor_1a6ade914b560e62, []int{0} + return fileDescriptor_c16b670e7e556100, []int{0} } func (m *RecentEDSNotification) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -79,22 +79,23 @@ func init() { } func init() { - proto.RegisterFile("share/p2p/shrexsub/pb/notification.proto", fileDescriptor_1a6ade914b560e62) + proto.RegisterFile("share/shwap/p2p/shrex/shrexsub/pb/notification.proto", fileDescriptor_c16b670e7e556100) } -var fileDescriptor_1a6ade914b560e62 = []byte{ - // 176 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x28, 0xce, 0x48, 0x2c, - 0x4a, 0xd5, 0x2f, 0x30, 0x2a, 0xd0, 0x2f, 0xce, 0x28, 0x4a, 0xad, 0x28, 0x2e, 0x4d, 0xd2, 0x2f, - 0x48, 0xd2, 0xcf, 0xcb, 0x2f, 0xc9, 0x4c, 0xcb, 0x4c, 0x4e, 0x2c, 0xc9, 0xcc, 0xcf, 0xd3, 0x2b, - 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x06, 0xab, 0xd4, 0x2b, 0x30, 0x2a, 0xd0, 0x03, 0xab, 0xd4, - 0x2b, 0x2e, 0x4d, 0x52, 0xf2, 0xe1, 0x12, 0x0d, 0x4a, 0x4d, 0x4e, 0xcd, 0x2b, 0x71, 0x75, 0x09, - 0xf6, 0x43, 0xd2, 0x23, 0x24, 0xc6, 0xc5, 0x96, 0x91, 0x9a, 0x99, 0x9e, 0x51, 0x22, 0xc1, 0xa8, - 0xc0, 0xa8, 0xc1, 0x12, 0x04, 0xe5, 0x09, 0x49, 0x73, 0x71, 0xa6, 0x24, 0x96, 0x24, 0xc6, 0x67, - 0x24, 0x16, 0x67, 0x48, 0x30, 0x29, 0x30, 0x6a, 0xf0, 0x04, 0x71, 0x80, 0x04, 0x3c, 0x12, 0x8b, - 0x33, 0x9c, 0x24, 0x4e, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6, - 0x09, 0x8f, 0xe5, 0x18, 0x2e, 0x3c, 0x96, 0x63, 0xb8, 0xf1, 0x58, 0x8e, 0x21, 0x89, 0x0d, 0xec, - 0x06, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x99, 0x16, 0xea, 0xc6, 0xaf, 0x00, 0x00, 0x00, +var fileDescriptor_c16b670e7e556100 = []byte{ + // 183 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x32, 0x29, 0xce, 0x48, 0x2c, + 0x4a, 0xd5, 0x2f, 0xce, 0x28, 0x4f, 0x2c, 0xd0, 0x2f, 0x30, 0x2a, 0xd0, 0x2f, 0xce, 0x28, 0x4a, + 0xad, 0x80, 0x90, 0xc5, 0xa5, 0x49, 0xfa, 0x05, 0x49, 0xfa, 0x79, 0xf9, 0x25, 0x99, 0x69, 0x99, + 0xc9, 0x89, 0x25, 0x99, 0xf9, 0x79, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0xc2, 0x60, 0x5d, + 0x7a, 0x05, 0x46, 0x05, 0x7a, 0x60, 0x95, 0x7a, 0xc5, 0xa5, 0x49, 0x4a, 0x3e, 0x5c, 0xa2, 0x41, + 0xa9, 0xc9, 0xa9, 0x79, 0x25, 0xae, 0x2e, 0xc1, 0x7e, 0x48, 0x7a, 0x84, 0xc4, 0xb8, 0xd8, 0x32, + 0x52, 0x33, 0xd3, 0x33, 0x4a, 0x24, 0x18, 0x15, 0x18, 0x35, 0x58, 0x82, 0xa0, 0x3c, 0x21, 0x69, + 0x2e, 0xce, 0x94, 0xc4, 0x92, 0xc4, 0xf8, 0x8c, 0xc4, 0xe2, 0x0c, 0x09, 0x26, 0x05, 0x46, 0x0d, + 0x9e, 0x20, 0x0e, 0x90, 0x80, 0x47, 0x62, 0x71, 0x86, 0x93, 0xc4, 0x89, 0x47, 0x72, 0x8c, 0x17, + 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c, 0xc3, 0x85, 0xc7, 0x72, 0x0c, + 0x37, 0x1e, 0xcb, 0x31, 0x24, 0xb1, 0x81, 0xdd, 0x60, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0xc0, + 0x55, 0x8a, 0x06, 0xbb, 0x00, 0x00, 0x00, } func (m *RecentEDSNotification) Marshal() (dAtA []byte, err error) { diff --git a/share/p2p/shrexsub/pb/notification.proto b/share/shwap/p2p/shrex/shrexsub/pb/notification.proto similarity index 100% rename from share/p2p/shrexsub/pb/notification.proto rename to share/shwap/p2p/shrex/shrexsub/pb/notification.proto diff --git a/share/p2p/shrexsub/pubsub.go b/share/shwap/p2p/shrex/shrexsub/pubsub.go similarity index 96% rename from share/p2p/shrexsub/pubsub.go rename to share/shwap/p2p/shrex/shrexsub/pubsub.go index 9254dbdb96..d1861cfe12 100644 --- a/share/p2p/shrexsub/pubsub.go +++ b/share/shwap/p2p/shrex/shrexsub/pubsub.go @@ -10,14 +10,14 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/celestiaorg/celestia-node/share" - pb "github.com/celestiaorg/celestia-node/share/p2p/shrexsub/pb" + pb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexsub/pb" ) var log = logging.Logger("shrex-sub") // pubsubTopic hardcodes the name of the EDS floodsub topic with the provided networkID. func pubsubTopicID(networkID string) string { - return fmt.Sprintf("%s/eds-sub/v0.1.0", networkID) + return fmt.Sprintf("%s/eds-sub/v0.2.0", networkID) } // ValidatorFn is an injectable func and governs EDS notification msg validity. diff --git a/share/p2p/shrexsub/pubsub_test.go b/share/shwap/p2p/shrex/shrexsub/pubsub_test.go similarity index 97% rename from share/p2p/shrexsub/pubsub_test.go rename to share/shwap/p2p/shrex/shrexsub/pubsub_test.go index 788d4ca73c..78c2141852 100644 --- a/share/p2p/shrexsub/pubsub_test.go +++ b/share/shwap/p2p/shrex/shrexsub/pubsub_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/libs/rand" - pb "github.com/celestiaorg/celestia-node/share/p2p/shrexsub/pb" + pb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexsub/pb" ) func TestPubSub(t *testing.T) { diff --git a/share/p2p/shrexsub/subscription.go b/share/shwap/p2p/shrex/shrexsub/subscription.go similarity index 94% rename from share/p2p/shrexsub/subscription.go rename to share/shwap/p2p/shrex/shrexsub/subscription.go index 32a3e65e51..5021f090c2 100644 --- a/share/p2p/shrexsub/subscription.go +++ b/share/shwap/p2p/shrex/shrexsub/subscription.go @@ -6,7 +6,7 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" - pb "github.com/celestiaorg/celestia-node/share/p2p/shrexsub/pb" + pb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexsub/pb" ) // Subscription is a wrapper over pubsub.Subscription that handles diff --git a/share/shwap/row.go b/share/shwap/row.go index 018e53e6bb..bc0b6ac507 100644 --- a/share/shwap/row.go +++ b/share/shwap/row.go @@ -10,6 +10,9 @@ import ( "github.com/celestiaorg/celestia-node/share/shwap/pb" ) +// RowName is the name identifier for the row container. +const RowName = "row_v0" + // RowSide enumerates the possible sides of a row within an Extended Data Square (EDS). type RowSide int diff --git a/share/shwap/row_namespace_data.go b/share/shwap/row_namespace_data.go index c03d47d971..8ef47952fc 100644 --- a/share/shwap/row_namespace_data.go +++ b/share/shwap/row_namespace_data.go @@ -13,6 +13,9 @@ import ( "github.com/celestiaorg/celestia-node/share/shwap/pb" ) +// RowNamespaceDataName is the name identifier for the row namespace data container. +const RowNamespaceDataName = "rnd_v0" + // ErrNamespaceOutsideRange is returned by RowNamespaceDataFromShares when the target namespace is // outside of the namespace range for the given row. In this case, the implementation cannot return // the non-inclusion proof and will return ErrNamespaceOutsideRange. diff --git a/share/shwap/row_namespace_data_test.go b/share/shwap/row_namespace_data_test.go index 87cf26ecaf..201e4f6cc1 100644 --- a/share/shwap/row_namespace_data_test.go +++ b/share/shwap/row_namespace_data_test.go @@ -66,7 +66,7 @@ func TestValidateNamespacedRow(t *testing.T) { for amount := 1; amount < sharesAmount; amount++ { randEDS, root := edstest.RandEDSWithNamespace(t, namespace, amount, odsSize) rsmt2d := &eds.Rsmt2D{ExtendedDataSquare: randEDS} - nd, err := eds.NamespacedData(ctx, rsmt2d, namespace) + nd, err := eds.NamespaceData(ctx, rsmt2d, namespace) require.NoError(t, err) require.True(t, len(nd) > 0) @@ -88,7 +88,7 @@ func TestNamespacedRowProtoEncoding(t *testing.T) { namespace := sharetest.RandV0Namespace() randEDS, _ := edstest.RandEDSWithNamespace(t, namespace, odsSize, odsSize) rsmt2d := &eds.Rsmt2D{ExtendedDataSquare: randEDS} - nd, err := eds.NamespacedData(ctx, rsmt2d, namespace) + nd, err := eds.NamespaceData(ctx, rsmt2d, namespace) require.NoError(t, err) require.True(t, len(nd) > 0) diff --git a/share/shwap/sample.go b/share/shwap/sample.go index ae51bf4e8f..8562706e4a 100644 --- a/share/shwap/sample.go +++ b/share/shwap/sample.go @@ -13,6 +13,9 @@ import ( "github.com/celestiaorg/celestia-node/share/shwap/pb" ) +// SampleName is the name identifier for the sample container. +const SampleName = "sample_v0" + // ErrFailedVerification is returned when inclusion proof verification fails. It is returned // when the data and the proof do not match trusted data root. var ErrFailedVerification = errors.New("failed to verify inclusion")