From 263c5fe6120e8ceb58c9fbb713ad146091b3820e Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 7 Nov 2019 14:57:37 -0500 Subject: [PATCH 01/39] add support for certified address records --- pb/pstore.pb.go | 286 +++++++++++++++++++++++++++-------------- pb/pstore.proto | 7 + pstoreds/addr_book.go | 59 +++++++-- pstoremem/addr_book.go | 50 ++++++- 4 files changed, 295 insertions(+), 107 deletions(-) diff --git a/pb/pstore.pb.go b/pb/pstore.pb.go index c3dbaeb..eef627e 100644 --- a/pb/pstore.pb.go +++ b/pb/pstore.pb.go @@ -9,7 +9,6 @@ import ( proto "github.com/gogo/protobuf/proto" io "io" math "math" - math_bits "math/bits" ) // Reference imports to suppress errors if they are not otherwise used. @@ -21,7 +20,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package // AddrBookRecord represents a record for a peer in the address book. type AddrBookRecord struct { @@ -29,6 +28,9 @@ type AddrBookRecord struct { Id *ProtoPeerID `protobuf:"bytes,1,opt,name=id,proto3,customtype=ProtoPeerID" json:"id,omitempty"` // The multiaddresses. This is a sorted list where element 0 expires the soonest. Addrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,2,rep,name=addrs,proto3" json:"addrs,omitempty"` + // Tracks the last seen sequence counter for peer state records used to + // add certified addresses. + PeerStateSeq uint64 `protobuf:"varint,3,opt,name=peerStateSeq,proto3" json:"peerStateSeq,omitempty"` } func (m *AddrBookRecord) Reset() { *m = AddrBookRecord{} } @@ -45,7 +47,7 @@ func (m *AddrBookRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro return xxx_messageInfo_AddrBookRecord.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } @@ -71,6 +73,13 @@ func (m *AddrBookRecord) GetAddrs() []*AddrBookRecord_AddrEntry { return nil } +func (m *AddrBookRecord) GetPeerStateSeq() uint64 { + if m != nil { + return m.PeerStateSeq + } + return 0 +} + // AddrEntry represents a single multiaddress. type AddrBookRecord_AddrEntry struct { Addr *ProtoAddr `protobuf:"bytes,1,opt,name=addr,proto3,customtype=ProtoAddr" json:"addr,omitempty"` @@ -78,6 +87,8 @@ type AddrBookRecord_AddrEntry struct { Expiry int64 `protobuf:"varint,2,opt,name=expiry,proto3" json:"expiry,omitempty"` // The original TTL of this address. Ttl int64 `protobuf:"varint,3,opt,name=ttl,proto3" json:"ttl,omitempty"` + // Whether this address was certified by the addressed peer. + Certified bool `protobuf:"varint,4,opt,name=certified,proto3" json:"certified,omitempty"` } func (m *AddrBookRecord_AddrEntry) Reset() { *m = AddrBookRecord_AddrEntry{} } @@ -94,7 +105,7 @@ func (m *AddrBookRecord_AddrEntry) XXX_Marshal(b []byte, deterministic bool) ([] return xxx_messageInfo_AddrBookRecord_AddrEntry.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) + n, err := m.MarshalTo(b) if err != nil { return nil, err } @@ -127,6 +138,13 @@ func (m *AddrBookRecord_AddrEntry) GetTtl() int64 { return 0 } +func (m *AddrBookRecord_AddrEntry) GetCertified() bool { + if m != nil { + return m.Certified + } + return false +} + func init() { proto.RegisterType((*AddrBookRecord)(nil), "pstore.pb.AddrBookRecord") proto.RegisterType((*AddrBookRecord_AddrEntry)(nil), "pstore.pb.AddrBookRecord.AddrEntry") @@ -135,29 +153,32 @@ func init() { func init() { proto.RegisterFile("pstore.proto", fileDescriptor_f96873690e08a98f) } var fileDescriptor_f96873690e08a98f = []byte{ - // 255 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x28, 0x2e, 0xc9, - 0x2f, 0x4a, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x84, 0xf1, 0x92, 0xa4, 0x74, 0xd3, - 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, 0xf5, 0xd3, 0xf3, 0xd3, 0xf3, 0xf5, 0xc1, - 0x2a, 0x92, 0x4a, 0xd3, 0xc0, 0x3c, 0x30, 0x07, 0xcc, 0x82, 0xe8, 0x54, 0x3a, 0xc6, 0xc8, 0xc5, - 0xe7, 0x98, 0x92, 0x52, 0xe4, 0x94, 0x9f, 0x9f, 0x1d, 0x94, 0x9a, 0x9c, 0x5f, 0x94, 0x22, 0x24, - 0xcf, 0xc5, 0x94, 0x99, 0x22, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0xe3, 0xc4, 0x7f, 0xeb, 0x9e, 0x3c, - 0x77, 0x00, 0x48, 0x65, 0x40, 0x6a, 0x6a, 0x91, 0xa7, 0x4b, 0x10, 0x53, 0x66, 0x8a, 0x90, 0x25, - 0x17, 0x6b, 0x62, 0x4a, 0x4a, 0x51, 0xb1, 0x04, 0x93, 0x02, 0xb3, 0x06, 0xb7, 0x91, 0xb2, 0x1e, - 0xdc, 0x76, 0x3d, 0x54, 0xa3, 0xc0, 0x5c, 0xd7, 0xbc, 0x92, 0xa2, 0xca, 0x20, 0x88, 0x0e, 0xa9, - 0x08, 0x2e, 0x4e, 0xb8, 0x98, 0x90, 0x22, 0x17, 0x0b, 0x48, 0x14, 0x6a, 0x15, 0xef, 0xad, 0x7b, - 0xf2, 0x9c, 0x60, 0xab, 0x40, 0x2a, 0x82, 0xc0, 0x52, 0x42, 0x62, 0x5c, 0x6c, 0xa9, 0x15, 0x05, - 0x99, 0x45, 0x95, 0x12, 0x4c, 0x0a, 0x8c, 0x1a, 0xcc, 0x41, 0x50, 0x9e, 0x90, 0x00, 0x17, 0x73, - 0x49, 0x49, 0x8e, 0x04, 0x33, 0x58, 0x10, 0xc4, 0x74, 0x52, 0xf8, 0xf1, 0x50, 0x8e, 0xf1, 0xc0, - 0x23, 0x39, 0xc6, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x71, - 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, 0xe5, 0x18, 0x6e, 0x3c, 0x96, 0x63, 0x48, 0x62, 0x03, 0xfb, - 0xd8, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0xb1, 0x1a, 0x16, 0x43, 0x3b, 0x01, 0x00, 0x00, + // 300 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x90, 0xbd, 0x4e, 0xf3, 0x30, + 0x14, 0x86, 0xeb, 0x24, 0x5f, 0xf5, 0xc5, 0x0d, 0x3f, 0xf2, 0x80, 0xa2, 0x0a, 0x39, 0xa6, 0x2c, + 0x59, 0x48, 0x25, 0x98, 0x18, 0x89, 0x60, 0x60, 0xab, 0xdc, 0x2b, 0x68, 0xe2, 0xd3, 0x60, 0xf1, + 0xe3, 0xe0, 0xba, 0x88, 0xde, 0x01, 0x23, 0x97, 0xc4, 0xc8, 0xd8, 0x11, 0x75, 0xa8, 0x20, 0xb9, + 0x09, 0x46, 0x14, 0xb7, 0x14, 0x75, 0x3b, 0xcf, 0xf1, 0xfb, 0xf8, 0x1c, 0x1d, 0x1c, 0x94, 0x13, + 0xa3, 0x34, 0x24, 0xa5, 0x56, 0x46, 0x11, 0xff, 0x97, 0xb2, 0xee, 0x49, 0x21, 0xcd, 0xcd, 0x34, + 0x4b, 0x72, 0x75, 0xdf, 0x2f, 0x54, 0xa1, 0xfa, 0x36, 0x91, 0x4d, 0xc7, 0x96, 0x2c, 0xd8, 0x6a, + 0x65, 0xf6, 0x5e, 0x1c, 0xbc, 0x7b, 0x21, 0x84, 0x4e, 0x95, 0xba, 0xe5, 0x90, 0x2b, 0x2d, 0x48, + 0x84, 0x1d, 0x29, 0x42, 0xc4, 0x50, 0x1c, 0xa4, 0x7b, 0x8b, 0x65, 0xd4, 0x19, 0x34, 0xc9, 0x01, + 0x80, 0xbe, 0xbe, 0xe4, 0x8e, 0x14, 0xe4, 0x1c, 0xff, 0x1b, 0x09, 0xa1, 0x27, 0xa1, 0xc3, 0xdc, + 0xb8, 0x73, 0x7a, 0x9c, 0x6c, 0xa6, 0x27, 0xdb, 0x5f, 0x59, 0xbc, 0x7a, 0x30, 0x7a, 0xc6, 0x57, + 0x06, 0xe9, 0xe1, 0xa0, 0x04, 0xd0, 0x43, 0x33, 0x32, 0x30, 0x84, 0xc7, 0xd0, 0x65, 0x28, 0xf6, + 0xf8, 0x56, 0xaf, 0xfb, 0x84, 0xfd, 0x8d, 0x47, 0x8e, 0xb0, 0xd7, 0x98, 0xeb, 0x75, 0x76, 0x16, + 0xcb, 0xc8, 0xb7, 0xeb, 0x34, 0x09, 0x6e, 0x9f, 0xc8, 0x01, 0x6e, 0xc3, 0x73, 0x29, 0xf5, 0x2c, + 0x74, 0x18, 0x8a, 0x5d, 0xbe, 0x26, 0xb2, 0x8f, 0x5d, 0x63, 0xee, 0xec, 0x08, 0x97, 0x37, 0x25, + 0x39, 0xc4, 0x7e, 0x0e, 0xda, 0xc8, 0xb1, 0x04, 0x11, 0x7a, 0x0c, 0xc5, 0xff, 0xf9, 0x5f, 0x23, + 0x65, 0xdf, 0x5f, 0x14, 0xbd, 0x55, 0x14, 0xbd, 0x57, 0x14, 0xcd, 0x2b, 0x8a, 0x3e, 0x2b, 0x8a, + 0x5e, 0x6b, 0xda, 0x9a, 0xd7, 0xb4, 0xf5, 0x51, 0xd3, 0x56, 0xd6, 0xb6, 0x37, 0x3b, 0xfb, 0x09, + 0x00, 0x00, 0xff, 0xff, 0x46, 0xa8, 0x9b, 0xc7, 0x7d, 0x01, 0x00, 0x00, } func (m *AddrBookRecord) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -165,48 +186,44 @@ func (m *AddrBookRecord) Marshal() (dAtA []byte, err error) { } func (m *AddrBookRecord) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *AddrBookRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if len(m.Addrs) > 0 { - for iNdEx := len(m.Addrs) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Addrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPstore(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 + if m.Id != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPstore(dAtA, i, uint64(m.Id.Size())) + n1, err := m.Id.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } + i += n1 } - if m.Id != nil { - { - size := m.Id.Size() - i -= size - if _, err := m.Id.MarshalTo(dAtA[i:]); err != nil { + if len(m.Addrs) > 0 { + for _, msg := range m.Addrs { + dAtA[i] = 0x12 + i++ + i = encodeVarintPstore(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { return 0, err } - i = encodeVarintPstore(dAtA, i, uint64(size)) + i += n } - i-- - dAtA[i] = 0xa } - return len(dAtA) - i, nil + if m.PeerStateSeq != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPstore(dAtA, i, uint64(m.PeerStateSeq)) + } + return i, nil } func (m *AddrBookRecord_AddrEntry) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) + n, err := m.MarshalTo(dAtA) if err != nil { return nil, err } @@ -214,61 +231,63 @@ func (m *AddrBookRecord_AddrEntry) Marshal() (dAtA []byte, err error) { } func (m *AddrBookRecord_AddrEntry) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *AddrBookRecord_AddrEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) + var i int _ = i var l int _ = l - if m.Ttl != 0 { - i = encodeVarintPstore(dAtA, i, uint64(m.Ttl)) - i-- - dAtA[i] = 0x18 + if m.Addr != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPstore(dAtA, i, uint64(m.Addr.Size())) + n2, err := m.Addr.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n2 } if m.Expiry != 0 { - i = encodeVarintPstore(dAtA, i, uint64(m.Expiry)) - i-- dAtA[i] = 0x10 + i++ + i = encodeVarintPstore(dAtA, i, uint64(m.Expiry)) } - if m.Addr != nil { - { - size := m.Addr.Size() - i -= size - if _, err := m.Addr.MarshalTo(dAtA[i:]); err != nil { - return 0, err - } - i = encodeVarintPstore(dAtA, i, uint64(size)) + if m.Ttl != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPstore(dAtA, i, uint64(m.Ttl)) + } + if m.Certified { + dAtA[i] = 0x20 + i++ + if m.Certified { + dAtA[i] = 1 + } else { + dAtA[i] = 0 } - i-- - dAtA[i] = 0xa + i++ } - return len(dAtA) - i, nil + return i, nil } func encodeVarintPstore(dAtA []byte, offset int, v uint64) int { - offset -= sovPstore(v) - base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) - return base + return offset + 1 } func NewPopulatedAddrBookRecord(r randyPstore, easy bool) *AddrBookRecord { this := &AddrBookRecord{} this.Id = NewPopulatedProtoPeerID(r) - if r.Intn(5) != 0 { + if r.Intn(10) != 0 { v1 := r.Intn(5) this.Addrs = make([]*AddrBookRecord_AddrEntry, v1) for i := 0; i < v1; i++ { this.Addrs[i] = NewPopulatedAddrBookRecord_AddrEntry(r, easy) } } + this.PeerStateSeq = uint64(uint64(r.Uint32())) if !easy && r.Intn(10) != 0 { } return this @@ -285,6 +304,7 @@ func NewPopulatedAddrBookRecord_AddrEntry(r randyPstore, easy bool) *AddrBookRec if r.Intn(2) == 0 { this.Ttl *= -1 } + this.Certified = bool(bool(r.Intn(2) == 0)) if !easy && r.Intn(10) != 0 { } return this @@ -378,6 +398,9 @@ func (m *AddrBookRecord) Size() (n int) { n += 1 + l + sovPstore(uint64(l)) } } + if m.PeerStateSeq != 0 { + n += 1 + sovPstore(uint64(m.PeerStateSeq)) + } return n } @@ -397,11 +420,21 @@ func (m *AddrBookRecord_AddrEntry) Size() (n int) { if m.Ttl != 0 { n += 1 + sovPstore(uint64(m.Ttl)) } + if m.Certified { + n += 2 + } return n } func sovPstore(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n } func sozPstore(x uint64) (n int) { return sovPstore(uint64((x << 1) ^ uint64((int64(x) >> 63)))) @@ -504,6 +537,25 @@ func (m *AddrBookRecord) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerStateSeq", wireType) + } + m.PeerStateSeq = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPstore + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.PeerStateSeq |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPstore(dAtA[iNdEx:]) @@ -630,6 +682,26 @@ func (m *AddrBookRecord_AddrEntry) Unmarshal(dAtA []byte) error { break } } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Certified", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPstore + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Certified = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPstore(dAtA[iNdEx:]) @@ -657,7 +729,6 @@ func (m *AddrBookRecord_AddrEntry) Unmarshal(dAtA []byte) error { func skipPstore(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 - depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -689,8 +760,10 @@ func skipPstore(dAtA []byte) (n int, err error) { break } } + return iNdEx, nil case 1: iNdEx += 8 + return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -711,30 +784,55 @@ func skipPstore(dAtA []byte) (n int, err error) { return 0, ErrInvalidLengthPstore } iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthPstore + } + return iNdEx, nil case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupPstore + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPstore + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipPstore(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthPstore + } } - depth-- + return iNdEx, nil + case 4: + return iNdEx, nil case 5: iNdEx += 4 + return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } - if iNdEx < 0 { - return 0, ErrInvalidLengthPstore - } - if depth == 0 { - return iNdEx, nil - } } - return 0, io.ErrUnexpectedEOF + panic("unreachable") } var ( - ErrInvalidLengthPstore = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowPstore = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupPstore = fmt.Errorf("proto: unexpected end of group") + ErrInvalidLengthPstore = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowPstore = fmt.Errorf("proto: integer overflow") ) diff --git a/pb/pstore.proto b/pb/pstore.proto index 3894534..8a1f6e4 100644 --- a/pb/pstore.proto +++ b/pb/pstore.proto @@ -23,5 +23,12 @@ message AddrBookRecord { // The original TTL of this address. int64 ttl = 3; + + // Whether this address was certified by the addressed peer. + bool certified = 4; } + + // Tracks the last seen sequence counter for peer state records used to + // add certified addresses. + uint64 peerStateSeq = 3; } diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 2f75265..0cb5829 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -3,20 +3,22 @@ package pstoreds import ( "context" "fmt" + "github.com/libp2p/go-libp2p-core/crypto" + "github.com/libp2p/go-libp2p-core/routing" "sort" "sync" "time" ds "github.com/ipfs/go-datastore" - query "github.com/ipfs/go-datastore/query" + "github.com/ipfs/go-datastore/query" logging "github.com/ipfs/go-log" - peer "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peer" pstore "github.com/libp2p/go-libp2p-core/peerstore" pb "github.com/libp2p/go-libp2p-peerstore/pb" - pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem" + "github.com/libp2p/go-libp2p-peerstore/pstoremem" - lru "github.com/hashicorp/golang-lru" + "github.com/hashicorp/golang-lru" b32 "github.com/multiformats/go-base32" ma "github.com/multiformats/go-multiaddr" ) @@ -240,7 +242,35 @@ func (ab *dsAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio return } addrs = cleanAddrs(addrs) - ab.setAddrs(p, addrs, ttl, ttlExtend) + ab.setAddrs(p, addrs, ttl, ttlExtend, false) +} + +func (ab *dsAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl time.Duration) error { + state, err := routing.RoutingStateFromEnvelope(envelope) + if err != nil { + return err + } + + // ensure that the seq number from envelope is >= any previously received seq no + pr, err := ab.loadRecord(state.PeerID, true, false) + if err != nil { + return err + } + if pr.PeerStateSeq >= state.Seq { + // TODO: should this be an error? + return nil + } + pr.PeerStateSeq = state.Seq + pr.dirty = true + err = pr.flush(ab.ds) + if err != nil { + return err + } + + // TODO: remove addresses from previous RoutingStates (new state should completely replace old) + + addrs := cleanAddrs(state.Multiaddrs()) + return ab.setAddrs(state.PeerID, addrs, ttl, ttlExtend, true) } // SetAddr will add or update the TTL of an address in the AddrBook. @@ -255,7 +285,7 @@ func (ab *dsAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio ab.deleteAddrs(p, addrs) return } - ab.setAddrs(p, addrs, ttl, ttlOverride) + ab.setAddrs(p, addrs, ttl, ttlOverride, false) } // UpdateAddrs will update any addresses for a given peer and TTL combination to @@ -286,6 +316,16 @@ func (ab *dsAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.D // Addrs returns all of the non-expired addresses for a given peer. func (ab *dsAddrBook) Addrs(p peer.ID) []ma.Multiaddr { + return ab.addrs(p, true) +} + +// CertifiedAddrs returns all of the non-expired address that have been +// certified by the given peer. +func (ab *dsAddrBook) CertifiedAddrs(p peer.ID) []ma.Multiaddr { + return ab.addrs(p, false) +} + +func (ab *dsAddrBook) addrs(p peer.ID, includeUncertified bool) []ma.Multiaddr { pr, err := ab.loadRecord(p, true, true) if err != nil { log.Warning("failed to load peerstore entry for peer %v while querying addrs, err: %v", p, err) @@ -297,7 +337,9 @@ func (ab *dsAddrBook) Addrs(p peer.ID) []ma.Multiaddr { addrs := make([]ma.Multiaddr, 0, len(pr.Addrs)) for _, a := range pr.Addrs { - addrs = append(addrs, a.Addr) + if includeUncertified || a.Certified { + addrs = append(addrs, a.Addr) + } } return addrs } @@ -330,7 +372,7 @@ func (ab *dsAddrBook) ClearAddrs(p peer.ID) { } } -func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration, mode ttlWriteMode) (err error) { +func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration, mode ttlWriteMode, certified bool) (err error) { pr, err := ab.loadRecord(p, true, false) if err != nil { return fmt.Errorf("failed to load peerstore entry for peer %v while setting addrs, err: %v", p, err) @@ -380,6 +422,7 @@ Outer: Addr: &pb.ProtoAddr{Multiaddr: addr}, Ttl: int64(ttl), Expiry: newExp, + Certified: certified, } added = append(added, entry) // note: there's a minor chance that writing the record will fail, in which case we would've broadcast diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 90ffb17..4523df7 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -7,11 +7,13 @@ import ( "time" logging "github.com/ipfs/go-log" - peer "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/crypto" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/routing" ma "github.com/multiformats/go-multiaddr" pstore "github.com/libp2p/go-libp2p-core/peerstore" - addr "github.com/libp2p/go-libp2p-peerstore/addr" + "github.com/libp2p/go-libp2p-peerstore/addr" ) var log = logging.Logger("peerstore") @@ -20,6 +22,7 @@ type expiringAddr struct { Addr ma.Multiaddr TTL time.Duration Expires time.Time + Certified bool } func (e *expiringAddr) ExpiredBy(t time.Time) bool { @@ -44,6 +47,7 @@ func (s *addrSegments) get(p peer.ID) *addrSegment { // memoryAddrBook manages addresses. type memoryAddrBook struct { segments addrSegments + peerStateSeq map[peer.ID]uint64 ctx context.Context cancel func() @@ -134,6 +138,29 @@ func (mab *memoryAddrBook) AddAddr(p peer.ID, addr ma.Multiaddr, ttl time.Durati // (time-to-live), after which the address is no longer valid. // This function never reduces the TTL or expiration of an address. func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) { + mab.addAddrs(p, addrs, ttl, false) +} + +func (mab *memoryAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl time.Duration) error { + state, err := routing.RoutingStateFromEnvelope(envelope) + if err != nil { + return err + } + + // ensure seq is greater than last received + lastSeq, found := mab.peerStateSeq[state.PeerID] + if found && lastSeq >= state.Seq { + // TODO: should this be an error? + return nil + } + mab.peerStateSeq[state.PeerID] = state.Seq + // TODO: remove addresses from previous RoutingStates (new state should completely replace old) + + mab.addAddrs(state.PeerID, state.Multiaddrs(), ttl, true) + return nil +} + +func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration, certified bool) { // if ttl is zero, exit. nothing to do. if ttl <= 0 { return @@ -158,7 +185,7 @@ func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du a, found := amap[string(asBytes)] // won't allocate. if !found { // not found, save and announce it. - amap[string(asBytes)] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl} + amap[string(asBytes)] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl, Certified: certified} mab.subManager.BroadcastAddr(p, addr) } else { // Update expiration/TTL independently. @@ -231,8 +258,18 @@ func (mab *memoryAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL t } } -// Addresses returns all known (and valid) addresses for a given +// Addrs returns all known (and valid) addresses for a given peer func (mab *memoryAddrBook) Addrs(p peer.ID) []ma.Multiaddr { + return mab.addrs(p, true) +} + +// CertifiedAddrs returns all known (and valid) addressed that have been +// certified by the given peer. +func (mab *memoryAddrBook) CertifiedAddrs(p peer.ID) []ma.Multiaddr { + return mab.addrs(p, false) +} + +func (mab *memoryAddrBook) addrs(p peer.ID, includeUncertified bool) []ma.Multiaddr { s := mab.segments.get(p) s.RLock() defer s.RUnlock() @@ -245,7 +282,10 @@ func (mab *memoryAddrBook) Addrs(p peer.ID) []ma.Multiaddr { now := time.Now() good := make([]ma.Multiaddr, 0, len(amap)) for _, m := range amap { - if !m.ExpiredBy(now) { + if m.ExpiredBy(now) { + continue + } + if includeUncertified || m.Certified { good = append(good, m.Addr) } } From 58b79365e1e5e56e0c3e42d3b83fdfbe31abc63a Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 8 Nov 2019 11:44:52 -0500 Subject: [PATCH 02/39] store original signed routing records & add retreival method --- pb/pstore.pb.go | 137 +++++++++++++++++++++++++++++------------ pb/pstore.proto | 9 ++- pstoreds/addr_book.go | 28 ++++++++- pstoremem/addr_book.go | 8 +++ 4 files changed, 138 insertions(+), 44 deletions(-) diff --git a/pb/pstore.pb.go b/pb/pstore.pb.go index eef627e..d5c160b 100644 --- a/pb/pstore.pb.go +++ b/pb/pstore.pb.go @@ -28,9 +28,11 @@ type AddrBookRecord struct { Id *ProtoPeerID `protobuf:"bytes,1,opt,name=id,proto3,customtype=ProtoPeerID" json:"id,omitempty"` // The multiaddresses. This is a sorted list where element 0 expires the soonest. Addrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,2,rep,name=addrs,proto3" json:"addrs,omitempty"` - // Tracks the last seen sequence counter for peer state records used to - // add certified addresses. - PeerStateSeq uint64 `protobuf:"varint,3,opt,name=peerStateSeq,proto3" json:"peerStateSeq,omitempty"` + // The Seq counter value from the most recent RoutingState record for this peer. + RoutingStateSeq uint64 `protobuf:"varint,3,opt,name=routingStateSeq,proto3" json:"routingStateSeq,omitempty"` + // A serialized SignedEnvelope containing the most recent RoutingState record. + // TODO: import the SignedEnvelope type instead + SignedRoutingRecord []byte `protobuf:"bytes,4,opt,name=signedRoutingRecord,proto3" json:"signedRoutingRecord,omitempty"` } func (m *AddrBookRecord) Reset() { *m = AddrBookRecord{} } @@ -73,13 +75,20 @@ func (m *AddrBookRecord) GetAddrs() []*AddrBookRecord_AddrEntry { return nil } -func (m *AddrBookRecord) GetPeerStateSeq() uint64 { +func (m *AddrBookRecord) GetRoutingStateSeq() uint64 { if m != nil { - return m.PeerStateSeq + return m.RoutingStateSeq } return 0 } +func (m *AddrBookRecord) GetSignedRoutingRecord() []byte { + if m != nil { + return m.SignedRoutingRecord + } + return nil +} + // AddrEntry represents a single multiaddress. type AddrBookRecord_AddrEntry struct { Addr *ProtoAddr `protobuf:"bytes,1,opt,name=addr,proto3,customtype=ProtoAddr" json:"addr,omitempty"` @@ -153,26 +162,27 @@ func init() { func init() { proto.RegisterFile("pstore.proto", fileDescriptor_f96873690e08a98f) } var fileDescriptor_f96873690e08a98f = []byte{ - // 300 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x90, 0xbd, 0x4e, 0xf3, 0x30, - 0x14, 0x86, 0xeb, 0x24, 0x5f, 0xf5, 0xc5, 0x0d, 0x3f, 0xf2, 0x80, 0xa2, 0x0a, 0x39, 0xa6, 0x2c, - 0x59, 0x48, 0x25, 0x98, 0x18, 0x89, 0x60, 0x60, 0xab, 0xdc, 0x2b, 0x68, 0xe2, 0xd3, 0x60, 0xf1, - 0xe3, 0xe0, 0xba, 0x88, 0xde, 0x01, 0x23, 0x97, 0xc4, 0xc8, 0xd8, 0x11, 0x75, 0xa8, 0x20, 0xb9, - 0x09, 0x46, 0x14, 0xb7, 0x14, 0x75, 0x3b, 0xcf, 0xf1, 0xfb, 0xf8, 0x1c, 0x1d, 0x1c, 0x94, 0x13, - 0xa3, 0x34, 0x24, 0xa5, 0x56, 0x46, 0x11, 0xff, 0x97, 0xb2, 0xee, 0x49, 0x21, 0xcd, 0xcd, 0x34, - 0x4b, 0x72, 0x75, 0xdf, 0x2f, 0x54, 0xa1, 0xfa, 0x36, 0x91, 0x4d, 0xc7, 0x96, 0x2c, 0xd8, 0x6a, - 0x65, 0xf6, 0x5e, 0x1c, 0xbc, 0x7b, 0x21, 0x84, 0x4e, 0x95, 0xba, 0xe5, 0x90, 0x2b, 0x2d, 0x48, - 0x84, 0x1d, 0x29, 0x42, 0xc4, 0x50, 0x1c, 0xa4, 0x7b, 0x8b, 0x65, 0xd4, 0x19, 0x34, 0xc9, 0x01, - 0x80, 0xbe, 0xbe, 0xe4, 0x8e, 0x14, 0xe4, 0x1c, 0xff, 0x1b, 0x09, 0xa1, 0x27, 0xa1, 0xc3, 0xdc, - 0xb8, 0x73, 0x7a, 0x9c, 0x6c, 0xa6, 0x27, 0xdb, 0x5f, 0x59, 0xbc, 0x7a, 0x30, 0x7a, 0xc6, 0x57, - 0x06, 0xe9, 0xe1, 0xa0, 0x04, 0xd0, 0x43, 0x33, 0x32, 0x30, 0x84, 0xc7, 0xd0, 0x65, 0x28, 0xf6, - 0xf8, 0x56, 0xaf, 0xfb, 0x84, 0xfd, 0x8d, 0x47, 0x8e, 0xb0, 0xd7, 0x98, 0xeb, 0x75, 0x76, 0x16, - 0xcb, 0xc8, 0xb7, 0xeb, 0x34, 0x09, 0x6e, 0x9f, 0xc8, 0x01, 0x6e, 0xc3, 0x73, 0x29, 0xf5, 0x2c, - 0x74, 0x18, 0x8a, 0x5d, 0xbe, 0x26, 0xb2, 0x8f, 0x5d, 0x63, 0xee, 0xec, 0x08, 0x97, 0x37, 0x25, - 0x39, 0xc4, 0x7e, 0x0e, 0xda, 0xc8, 0xb1, 0x04, 0x11, 0x7a, 0x0c, 0xc5, 0xff, 0xf9, 0x5f, 0x23, - 0x65, 0xdf, 0x5f, 0x14, 0xbd, 0x55, 0x14, 0xbd, 0x57, 0x14, 0xcd, 0x2b, 0x8a, 0x3e, 0x2b, 0x8a, - 0x5e, 0x6b, 0xda, 0x9a, 0xd7, 0xb4, 0xf5, 0x51, 0xd3, 0x56, 0xd6, 0xb6, 0x37, 0x3b, 0xfb, 0x09, - 0x00, 0x00, 0xff, 0xff, 0x46, 0xa8, 0x9b, 0xc7, 0x7d, 0x01, 0x00, 0x00, + // 318 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x50, 0x31, 0x4f, 0xf3, 0x30, + 0x10, 0x8d, 0x93, 0x7c, 0xd5, 0x17, 0xb7, 0x50, 0x64, 0x24, 0x14, 0x55, 0xc8, 0x09, 0xb0, 0x64, + 0x21, 0x45, 0x30, 0x31, 0x12, 0xc1, 0xc0, 0x56, 0xb9, 0xbf, 0xa0, 0x89, 0xdd, 0x60, 0x01, 0x71, + 0x70, 0x1c, 0x44, 0xff, 0x05, 0x3f, 0xa9, 0x23, 0x63, 0x47, 0xd4, 0xa1, 0x82, 0xe4, 0x4f, 0x30, + 0xa2, 0x38, 0xa5, 0x08, 0xc4, 0x76, 0xef, 0xde, 0xbb, 0x7b, 0xef, 0x0e, 0xf6, 0xf2, 0x42, 0x09, + 0xc9, 0xc2, 0x5c, 0x0a, 0x25, 0x90, 0xf3, 0x85, 0xe2, 0xc1, 0x71, 0xca, 0xd5, 0x4d, 0x19, 0x87, + 0x89, 0xb8, 0x1f, 0xa6, 0x22, 0x15, 0x43, 0xad, 0x88, 0xcb, 0xa9, 0x46, 0x1a, 0xe8, 0xaa, 0x9d, + 0x3c, 0x9c, 0x9b, 0x70, 0xfb, 0x82, 0x52, 0x19, 0x09, 0x71, 0x4b, 0x58, 0x22, 0x24, 0x45, 0x1e, + 0x34, 0x39, 0x75, 0x81, 0x0f, 0x82, 0x5e, 0xd4, 0x5f, 0xae, 0xbc, 0xee, 0xa8, 0x51, 0x8e, 0x18, + 0x93, 0xd7, 0x97, 0xc4, 0xe4, 0x14, 0x9d, 0xc3, 0x7f, 0x13, 0x4a, 0x65, 0xe1, 0x9a, 0xbe, 0x15, + 0x74, 0x4f, 0x8f, 0xc2, 0x8d, 0x7b, 0xf8, 0x73, 0x95, 0x86, 0x57, 0x99, 0x92, 0x33, 0xd2, 0x4e, + 0xa0, 0x00, 0xf6, 0xa5, 0x28, 0x15, 0xcf, 0xd2, 0xb1, 0x9a, 0x28, 0x36, 0x66, 0x0f, 0xae, 0xe5, + 0x83, 0xc0, 0x26, 0xbf, 0xdb, 0xe8, 0x04, 0xee, 0x16, 0x3c, 0xcd, 0x18, 0x25, 0x2d, 0xd1, 0x6e, + 0x74, 0xed, 0x26, 0x16, 0xf9, 0x8b, 0x1a, 0x3c, 0x42, 0x67, 0xe3, 0x87, 0x0e, 0xa0, 0xdd, 0x38, + 0xae, 0xcf, 0xd8, 0x5a, 0xae, 0x3c, 0x47, 0x9f, 0xd1, 0x28, 0x88, 0xa6, 0xd0, 0x1e, 0xec, 0xb0, + 0xa7, 0x9c, 0xcb, 0x99, 0x6b, 0xfa, 0x20, 0xb0, 0xc8, 0x1a, 0xa1, 0x1d, 0x68, 0x29, 0x75, 0xa7, + 0x73, 0x59, 0xa4, 0x29, 0xd1, 0x3e, 0x74, 0x12, 0x26, 0x15, 0x9f, 0x72, 0xd6, 0x26, 0xf8, 0x4f, + 0xbe, 0x1b, 0x91, 0xff, 0xf1, 0x8e, 0xc1, 0xbc, 0xc2, 0xe0, 0xa5, 0xc2, 0x60, 0x51, 0x61, 0xf0, + 0x56, 0x61, 0xf0, 0x5c, 0x63, 0x63, 0x51, 0x63, 0xe3, 0xb5, 0xc6, 0x46, 0xdc, 0xd1, 0xbf, 0x3e, + 0xfb, 0x0c, 0x00, 0x00, 0xff, 0xff, 0xbd, 0x79, 0x1e, 0x1b, 0xb5, 0x01, 0x00, 0x00, } func (m *AddrBookRecord) Marshal() (dAtA []byte, err error) { @@ -212,10 +222,16 @@ func (m *AddrBookRecord) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.PeerStateSeq != 0 { + if m.RoutingStateSeq != 0 { dAtA[i] = 0x18 i++ - i = encodeVarintPstore(dAtA, i, uint64(m.PeerStateSeq)) + i = encodeVarintPstore(dAtA, i, uint64(m.RoutingStateSeq)) + } + if len(m.SignedRoutingRecord) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintPstore(dAtA, i, uint64(len(m.SignedRoutingRecord))) + i += copy(dAtA[i:], m.SignedRoutingRecord) } return i, nil } @@ -287,7 +303,12 @@ func NewPopulatedAddrBookRecord(r randyPstore, easy bool) *AddrBookRecord { this.Addrs[i] = NewPopulatedAddrBookRecord_AddrEntry(r, easy) } } - this.PeerStateSeq = uint64(uint64(r.Uint32())) + this.RoutingStateSeq = uint64(uint64(r.Uint32())) + v2 := r.Intn(100) + this.SignedRoutingRecord = make([]byte, v2) + for i := 0; i < v2; i++ { + this.SignedRoutingRecord[i] = byte(r.Intn(256)) + } if !easy && r.Intn(10) != 0 { } return this @@ -329,9 +350,9 @@ func randUTF8RunePstore(r randyPstore) rune { return rune(ru + 61) } func randStringPstore(r randyPstore) string { - v2 := r.Intn(100) - tmps := make([]rune, v2) - for i := 0; i < v2; i++ { + v3 := r.Intn(100) + tmps := make([]rune, v3) + for i := 0; i < v3; i++ { tmps[i] = randUTF8RunePstore(r) } return string(tmps) @@ -353,11 +374,11 @@ func randFieldPstore(dAtA []byte, r randyPstore, fieldNumber int, wire int) []by switch wire { case 0: dAtA = encodeVarintPopulatePstore(dAtA, uint64(key)) - v3 := r.Int63() + v4 := r.Int63() if r.Intn(2) == 0 { - v3 *= -1 + v4 *= -1 } - dAtA = encodeVarintPopulatePstore(dAtA, uint64(v3)) + dAtA = encodeVarintPopulatePstore(dAtA, uint64(v4)) case 1: dAtA = encodeVarintPopulatePstore(dAtA, uint64(key)) dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) @@ -398,8 +419,12 @@ func (m *AddrBookRecord) Size() (n int) { n += 1 + l + sovPstore(uint64(l)) } } - if m.PeerStateSeq != 0 { - n += 1 + sovPstore(uint64(m.PeerStateSeq)) + if m.RoutingStateSeq != 0 { + n += 1 + sovPstore(uint64(m.RoutingStateSeq)) + } + l = len(m.SignedRoutingRecord) + if l > 0 { + n += 1 + l + sovPstore(uint64(l)) } return n } @@ -539,9 +564,28 @@ func (m *AddrBookRecord) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field PeerStateSeq", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field RoutingStateSeq", wireType) + } + m.RoutingStateSeq = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPstore + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RoutingStateSeq |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } } - m.PeerStateSeq = 0 + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SignedRoutingRecord", wireType) + } + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPstore @@ -551,11 +595,26 @@ func (m *AddrBookRecord) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.PeerStateSeq |= uint64(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } + if byteLen < 0 { + return ErrInvalidLengthPstore + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthPstore + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SignedRoutingRecord = append(m.SignedRoutingRecord[:0], dAtA[iNdEx:postIndex]...) + if m.SignedRoutingRecord == nil { + m.SignedRoutingRecord = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPstore(dAtA[iNdEx:]) diff --git a/pb/pstore.proto b/pb/pstore.proto index 8a1f6e4..410cdab 100644 --- a/pb/pstore.proto +++ b/pb/pstore.proto @@ -28,7 +28,10 @@ message AddrBookRecord { bool certified = 4; } - // Tracks the last seen sequence counter for peer state records used to - // add certified addresses. - uint64 peerStateSeq = 3; + // The Seq counter value from the most recent RoutingState record for this peer. + uint64 routingStateSeq = 3; + + // A serialized SignedEnvelope containing the most recent RoutingState record. + // TODO: import the SignedEnvelope type instead + bytes signedRoutingRecord = 4; } diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 0cb5829..1d0de05 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -256,11 +256,18 @@ func (ab *dsAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl tim if err != nil { return err } - if pr.PeerStateSeq >= state.Seq { + if pr.RoutingStateSeq >= state.Seq { // TODO: should this be an error? return nil } - pr.PeerStateSeq = state.Seq + + envelopeBytes, err := envelope.Marshal() + if err != nil { + return err + } + + pr.RoutingStateSeq = state.Seq + pr.SignedRoutingRecord = envelopeBytes pr.dirty = true err = pr.flush(ab.ds) if err != nil { @@ -273,6 +280,23 @@ func (ab *dsAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl tim return ab.setAddrs(state.PeerID, addrs, ttl, ttlExtend, true) } +func (ab *dsAddrBook) SignedRoutingState(p peer.ID) *crypto.SignedEnvelope { + pr, err := ab.loadRecord(p, true, false) + if err != nil { + log.Errorf("unable to load record for peer %s: %v", p.Pretty(), err) + return nil + } + if len(pr.SignedRoutingRecord) == 0 { + return nil + } + envelope, err := crypto.UnmarshalEnvelope(pr.SignedRoutingRecord) + if err != nil { + log.Errorf("unable to unmarshal stored signed routing record for peer %s: %v", p.Pretty(), err) + return nil + } + return envelope +} + // SetAddr will add or update the TTL of an address in the AddrBook. func (ab *dsAddrBook) SetAddr(p peer.ID, addr ma.Multiaddr, ttl time.Duration) { ab.SetAddrs(p, []ma.Multiaddr{addr}, ttl) diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 4523df7..3f68b51 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -47,6 +47,8 @@ func (s *addrSegments) get(p peer.ID) *addrSegment { // memoryAddrBook manages addresses. type memoryAddrBook struct { segments addrSegments + + signedRoutingStates map[peer.ID]*crypto.SignedEnvelope peerStateSeq map[peer.ID]uint64 ctx context.Context @@ -154,6 +156,8 @@ func (mab *memoryAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, tt return nil } mab.peerStateSeq[state.PeerID] = state.Seq + mab.signedRoutingStates[state.PeerID] = envelope + // TODO: remove addresses from previous RoutingStates (new state should completely replace old) mab.addAddrs(state.PeerID, state.Multiaddrs(), ttl, true) @@ -293,6 +297,10 @@ func (mab *memoryAddrBook) addrs(p peer.ID, includeUncertified bool) []ma.Multia return good } +func (mab *memoryAddrBook) SignedRoutingState(p peer.ID) *crypto.SignedEnvelope { + return mab.signedRoutingStates[p] +} + // ClearAddrs removes all previously stored addresses func (mab *memoryAddrBook) ClearAddrs(p peer.ID) { s := mab.segments.get(p) From ad09bee35a57159c6f9cab2a5001e01166ab0ed7 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 8 Nov 2019 11:45:09 -0500 Subject: [PATCH 03/39] go fmt --- pstoreds/addr_book.go | 6 +++--- pstoremem/addr_book.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 1d0de05..1ec38e2 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -443,9 +443,9 @@ Outer: } addr := addrs[i] entry := &pb.AddrBookRecord_AddrEntry{ - Addr: &pb.ProtoAddr{Multiaddr: addr}, - Ttl: int64(ttl), - Expiry: newExp, + Addr: &pb.ProtoAddr{Multiaddr: addr}, + Ttl: int64(ttl), + Expiry: newExp, Certified: certified, } added = append(added, entry) diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 3f68b51..77d16b0 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -19,9 +19,9 @@ import ( var log = logging.Logger("peerstore") type expiringAddr struct { - Addr ma.Multiaddr - TTL time.Duration - Expires time.Time + Addr ma.Multiaddr + TTL time.Duration + Expires time.Time Certified bool } @@ -49,7 +49,7 @@ type memoryAddrBook struct { segments addrSegments signedRoutingStates map[peer.ID]*crypto.SignedEnvelope - peerStateSeq map[peer.ID]uint64 + peerStateSeq map[peer.ID]uint64 ctx context.Context cancel func() From 5ba6eced668850d939ed5688acca166b7db4854c Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 8 Nov 2019 12:56:01 -0500 Subject: [PATCH 04/39] add tests for certified addrs --- test/addr_book_suite.go | 45 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index ea6078c..3c7d083 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -1,6 +1,10 @@ package test import ( + "github.com/libp2p/go-libp2p-core/crypto" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/routing" + "github.com/libp2p/go-libp2p-core/test" "testing" "time" @@ -16,6 +20,7 @@ var addressBookSuite = map[string]func(book pstore.AddrBook) func(*testing.T){ "AddressesExpire": testAddressesExpire, "ClearWithIter": testClearWithIterator, "PeersWithAddresses": testPeersWithAddrs, + "CertifiedAddresses": testCertifiedAddresses, } type AddrBookFactory func() (pstore.AddrBook, func()) @@ -331,3 +336,43 @@ func testPeersWithAddrs(m pstore.AddrBook) func(t *testing.T) { }) } } + +func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { + return func(t *testing.T) { + priv, _, err := test.RandTestKeyPair(crypto.Ed25519, 256) + if err != nil { + t.Errorf("error generating testing keys: %v", err) + } + + id, _ := peer.IDFromPrivateKey(priv) + allAddrs := GenerateAddrs(10) + certifiedAddrs := allAddrs[:5] + uncertifiedAddrs := allAddrs[5:] + info := peer.AddrInfo{ID: id, Addrs: certifiedAddrs} + envelope, err := routing.RoutingStateFromAddrInfo(&info).ToSignedEnvelope(priv) + if err != nil { + t.Errorf("error creating signed routing record: %v", err) + } + + // add the signed record to addr book + err = m.AddCertifiedAddrs(envelope, time.Hour) + if err != nil { + t.Errorf("error adding signed routing record to addrbook: %v", err) + } + + // add a few non-certified addrs + m.AddAddrs(id, uncertifiedAddrs, time.Hour) + + // we should get only certified addrs back from CertifiedAddrs + AssertAddressesEqual(t, certifiedAddrs, m.CertifiedAddrs(id)) + + // we should get everything back from Addrs + AssertAddressesEqual(t, allAddrs, m.Addrs(id)) + + // we should be able to retrieve the original envelope + envelope2 := m.SignedRoutingState(id) + if envelope2 == nil || !envelope.Equals(envelope2) { + t.Error("unable to retrieve signed routing record from addrbook") + } + } +} From a05a1e1b3d2d3a4ee0616b799f766617322bc55d Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 8 Nov 2019 13:08:17 -0500 Subject: [PATCH 05/39] fix bugs adding certified addrs --- pstoreds/addr_book.go | 42 ++++++++++++++++++++++++++++-------------- pstoremem/addr_book.go | 15 +++++++++++---- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 1ec38e2..c7877c0 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -252,32 +252,46 @@ func (ab *dsAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl tim } // ensure that the seq number from envelope is >= any previously received seq no - pr, err := ab.loadRecord(state.PeerID, true, false) + if ab.latestRoutingStateSeq(state.PeerID) >= state.Seq { + // TODO: should this be an error? + return nil + } + + addrs := cleanAddrs(state.Multiaddrs()) + err = ab.setAddrs(state.PeerID, addrs, ttl, ttlExtend, true) if err != nil { return err } - if pr.RoutingStateSeq >= state.Seq { - // TODO: should this be an error? - return nil + + return ab.storeRoutingState(state.PeerID, state.Seq, envelope) +} + +func (ab *dsAddrBook) latestRoutingStateSeq(p peer.ID) uint64 { + pr, err := ab.loadRecord(p, true, false) + if err != nil { + return 0 } + return pr.RoutingStateSeq +} +func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelope *crypto.SignedEnvelope) error { + // reload record and add routing state + // this has to be done after we add the addresses, since if + // we try to flush a datastore record with no addresses, + // it will just get deleted envelopeBytes, err := envelope.Marshal() if err != nil { return err } - - pr.RoutingStateSeq = state.Seq - pr.SignedRoutingRecord = envelopeBytes - pr.dirty = true - err = pr.flush(ab.ds) + pr, err := ab.loadRecord(p, true, false) if err != nil { return err } - - // TODO: remove addresses from previous RoutingStates (new state should completely replace old) - - addrs := cleanAddrs(state.Multiaddrs()) - return ab.setAddrs(state.PeerID, addrs, ttl, ttlExtend, true) + pr.RoutingStateSeq = seq + pr.SignedRoutingRecord = envelopeBytes + pr.dirty = true + err = pr.flush(ab.ds) + return err } func (ab *dsAddrBook) SignedRoutingState(p peer.ID) *crypto.SignedEnvelope { diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 77d16b0..0614f0d 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -69,9 +69,11 @@ func NewAddrBook() pstore.AddrBook { } return ret }(), - subManager: NewAddrSubManager(), - ctx: ctx, - cancel: cancel, + subManager: NewAddrSubManager(), + ctx: ctx, + cancel: cancel, + signedRoutingStates: make(map[peer.ID]*crypto.SignedEnvelope), + peerStateSeq: make(map[peer.ID]uint64), } go ab.background() @@ -232,7 +234,12 @@ func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // re-set all of them for new ttl. aBytes := addr.Bytes() if ttl > 0 { - amap[string(aBytes)] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl} + certified := false + existing := amap[string(aBytes)] + if existing != nil { + certified = existing.Certified + } + amap[string(aBytes)] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl, Certified: certified} mab.subManager.BroadcastAddr(p, addr) } else { delete(amap, string(aBytes)) From 731dde00211ee9864f0d484dad955eca1b2ff8a1 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 11 Nov 2019 10:42:24 -0500 Subject: [PATCH 06/39] use separate lists of signed & unsigned addrs --- pb/pstore.pb.go | 387 ++++++++++++++++++++++++++++++++--------- pb/pstore.proto | 22 ++- pb/pstorepb_test.go | 54 ++++++ pstoreds/addr_book.go | 83 ++++++--- pstoremem/addr_book.go | 232 +++++++++++++----------- 5 files changed, 560 insertions(+), 218 deletions(-) diff --git a/pb/pstore.pb.go b/pb/pstore.pb.go index d5c160b..b398e54 100644 --- a/pb/pstore.pb.go +++ b/pb/pstore.pb.go @@ -28,11 +28,10 @@ type AddrBookRecord struct { Id *ProtoPeerID `protobuf:"bytes,1,opt,name=id,proto3,customtype=ProtoPeerID" json:"id,omitempty"` // The multiaddresses. This is a sorted list where element 0 expires the soonest. Addrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,2,rep,name=addrs,proto3" json:"addrs,omitempty"` - // The Seq counter value from the most recent RoutingState record for this peer. - RoutingStateSeq uint64 `protobuf:"varint,3,opt,name=routingStateSeq,proto3" json:"routingStateSeq,omitempty"` - // A serialized SignedEnvelope containing the most recent RoutingState record. - // TODO: import the SignedEnvelope type instead - SignedRoutingRecord []byte `protobuf:"bytes,4,opt,name=signedRoutingRecord,proto3" json:"signedRoutingRecord,omitempty"` + // Multiaddresses that were sourced from a signed RoutingState record. + SignedAddrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,3,rep,name=signedAddrs,proto3" json:"signedAddrs,omitempty"` + // The most recently received signed RoutingState record. + CertifiedRecord *AddrBookRecord_CertifiedRecord `protobuf:"bytes,4,opt,name=certifiedRecord,proto3" json:"certifiedRecord,omitempty"` } func (m *AddrBookRecord) Reset() { *m = AddrBookRecord{} } @@ -75,16 +74,16 @@ func (m *AddrBookRecord) GetAddrs() []*AddrBookRecord_AddrEntry { return nil } -func (m *AddrBookRecord) GetRoutingStateSeq() uint64 { +func (m *AddrBookRecord) GetSignedAddrs() []*AddrBookRecord_AddrEntry { if m != nil { - return m.RoutingStateSeq + return m.SignedAddrs } - return 0 + return nil } -func (m *AddrBookRecord) GetSignedRoutingRecord() []byte { +func (m *AddrBookRecord) GetCertifiedRecord() *AddrBookRecord_CertifiedRecord { if m != nil { - return m.SignedRoutingRecord + return m.CertifiedRecord } return nil } @@ -96,8 +95,6 @@ type AddrBookRecord_AddrEntry struct { Expiry int64 `protobuf:"varint,2,opt,name=expiry,proto3" json:"expiry,omitempty"` // The original TTL of this address. Ttl int64 `protobuf:"varint,3,opt,name=ttl,proto3" json:"ttl,omitempty"` - // Whether this address was certified by the addressed peer. - Certified bool `protobuf:"varint,4,opt,name=certified,proto3" json:"certified,omitempty"` } func (m *AddrBookRecord_AddrEntry) Reset() { *m = AddrBookRecord_AddrEntry{} } @@ -147,42 +144,93 @@ func (m *AddrBookRecord_AddrEntry) GetTtl() int64 { return 0 } -func (m *AddrBookRecord_AddrEntry) GetCertified() bool { +// CertifiedRecord contains a serialized RoutingState record used to +// populate the signedAddrs list. +type AddrBookRecord_CertifiedRecord struct { + // The Seq counter from the RoutingState record + Seq uint64 `protobuf:"varint,1,opt,name=seq,proto3" json:"seq,omitempty"` + // The serialized bytes of the SignedEnvelope containing the RoutingState record. + Raw []byte `protobuf:"bytes,2,opt,name=raw,proto3" json:"raw,omitempty"` +} + +func (m *AddrBookRecord_CertifiedRecord) Reset() { *m = AddrBookRecord_CertifiedRecord{} } +func (m *AddrBookRecord_CertifiedRecord) String() string { return proto.CompactTextString(m) } +func (*AddrBookRecord_CertifiedRecord) ProtoMessage() {} +func (*AddrBookRecord_CertifiedRecord) Descriptor() ([]byte, []int) { + return fileDescriptor_f96873690e08a98f, []int{0, 1} +} +func (m *AddrBookRecord_CertifiedRecord) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *AddrBookRecord_CertifiedRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_AddrBookRecord_CertifiedRecord.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *AddrBookRecord_CertifiedRecord) XXX_Merge(src proto.Message) { + xxx_messageInfo_AddrBookRecord_CertifiedRecord.Merge(m, src) +} +func (m *AddrBookRecord_CertifiedRecord) XXX_Size() int { + return m.Size() +} +func (m *AddrBookRecord_CertifiedRecord) XXX_DiscardUnknown() { + xxx_messageInfo_AddrBookRecord_CertifiedRecord.DiscardUnknown(m) +} + +var xxx_messageInfo_AddrBookRecord_CertifiedRecord proto.InternalMessageInfo + +func (m *AddrBookRecord_CertifiedRecord) GetSeq() uint64 { + if m != nil { + return m.Seq + } + return 0 +} + +func (m *AddrBookRecord_CertifiedRecord) GetRaw() []byte { if m != nil { - return m.Certified + return m.Raw } - return false + return nil } func init() { proto.RegisterType((*AddrBookRecord)(nil), "pstore.pb.AddrBookRecord") proto.RegisterType((*AddrBookRecord_AddrEntry)(nil), "pstore.pb.AddrBookRecord.AddrEntry") + proto.RegisterType((*AddrBookRecord_CertifiedRecord)(nil), "pstore.pb.AddrBookRecord.CertifiedRecord") } func init() { proto.RegisterFile("pstore.proto", fileDescriptor_f96873690e08a98f) } var fileDescriptor_f96873690e08a98f = []byte{ - // 318 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x50, 0x31, 0x4f, 0xf3, 0x30, - 0x10, 0x8d, 0x93, 0x7c, 0xd5, 0x17, 0xb7, 0x50, 0x64, 0x24, 0x14, 0x55, 0xc8, 0x09, 0xb0, 0x64, - 0x21, 0x45, 0x30, 0x31, 0x12, 0xc1, 0xc0, 0x56, 0xb9, 0xbf, 0xa0, 0x89, 0xdd, 0x60, 0x01, 0x71, - 0x70, 0x1c, 0x44, 0xff, 0x05, 0x3f, 0xa9, 0x23, 0x63, 0x47, 0xd4, 0xa1, 0x82, 0xe4, 0x4f, 0x30, - 0xa2, 0x38, 0xa5, 0x08, 0xc4, 0x76, 0xef, 0xde, 0xbb, 0x7b, 0xef, 0x0e, 0xf6, 0xf2, 0x42, 0x09, - 0xc9, 0xc2, 0x5c, 0x0a, 0x25, 0x90, 0xf3, 0x85, 0xe2, 0xc1, 0x71, 0xca, 0xd5, 0x4d, 0x19, 0x87, - 0x89, 0xb8, 0x1f, 0xa6, 0x22, 0x15, 0x43, 0xad, 0x88, 0xcb, 0xa9, 0x46, 0x1a, 0xe8, 0xaa, 0x9d, - 0x3c, 0x9c, 0x9b, 0x70, 0xfb, 0x82, 0x52, 0x19, 0x09, 0x71, 0x4b, 0x58, 0x22, 0x24, 0x45, 0x1e, - 0x34, 0x39, 0x75, 0x81, 0x0f, 0x82, 0x5e, 0xd4, 0x5f, 0xae, 0xbc, 0xee, 0xa8, 0x51, 0x8e, 0x18, - 0x93, 0xd7, 0x97, 0xc4, 0xe4, 0x14, 0x9d, 0xc3, 0x7f, 0x13, 0x4a, 0x65, 0xe1, 0x9a, 0xbe, 0x15, - 0x74, 0x4f, 0x8f, 0xc2, 0x8d, 0x7b, 0xf8, 0x73, 0x95, 0x86, 0x57, 0x99, 0x92, 0x33, 0xd2, 0x4e, - 0xa0, 0x00, 0xf6, 0xa5, 0x28, 0x15, 0xcf, 0xd2, 0xb1, 0x9a, 0x28, 0x36, 0x66, 0x0f, 0xae, 0xe5, - 0x83, 0xc0, 0x26, 0xbf, 0xdb, 0xe8, 0x04, 0xee, 0x16, 0x3c, 0xcd, 0x18, 0x25, 0x2d, 0xd1, 0x6e, - 0x74, 0xed, 0x26, 0x16, 0xf9, 0x8b, 0x1a, 0x3c, 0x42, 0x67, 0xe3, 0x87, 0x0e, 0xa0, 0xdd, 0x38, - 0xae, 0xcf, 0xd8, 0x5a, 0xae, 0x3c, 0x47, 0x9f, 0xd1, 0x28, 0x88, 0xa6, 0xd0, 0x1e, 0xec, 0xb0, - 0xa7, 0x9c, 0xcb, 0x99, 0x6b, 0xfa, 0x20, 0xb0, 0xc8, 0x1a, 0xa1, 0x1d, 0x68, 0x29, 0x75, 0xa7, - 0x73, 0x59, 0xa4, 0x29, 0xd1, 0x3e, 0x74, 0x12, 0x26, 0x15, 0x9f, 0x72, 0xd6, 0x26, 0xf8, 0x4f, - 0xbe, 0x1b, 0x91, 0xff, 0xf1, 0x8e, 0xc1, 0xbc, 0xc2, 0xe0, 0xa5, 0xc2, 0x60, 0x51, 0x61, 0xf0, - 0x56, 0x61, 0xf0, 0x5c, 0x63, 0x63, 0x51, 0x63, 0xe3, 0xb5, 0xc6, 0x46, 0xdc, 0xd1, 0xbf, 0x3e, - 0xfb, 0x0c, 0x00, 0x00, 0xff, 0xff, 0xbd, 0x79, 0x1e, 0x1b, 0xb5, 0x01, 0x00, 0x00, + // 331 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x91, 0xb1, 0x4f, 0xc2, 0x40, + 0x14, 0xc6, 0xb9, 0x16, 0x49, 0xb8, 0xa2, 0x98, 0x1b, 0x4c, 0xc3, 0x70, 0x54, 0x5d, 0xea, 0x60, + 0x49, 0x30, 0x0e, 0x8e, 0xa0, 0x0c, 0x6e, 0xe4, 0x5c, 0x5c, 0x29, 0x77, 0xd4, 0x8b, 0xca, 0xe1, + 0xf5, 0x88, 0xf2, 0x5f, 0xb8, 0xfa, 0xdf, 0x38, 0x3a, 0x32, 0x1a, 0x06, 0xa2, 0xe5, 0x9f, 0x70, + 0x34, 0xf7, 0x8a, 0x44, 0x48, 0x4c, 0xdc, 0xbe, 0xef, 0xf5, 0xfb, 0x7e, 0xaf, 0x79, 0x87, 0x2b, + 0xa3, 0xd4, 0x28, 0x2d, 0xa2, 0x91, 0x56, 0x46, 0x91, 0xf2, 0x8f, 0x8b, 0x6b, 0xc7, 0x89, 0x34, + 0x37, 0xe3, 0x38, 0xea, 0xab, 0xfb, 0x46, 0xa2, 0x12, 0xd5, 0x80, 0x44, 0x3c, 0x1e, 0x80, 0x03, + 0x03, 0x2a, 0x6f, 0x1e, 0xbc, 0xb8, 0x78, 0xa7, 0xc5, 0xb9, 0x6e, 0x2b, 0x75, 0xcb, 0x44, 0x5f, + 0x69, 0x4e, 0xea, 0xd8, 0x91, 0xdc, 0x47, 0x01, 0x0a, 0x2b, 0xed, 0xea, 0x6c, 0x5e, 0xf7, 0xba, + 0x36, 0xd9, 0x15, 0x42, 0x5f, 0x5e, 0x30, 0x47, 0x72, 0x72, 0x86, 0xb7, 0x7a, 0x9c, 0xeb, 0xd4, + 0x77, 0x02, 0x37, 0xf4, 0x9a, 0x87, 0xd1, 0x6a, 0x7b, 0xb4, 0x8e, 0x02, 0xdb, 0x19, 0x1a, 0x3d, + 0x61, 0x79, 0x83, 0x74, 0xb0, 0x97, 0xca, 0x64, 0x28, 0x78, 0x0b, 0x00, 0xee, 0xff, 0x01, 0xbf, + 0x7b, 0xe4, 0x0a, 0x57, 0xfb, 0x42, 0x1b, 0x39, 0x90, 0x82, 0xe7, 0x49, 0xbf, 0x18, 0xa0, 0xd0, + 0x6b, 0x1e, 0xfd, 0x8d, 0x3a, 0x5f, 0x2f, 0xb0, 0x4d, 0x42, 0xed, 0x1a, 0x97, 0x57, 0xeb, 0xc8, + 0x3e, 0x2e, 0xda, 0x3f, 0x5e, 0x9e, 0x61, 0x7b, 0x36, 0xaf, 0x97, 0xe1, 0x0c, 0x36, 0xc1, 0xe0, + 0x13, 0xd9, 0xc3, 0x25, 0xf1, 0x34, 0x92, 0x7a, 0xe2, 0x3b, 0x01, 0x0a, 0x5d, 0xb6, 0x74, 0x64, + 0x17, 0xbb, 0xc6, 0xdc, 0xf9, 0x2e, 0x0c, 0xad, 0xac, 0x9d, 0xe2, 0xea, 0xc6, 0x76, 0x1b, 0x4a, + 0xc5, 0x03, 0xe0, 0x8b, 0xcc, 0x4a, 0x3b, 0xd1, 0xbd, 0x47, 0x60, 0x55, 0x98, 0x95, 0xed, 0xe0, + 0xeb, 0x93, 0xa2, 0xd7, 0x8c, 0xa2, 0xb7, 0x8c, 0xa2, 0x69, 0x46, 0xd1, 0x47, 0x46, 0xd1, 0xf3, + 0x82, 0x16, 0xa6, 0x0b, 0x5a, 0x78, 0x5f, 0xd0, 0x42, 0x5c, 0x82, 0x47, 0x3c, 0xf9, 0x0e, 0x00, + 0x00, 0xff, 0xff, 0x26, 0xc5, 0x77, 0x15, 0x0e, 0x02, 0x00, 0x00, } func (m *AddrBookRecord) Marshal() (dAtA []byte, err error) { @@ -222,16 +270,27 @@ func (m *AddrBookRecord) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.RoutingStateSeq != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPstore(dAtA, i, uint64(m.RoutingStateSeq)) + if len(m.SignedAddrs) > 0 { + for _, msg := range m.SignedAddrs { + dAtA[i] = 0x1a + i++ + i = encodeVarintPstore(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } } - if len(m.SignedRoutingRecord) > 0 { + if m.CertifiedRecord != nil { dAtA[i] = 0x22 i++ - i = encodeVarintPstore(dAtA, i, uint64(len(m.SignedRoutingRecord))) - i += copy(dAtA[i:], m.SignedRoutingRecord) + i = encodeVarintPstore(dAtA, i, uint64(m.CertifiedRecord.Size())) + n2, err := m.CertifiedRecord.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n2 } return i, nil } @@ -255,11 +314,11 @@ func (m *AddrBookRecord_AddrEntry) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPstore(dAtA, i, uint64(m.Addr.Size())) - n2, err := m.Addr.MarshalTo(dAtA[i:]) + n3, err := m.Addr.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n2 + i += n3 } if m.Expiry != 0 { dAtA[i] = 0x10 @@ -271,15 +330,34 @@ func (m *AddrBookRecord_AddrEntry) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPstore(dAtA, i, uint64(m.Ttl)) } - if m.Certified { - dAtA[i] = 0x20 + return i, nil +} + +func (m *AddrBookRecord_CertifiedRecord) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AddrBookRecord_CertifiedRecord) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Seq != 0 { + dAtA[i] = 0x8 i++ - if m.Certified { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } + i = encodeVarintPstore(dAtA, i, uint64(m.Seq)) + } + if len(m.Raw) > 0 { + dAtA[i] = 0x12 i++ + i = encodeVarintPstore(dAtA, i, uint64(len(m.Raw))) + i += copy(dAtA[i:], m.Raw) } return i, nil } @@ -303,11 +381,15 @@ func NewPopulatedAddrBookRecord(r randyPstore, easy bool) *AddrBookRecord { this.Addrs[i] = NewPopulatedAddrBookRecord_AddrEntry(r, easy) } } - this.RoutingStateSeq = uint64(uint64(r.Uint32())) - v2 := r.Intn(100) - this.SignedRoutingRecord = make([]byte, v2) - for i := 0; i < v2; i++ { - this.SignedRoutingRecord[i] = byte(r.Intn(256)) + if r.Intn(10) != 0 { + v2 := r.Intn(5) + this.SignedAddrs = make([]*AddrBookRecord_AddrEntry, v2) + for i := 0; i < v2; i++ { + this.SignedAddrs[i] = NewPopulatedAddrBookRecord_AddrEntry(r, easy) + } + } + if r.Intn(10) != 0 { + this.CertifiedRecord = NewPopulatedAddrBookRecord_CertifiedRecord(r, easy) } if !easy && r.Intn(10) != 0 { } @@ -325,7 +407,19 @@ func NewPopulatedAddrBookRecord_AddrEntry(r randyPstore, easy bool) *AddrBookRec if r.Intn(2) == 0 { this.Ttl *= -1 } - this.Certified = bool(bool(r.Intn(2) == 0)) + if !easy && r.Intn(10) != 0 { + } + return this +} + +func NewPopulatedAddrBookRecord_CertifiedRecord(r randyPstore, easy bool) *AddrBookRecord_CertifiedRecord { + this := &AddrBookRecord_CertifiedRecord{} + this.Seq = uint64(uint64(r.Uint32())) + v3 := r.Intn(100) + this.Raw = make([]byte, v3) + for i := 0; i < v3; i++ { + this.Raw[i] = byte(r.Intn(256)) + } if !easy && r.Intn(10) != 0 { } return this @@ -350,9 +444,9 @@ func randUTF8RunePstore(r randyPstore) rune { return rune(ru + 61) } func randStringPstore(r randyPstore) string { - v3 := r.Intn(100) - tmps := make([]rune, v3) - for i := 0; i < v3; i++ { + v4 := r.Intn(100) + tmps := make([]rune, v4) + for i := 0; i < v4; i++ { tmps[i] = randUTF8RunePstore(r) } return string(tmps) @@ -374,11 +468,11 @@ func randFieldPstore(dAtA []byte, r randyPstore, fieldNumber int, wire int) []by switch wire { case 0: dAtA = encodeVarintPopulatePstore(dAtA, uint64(key)) - v4 := r.Int63() + v5 := r.Int63() if r.Intn(2) == 0 { - v4 *= -1 + v5 *= -1 } - dAtA = encodeVarintPopulatePstore(dAtA, uint64(v4)) + dAtA = encodeVarintPopulatePstore(dAtA, uint64(v5)) case 1: dAtA = encodeVarintPopulatePstore(dAtA, uint64(key)) dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) @@ -419,11 +513,14 @@ func (m *AddrBookRecord) Size() (n int) { n += 1 + l + sovPstore(uint64(l)) } } - if m.RoutingStateSeq != 0 { - n += 1 + sovPstore(uint64(m.RoutingStateSeq)) + if len(m.SignedAddrs) > 0 { + for _, e := range m.SignedAddrs { + l = e.Size() + n += 1 + l + sovPstore(uint64(l)) + } } - l = len(m.SignedRoutingRecord) - if l > 0 { + if m.CertifiedRecord != nil { + l = m.CertifiedRecord.Size() n += 1 + l + sovPstore(uint64(l)) } return n @@ -445,8 +542,21 @@ func (m *AddrBookRecord_AddrEntry) Size() (n int) { if m.Ttl != 0 { n += 1 + sovPstore(uint64(m.Ttl)) } - if m.Certified { - n += 2 + return n +} + +func (m *AddrBookRecord_CertifiedRecord) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Seq != 0 { + n += 1 + sovPstore(uint64(m.Seq)) + } + l = len(m.Raw) + if l > 0 { + n += 1 + l + sovPstore(uint64(l)) } return n } @@ -563,10 +673,10 @@ func (m *AddrBookRecord) Unmarshal(dAtA []byte) error { } iNdEx = postIndex case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RoutingStateSeq", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SignedAddrs", wireType) } - m.RoutingStateSeq = 0 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPstore @@ -576,16 +686,31 @@ func (m *AddrBookRecord) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.RoutingStateSeq |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } + if msglen < 0 { + return ErrInvalidLengthPstore + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPstore + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SignedAddrs = append(m.SignedAddrs, &AddrBookRecord_AddrEntry{}) + if err := m.SignedAddrs[len(m.SignedAddrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SignedRoutingRecord", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CertifiedRecord", wireType) } - var byteLen int + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPstore @@ -595,24 +720,26 @@ func (m *AddrBookRecord) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { + if msglen < 0 { return ErrInvalidLengthPstore } - postIndex := iNdEx + byteLen + postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthPstore } if postIndex > l { return io.ErrUnexpectedEOF } - m.SignedRoutingRecord = append(m.SignedRoutingRecord[:0], dAtA[iNdEx:postIndex]...) - if m.SignedRoutingRecord == nil { - m.SignedRoutingRecord = []byte{} + if m.CertifiedRecord == nil { + m.CertifiedRecord = &AddrBookRecord_CertifiedRecord{} + } + if err := m.CertifiedRecord.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err } iNdEx = postIndex default: @@ -741,11 +868,83 @@ func (m *AddrBookRecord_AddrEntry) Unmarshal(dAtA []byte) error { break } } - case 4: + default: + iNdEx = preIndex + skippy, err := skipPstore(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPstore + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthPstore + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *AddrBookRecord_CertifiedRecord) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPstore + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CertifiedRecord: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CertifiedRecord: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Certified", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Seq", wireType) + } + m.Seq = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPstore + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Seq |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Raw", wireType) } - var v int + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPstore @@ -755,12 +954,26 @@ func (m *AddrBookRecord_AddrEntry) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } - m.Certified = bool(v != 0) + if byteLen < 0 { + return ErrInvalidLengthPstore + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthPstore + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Raw = append(m.Raw[:0], dAtA[iNdEx:postIndex]...) + if m.Raw == nil { + m.Raw = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPstore(dAtA[iNdEx:]) diff --git a/pb/pstore.proto b/pb/pstore.proto index 410cdab..e0724e2 100644 --- a/pb/pstore.proto +++ b/pb/pstore.proto @@ -14,6 +14,12 @@ message AddrBookRecord { // The multiaddresses. This is a sorted list where element 0 expires the soonest. repeated AddrEntry addrs = 2; + // Multiaddresses that were sourced from a signed RoutingState record. + repeated AddrEntry signedAddrs = 3; + + // The most recently received signed RoutingState record. + CertifiedRecord certifiedRecord = 4; + // AddrEntry represents a single multiaddress. message AddrEntry { bytes addr = 1 [(gogoproto.customtype) = "ProtoAddr"]; @@ -23,15 +29,15 @@ message AddrBookRecord { // The original TTL of this address. int64 ttl = 3; - - // Whether this address was certified by the addressed peer. - bool certified = 4; } - // The Seq counter value from the most recent RoutingState record for this peer. - uint64 routingStateSeq = 3; + // CertifiedRecord contains a serialized RoutingState record used to + // populate the signedAddrs list. + message CertifiedRecord { + // The Seq counter from the RoutingState record + uint64 seq = 1; - // A serialized SignedEnvelope containing the most recent RoutingState record. - // TODO: import the SignedEnvelope type instead - bytes signedRoutingRecord = 4; + // The serialized bytes of the SignedEnvelope containing the RoutingState record. + bytes raw = 2; + } } diff --git a/pb/pstorepb_test.go b/pb/pstorepb_test.go index 3569d04..12d2980 100644 --- a/pb/pstorepb_test.go +++ b/pb/pstorepb_test.go @@ -98,6 +98,46 @@ func BenchmarkAddrBookRecord_AddrEntryProtoUnmarshal(b *testing.B) { b.SetBytes(int64(total / b.N)) } +func BenchmarkAddrBookRecord_CertifiedRecordProtoMarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*AddrBookRecord_CertifiedRecord, 10000) + for i := 0; i < 10000; i++ { + pops[i] = NewPopulatedAddrBookRecord_CertifiedRecord(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) + if err != nil { + panic(err) + } + total += len(dAtA) + } + b.SetBytes(int64(total / b.N)) +} + +func BenchmarkAddrBookRecord_CertifiedRecordProtoUnmarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + datas := make([][]byte, 10000) + for i := 0; i < 10000; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedAddrBookRecord_CertifiedRecord(popr, false)) + if err != nil { + panic(err) + } + datas[i] = dAtA + } + msg := &AddrBookRecord_CertifiedRecord{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += len(datas[i%10000]) + if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { + panic(err) + } + } + b.SetBytes(int64(total / b.N)) +} + func BenchmarkAddrBookRecordSize(b *testing.B) { popr := math_rand.New(math_rand.NewSource(616)) total := 0 @@ -126,4 +166,18 @@ func BenchmarkAddrBookRecord_AddrEntrySize(b *testing.B) { b.SetBytes(int64(total / b.N)) } +func BenchmarkAddrBookRecord_CertifiedRecordSize(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*AddrBookRecord_CertifiedRecord, 1000) + for i := 0; i < 1000; i++ { + pops[i] = NewPopulatedAddrBookRecord_CertifiedRecord(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += pops[i%1000].Size() + } + b.SetBytes(int64(total / b.N)) +} + //These tests are generated by github.com/gogo/protobuf/plugin/testgen diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index c7877c0..40d31a2 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -49,7 +49,7 @@ type addrsRecord struct { // marked for deletion, in which case we call ds.Delete. To be called within a lock. func (r *addrsRecord) flush(write ds.Write) (err error) { key := addrBookBase.ChildString(b32.RawStdEncoding.EncodeToString([]byte(r.Id.ID))) - if len(r.Addrs) == 0 { + if len(r.Addrs) == 0 && len(r.SignedAddrs) == 0 { if err = write.Delete(key); err == nil { r.dirty = false } @@ -85,37 +85,60 @@ func (r *addrsRecord) flush(write ds.Write) (err error) { // If the return value is true, the caller should perform a flush immediately to sync the record with the store. func (r *addrsRecord) clean() (chgd bool) { now := time.Now().Unix() - if !r.dirty && len(r.Addrs) > 0 && r.Addrs[0].Expiry > now { + addrsLen := len(r.Addrs) + signedAddrsLen := len(r.SignedAddrs) + + if !r.dirty && !r.hasExpiredAddrs(now) { // record is not dirty, and we have no expired entries to purge. return false } - if len(r.Addrs) == 0 { + if addrsLen == 0 && signedAddrsLen == 0 { // this is a ghost record; let's signal it has to be written. // flush() will take care of doing the deletion. return true } - if r.dirty && len(r.Addrs) > 1 { - // the record has been modified, so it may need resorting. - // we keep addresses sorted by expiration, where 0 is the soonest expiring. + if r.dirty && addrsLen > 1 { + sort.Slice(r.Addrs, func(i, j int) bool { + return r.Addrs[i].Expiry < r.Addrs[j].Expiry + }) + } + + if r.dirty && signedAddrsLen > 1 { sort.Slice(r.Addrs, func(i, j int) bool { return r.Addrs[i].Expiry < r.Addrs[j].Expiry }) } + r.Addrs = removeExpired(r.Addrs, now) + r.SignedAddrs = removeExpired(r.SignedAddrs, now) + + return r.dirty || len(r.Addrs) != addrsLen || len(r.SignedAddrs) != signedAddrsLen +} + +func (r *addrsRecord) hasExpiredAddrs(now int64) bool { + if len(r.Addrs) > 0 && r.Addrs[0].Expiry <= now { + return true + } + if len(r.SignedAddrs) > 0 && r.SignedAddrs[0].Expiry <= now { + return true + } + return false +} + +func removeExpired(entries []*pb.AddrBookRecord_AddrEntry, now int64) []*pb.AddrBookRecord_AddrEntry { // since addresses are sorted by expiration, we find the first // survivor and split the slice on its index. pivot := -1 - for i, addr := range r.Addrs { + for i, addr := range entries { if addr.Expiry > now { break } pivot = i } - r.Addrs = r.Addrs[pivot+1:] - return r.dirty || pivot >= 0 + return entries[pivot+1:] } // dsAddrBook is an address book backed by a Datastore with a GC procedure to purge expired entries. It uses an @@ -268,10 +291,10 @@ func (ab *dsAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl tim func (ab *dsAddrBook) latestRoutingStateSeq(p peer.ID) uint64 { pr, err := ab.loadRecord(p, true, false) - if err != nil { + if err != nil || pr.CertifiedRecord == nil { return 0 } - return pr.RoutingStateSeq + return pr.CertifiedRecord.Seq } func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelope *crypto.SignedEnvelope) error { @@ -287,8 +310,10 @@ func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelope *crypto. if err != nil { return err } - pr.RoutingStateSeq = seq - pr.SignedRoutingRecord = envelopeBytes + pr.CertifiedRecord = &pb.AddrBookRecord_CertifiedRecord{ + Seq: seq, + Raw: envelopeBytes, + } pr.dirty = true err = pr.flush(ab.ds) return err @@ -300,10 +325,10 @@ func (ab *dsAddrBook) SignedRoutingState(p peer.ID) *crypto.SignedEnvelope { log.Errorf("unable to load record for peer %s: %v", p.Pretty(), err) return nil } - if len(pr.SignedRoutingRecord) == 0 { + if len(pr.CertifiedRecord.Raw) == 0 { return nil } - envelope, err := crypto.UnmarshalEnvelope(pr.SignedRoutingRecord) + envelope, err := crypto.UnmarshalEnvelope(pr.CertifiedRecord.Raw) if err != nil { log.Errorf("unable to unmarshal stored signed routing record for peer %s: %v", p.Pretty(), err) return nil @@ -339,7 +364,8 @@ func (ab *dsAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.D defer pr.Unlock() newExp := time.Now().Add(newTTL).Unix() - for _, entry := range pr.Addrs { + allAddrs := append(pr.Addrs, pr.SignedAddrs...) + for _, entry := range allAddrs { if entry.Ttl != int64(oldTTL) { continue } @@ -374,11 +400,14 @@ func (ab *dsAddrBook) addrs(p peer.ID, includeUncertified bool) []ma.Multiaddr { defer pr.RUnlock() addrs := make([]ma.Multiaddr, 0, len(pr.Addrs)) - for _, a := range pr.Addrs { - if includeUncertified || a.Certified { + if includeUncertified { + for _, a := range pr.Addrs { addrs = append(addrs, a.Addr) } } + for _, a := range pr.SignedAddrs { + addrs = append(addrs, a.Addr) + } return addrs } @@ -422,9 +451,16 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio newExp := time.Now().Add(ttl).Unix() existed := make([]bool, len(addrs)) // keeps track of which addrs we found. + var entryList *[]*pb.AddrBookRecord_AddrEntry + if certified { + entryList = &pr.SignedAddrs + } else { + entryList = &pr.Addrs + } + Outer: for i, incoming := range addrs { - for _, have := range pr.Addrs { + for _, have := range *entryList { if incoming.Equal(have.Addr) { existed[i] = true switch mode { @@ -457,10 +493,9 @@ Outer: } addr := addrs[i] entry := &pb.AddrBookRecord_AddrEntry{ - Addr: &pb.ProtoAddr{Multiaddr: addr}, - Ttl: int64(ttl), - Expiry: newExp, - Certified: certified, + Addr: &pb.ProtoAddr{Multiaddr: addr}, + Ttl: int64(ttl), + Expiry: newExp, } added = append(added, entry) // note: there's a minor chance that writing the record will fail, in which case we would've broadcast @@ -468,7 +503,7 @@ Outer: ab.subsManager.BroadcastAddr(p, addr) } - pr.Addrs = append(pr.Addrs, added...) + *entryList = append(*entryList, added...) pr.dirty = true pr.clean() return pr.flush(ab.ds) diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 0614f0d..4c5ce69 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -40,16 +40,98 @@ type addrSegment struct { addrs map[peer.ID]map[string]*expiringAddr } -func (s *addrSegments) get(p peer.ID) *addrSegment { - return s[byte(p[len(p)-1])] +func (s *addrSegment) validAddrs(p peer.ID) []ma.Multiaddr { + s.RLock() + defer s.RUnlock() + + amap, found := s.addrs[p] + if !found { + return nil + } + + now := time.Now() + good := make([]ma.Multiaddr, 0, len(amap)) + for _, m := range amap { + if !m.ExpiredBy(now) { + good = append(good, m.Addr) + } + } + + return good +} + +func (s *addrSegment) updateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.Duration) { + s.Lock() + defer s.Unlock() + + amap, found := s.addrs[p] + if !found { + return + } + + exp := time.Now().Add(newTTL) + for k, addr := range amap { + if oldTTL == addr.TTL { + addr.TTL = newTTL + addr.Expires = exp + amap[k] = addr + } + } +} + +func (segments *addrSegments) get(p peer.ID) *addrSegment { + return segments[byte(p[len(p)-1])] +} + +func (segments *addrSegments) gc() { + now := time.Now() + for _, s := range segments { + s.Lock() + for p, amap := range s.addrs { + for k, addr := range amap { + if addr.ExpiredBy(now) { + delete(amap, k) + } + } + if len(amap) == 0 { + delete(s.addrs, p) + } + } + s.Unlock() + } +} + +func (segments *addrSegments) peersWithAddrs() peer.IDSlice { + var pids peer.IDSlice + for _, s := range segments { + s.RLock() + for pid, _ := range s.addrs { + pids = append(pids, pid) + } + s.RUnlock() + } + return pids +} + +func (segments *addrSegments) clear(p peer.ID) { + s := segments.get(p) + s.Lock() + defer s.Unlock() + + delete(s.addrs, p) +} + +type certifiedRecord struct { + Seq uint64 + Envelope *crypto.SignedEnvelope } // memoryAddrBook manages addresses. type memoryAddrBook struct { - segments addrSegments + segments addrSegments + signedSegments addrSegments - signedRoutingStates map[peer.ID]*crypto.SignedEnvelope - peerStateSeq map[peer.ID]uint64 + signedRoutingStates map[peer.ID]*certifiedRecord ctx context.Context cancel func() @@ -62,18 +144,20 @@ var _ pstore.AddrBook = (*memoryAddrBook)(nil) func NewAddrBook() pstore.AddrBook { ctx, cancel := context.WithCancel(context.Background()) + makeSegments := func() (ret addrSegments) { + for i, _ := range ret { + ret[i] = &addrSegment{addrs: make(map[peer.ID]map[string]*expiringAddr)} + } + return ret + } + ab := &memoryAddrBook{ - segments: func() (ret addrSegments) { - for i, _ := range ret { - ret[i] = &addrSegment{addrs: make(map[peer.ID]map[string]*expiringAddr)} - } - return ret - }(), + segments: makeSegments(), + signedSegments: makeSegments(), subManager: NewAddrSubManager(), ctx: ctx, cancel: cancel, - signedRoutingStates: make(map[peer.ID]*crypto.SignedEnvelope), - peerStateSeq: make(map[peer.ID]uint64), + signedRoutingStates: make(map[peer.ID]*certifiedRecord), } go ab.background() @@ -103,34 +187,18 @@ func (mab *memoryAddrBook) Close() error { // gc garbage collects the in-memory address book. func (mab *memoryAddrBook) gc() { - now := time.Now() - for _, s := range mab.segments { - s.Lock() - for p, amap := range s.addrs { - for k, addr := range amap { - if addr.ExpiredBy(now) { - delete(amap, k) - } - } - if len(amap) == 0 { - delete(s.addrs, p) - } + mab.segments.gc() + mab.signedSegments.gc() + + for _, p := range mab.signedSegments.peersWithAddrs() { + if len(mab.CertifiedAddrs(p)) == 0 { + delete(mab.signedRoutingStates, p) } - s.Unlock() } - } func (mab *memoryAddrBook) PeersWithAddrs() peer.IDSlice { - var pids peer.IDSlice - for _, s := range mab.segments { - s.RLock() - for pid, _ := range s.addrs { - pids = append(pids, pid) - } - s.RUnlock() - } - return pids + return append(mab.signedSegments.peersWithAddrs(), mab.segments.peersWithAddrs()...) } // AddAddr calls AddAddrs(p, []ma.Multiaddr{addr}, ttl) @@ -152,27 +220,28 @@ func (mab *memoryAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, tt } // ensure seq is greater than last received - lastSeq, found := mab.peerStateSeq[state.PeerID] - if found && lastSeq >= state.Seq { + lastState, found := mab.signedRoutingStates[state.PeerID] + if found && lastState.Seq >= state.Seq { // TODO: should this be an error? return nil } - mab.peerStateSeq[state.PeerID] = state.Seq - mab.signedRoutingStates[state.PeerID] = envelope - - // TODO: remove addresses from previous RoutingStates (new state should completely replace old) - + mab.signedRoutingStates[state.PeerID] = &certifiedRecord{Seq: state.Seq, Envelope: envelope} mab.addAddrs(state.PeerID, state.Multiaddrs(), ttl, true) return nil } -func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration, certified bool) { +func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration, signed bool) { // if ttl is zero, exit. nothing to do. if ttl <= 0 { return } - s := mab.segments.get(p) + var s *addrSegment + if signed { + s = mab.signedSegments.get(p) + } else { + s = mab.segments.get(p) + } s.Lock() defer s.Unlock() @@ -191,7 +260,7 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du a, found := amap[string(asBytes)] // won't allocate. if !found { // not found, save and announce it. - amap[string(asBytes)] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl, Certified: certified} + amap[string(asBytes)] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl} mab.subManager.BroadcastAddr(p, addr) } else { // Update expiration/TTL independently. @@ -234,12 +303,7 @@ func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // re-set all of them for new ttl. aBytes := addr.Bytes() if ttl > 0 { - certified := false - existing := amap[string(aBytes)] - if existing != nil { - certified = existing.Certified - } - amap[string(aBytes)] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl, Certified: certified} + amap[string(aBytes)] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl} mab.subManager.BroadcastAddr(p, addr) } else { delete(amap, string(aBytes)) @@ -250,71 +314,41 @@ func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // UpdateAddrs updates the addresses associated with the given peer that have // the given oldTTL to have the given newTTL. func (mab *memoryAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.Duration) { - s := mab.segments.get(p) - s.Lock() - defer s.Unlock() - - amap, found := s.addrs[p] - if !found { - return - } + mab.segments.get(p).updateAddrs(p, oldTTL, newTTL) + mab.signedSegments.get(p).updateAddrs(p, oldTTL, newTTL) - exp := time.Now().Add(newTTL) - for k, addr := range amap { - if oldTTL == addr.TTL { - addr.TTL = newTTL - addr.Expires = exp - amap[k] = addr - } + // updating may have caused all our certified addrs to expire. + // if so, remove the signed record we sourced them from. + if len(mab.CertifiedAddrs(p)) == 0 { + delete(mab.signedRoutingStates, p) } } // Addrs returns all known (and valid) addresses for a given peer func (mab *memoryAddrBook) Addrs(p peer.ID) []ma.Multiaddr { - return mab.addrs(p, true) + return append( + mab.signedSegments.get(p).validAddrs(p), + mab.segments.get(p).validAddrs(p)...) } // CertifiedAddrs returns all known (and valid) addressed that have been // certified by the given peer. func (mab *memoryAddrBook) CertifiedAddrs(p peer.ID) []ma.Multiaddr { - return mab.addrs(p, false) -} - -func (mab *memoryAddrBook) addrs(p peer.ID, includeUncertified bool) []ma.Multiaddr { - s := mab.segments.get(p) - s.RLock() - defer s.RUnlock() - - amap, found := s.addrs[p] - if !found { - return nil - } - - now := time.Now() - good := make([]ma.Multiaddr, 0, len(amap)) - for _, m := range amap { - if m.ExpiredBy(now) { - continue - } - if includeUncertified || m.Certified { - good = append(good, m.Addr) - } - } - - return good + return mab.signedSegments.get(p).validAddrs(p) } func (mab *memoryAddrBook) SignedRoutingState(p peer.ID) *crypto.SignedEnvelope { - return mab.signedRoutingStates[p] + r := mab.signedRoutingStates[p] + if r != nil { + return r.Envelope + } + return nil } // ClearAddrs removes all previously stored addresses func (mab *memoryAddrBook) ClearAddrs(p peer.ID) { - s := mab.segments.get(p) - s.Lock() - defer s.Unlock() - - delete(s.addrs, p) + mab.segments.clear(p) + mab.signedSegments.clear(p) } // AddrStream returns a channel on which all new addresses discovered for a From 1c5732a7897b8053f6fa790b17e560eb75900ffb Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 11 Nov 2019 12:16:53 -0500 Subject: [PATCH 07/39] remove signed record when addrs expire --- pstoreds/addr_book.go | 19 ++++++++++++------- test/addr_book_suite.go | 7 +++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 40d31a2..3276e57 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -49,6 +49,11 @@ type addrsRecord struct { // marked for deletion, in which case we call ds.Delete. To be called within a lock. func (r *addrsRecord) flush(write ds.Write) (err error) { key := addrBookBase.ChildString(b32.RawStdEncoding.EncodeToString([]byte(r.Id.ID))) + + if len(r.SignedAddrs) == 0 { + r.CertifiedRecord = nil + } + if len(r.Addrs) == 0 && len(r.SignedAddrs) == 0 { if err = write.Delete(key); err == nil { r.dirty = false @@ -291,7 +296,7 @@ func (ab *dsAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl tim func (ab *dsAddrBook) latestRoutingStateSeq(p peer.ID) uint64 { pr, err := ab.loadRecord(p, true, false) - if err != nil || pr.CertifiedRecord == nil { + if err != nil || pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { return 0 } return pr.CertifiedRecord.Seq @@ -325,7 +330,7 @@ func (ab *dsAddrBook) SignedRoutingState(p peer.ID) *crypto.SignedEnvelope { log.Errorf("unable to load record for peer %s: %v", p.Pretty(), err) return nil } - if len(pr.CertifiedRecord.Raw) == 0 { + if pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { return nil } envelope, err := crypto.UnmarshalEnvelope(pr.CertifiedRecord.Raw) @@ -400,14 +405,14 @@ func (ab *dsAddrBook) addrs(p peer.ID, includeUncertified bool) []ma.Multiaddr { defer pr.RUnlock() addrs := make([]ma.Multiaddr, 0, len(pr.Addrs)) + for _, a := range pr.SignedAddrs { + addrs = append(addrs, a.Addr) + } if includeUncertified { for _, a := range pr.Addrs { addrs = append(addrs, a.Addr) } } - for _, a := range pr.SignedAddrs { - addrs = append(addrs, a.Addr) - } return addrs } @@ -439,7 +444,7 @@ func (ab *dsAddrBook) ClearAddrs(p peer.ID) { } } -func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration, mode ttlWriteMode, certified bool) (err error) { +func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration, mode ttlWriteMode, signed bool) (err error) { pr, err := ab.loadRecord(p, true, false) if err != nil { return fmt.Errorf("failed to load peerstore entry for peer %v while setting addrs, err: %v", p, err) @@ -452,7 +457,7 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio existed := make([]bool, len(addrs)) // keeps track of which addrs we found. var entryList *[]*pb.AddrBookRecord_AddrEntry - if certified { + if signed { entryList = &pr.SignedAddrs } else { entryList = &pr.Addrs diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index 3c7d083..3a24c1c 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -374,5 +374,12 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { if envelope2 == nil || !envelope.Equals(envelope2) { t.Error("unable to retrieve signed routing record from addrbook") } + + // update TTL on signed addrs to -1 to remove them. + // the signed routing record should be deleted + m.UpdateAddrs(id, time.Hour, -1) + if m.SignedRoutingState(id) != nil { + t.Error("expected signed routing record to be removed when addresses expire") + } } } From 8df072f81c73df1b8a2549321bf33a980ccbacb5 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 11 Nov 2019 22:47:14 -0500 Subject: [PATCH 08/39] add test for setting all TTLs to -1 --- test/addr_book_suite.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index 3a24c1c..86e4ab5 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -173,6 +173,12 @@ func testSetNegativeTTLClears(m pstore.AddrBook) func(t *testing.T) { survivors = append(survivors[0:74], survivors[75:]...) AssertAddressesEqual(t, survivors, m.Addrs(id)) + + // remove _all_ the addresses + m.SetAddrs(id, survivors, -1) + if len(m.Addrs(id)) != 0 { + t.Error("expected empty address list after clearing all addresses") + } } } @@ -369,6 +375,14 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { // we should get everything back from Addrs AssertAddressesEqual(t, allAddrs, m.Addrs(id)) + // adding a previously certified address via AddAddrs should not result + // in duplicates + m.AddAddrs(id, certifiedAddrs, time.Hour) + AssertAddressesEqual(t, allAddrs, m.Addrs(id)) + + // and they should still be considered certified + AssertAddressesEqual(t, certifiedAddrs, m.CertifiedAddrs(id)) + // we should be able to retrieve the original envelope envelope2 := m.SignedRoutingState(id) if envelope2 == nil || !envelope.Equals(envelope2) { From 6c26bc5f279e7eb91d919f41f3fa56682207a84c Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 11 Nov 2019 22:47:54 -0500 Subject: [PATCH 09/39] dedup addrs between signed & unsigned sets --- pstoreds/addr_book.go | 148 ++++++++++++++++++++++++++++------------ pstoremem/addr_book.go | 141 +++++++++++++++++++++++++++----------- test/addr_book_suite.go | 10 ++- 3 files changed, 214 insertions(+), 85 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 3276e57..87ba39d 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -273,6 +273,8 @@ func (ab *dsAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio ab.setAddrs(p, addrs, ttl, ttlExtend, false) } +// AddCertifiedAddrs adds addresses from a routing.RoutingState record +// contained in the given SignedEnvelope. func (ab *dsAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl time.Duration) error { state, err := routing.RoutingStateFromEnvelope(envelope) if err != nil { @@ -454,20 +456,9 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio defer pr.Unlock() newExp := time.Now().Add(ttl).Unix() - existed := make([]bool, len(addrs)) // keeps track of which addrs we found. - - var entryList *[]*pb.AddrBookRecord_AddrEntry - if signed { - entryList = &pr.SignedAddrs - } else { - entryList = &pr.Addrs - } - -Outer: - for i, incoming := range addrs { - for _, have := range *entryList { + updateExisting := func(entryList []*pb.AddrBookRecord_AddrEntry, incoming ma.Multiaddr) *pb.AddrBookRecord_AddrEntry { + for _, have := range entryList { if incoming.Equal(have.Addr) { - existed[i] = true switch mode { case ttlOverride: have.Ttl = int64(ttl) @@ -483,37 +474,118 @@ Outer: panic("BUG: unimplemented ttl mode") } - // we found the address, and addresses cannot be duplicate, - // so let's move on to the next. - continue Outer + return have } } + return nil } - // add addresses we didn't hold. + removeFromUnsigned := make([]ma.Multiaddr, 0) var added []*pb.AddrBookRecord_AddrEntry - for i, e := range existed { - if e { - continue + + for _, incoming := range addrs { + existingUnsigned := updateExisting(pr.Addrs, incoming) + + // we always check if the incoming addr is already in the + // signed addr list, even if we're not adding it from a + // signed source. if it does exist, we update its TTL and + // make sure it gets removed from the unsigned addr list + existingSigned := updateExisting(pr.SignedAddrs, incoming) + if signed || existingSigned != nil { + removeFromUnsigned = append(removeFromUnsigned, incoming) } - addr := addrs[i] - entry := &pb.AddrBookRecord_AddrEntry{ - Addr: &pb.ProtoAddr{Multiaddr: addr}, - Ttl: int64(ttl), - Expiry: newExp, + + // if we're adding a signed addr that already existed in the + // unsigned set, we move the existing unsigned entry over + // with the updated TTL, but we don't broadcast it as new + if signed && existingSigned == nil && existingUnsigned != nil { + added = append(added, existingUnsigned) + } + + // new addr, add & broadcast + if existingUnsigned == nil && existingSigned == nil { + entry := &pb.AddrBookRecord_AddrEntry{ + Addr: &pb.ProtoAddr{Multiaddr: incoming}, + Ttl: int64(ttl), + Expiry: newExp, + } + added = append(added, entry) + + // note: there's a minor chance that writing the record will fail, in which case we would've broadcast + // the addresses without persisting them. This is very unlikely and not much of an issue. + ab.subsManager.BroadcastAddr(p, incoming) } - added = append(added, entry) - // note: there's a minor chance that writing the record will fail, in which case we would've broadcast - // the addresses without persisting them. This is very unlikely and not much of an issue. - ab.subsManager.BroadcastAddr(p, addr) } - *entryList = append(*entryList, added...) + if signed { + pr.SignedAddrs = append(pr.SignedAddrs, added...) + } else { + pr.Addrs = append(pr.Addrs, added...) + } + + pr.Addrs = deleteInPlace(pr.Addrs, removeFromUnsigned) + pr.dirty = true pr.clean() return pr.flush(ab.ds) } +// deduplicateAddrs makes sure an address is only in the signed or unsigned addr list. +// if in both, keeps the signed addr & sets the TTL to whichever is greater +// Note! the caller MUST lock the addrsRecord before calling this +// TODO: refactor setAddrs & do this inline to avoid this extra nested loop +func deduplicateAddrs(pr *addrsRecord) { + if len(pr.SignedAddrs) == 0 { + return + } + + keepers := make([]*pb.AddrBookRecord_AddrEntry, 0, len(pr.Addrs)) + for _, unsigned := range pr.Addrs { + keepUnsigned := true + for _, signed := range pr.SignedAddrs { + if signed.Addr.Equal(unsigned.Addr) { + if unsigned.Expiry > signed.Expiry { + signed.Expiry = unsigned.Expiry + } + if unsigned.Ttl > signed.Ttl { + signed.Ttl = unsigned.Ttl + } + keepUnsigned = false + } + } + if keepUnsigned { + keepers = append(keepers, unsigned) + } + } + + if len(keepers) != len(pr.Addrs) { + pr.Addrs = keepers + pr.dirty = true + } +} + +// deletes addresses in place, avoiding copies until we encounter the first deletion. +// does not preserve order, but entries are re-sorted before flushing to disk anyway. +func deleteInPlace(s []*pb.AddrBookRecord_AddrEntry, addrs []ma.Multiaddr) []*pb.AddrBookRecord_AddrEntry { + if s == nil || len(addrs) == 0 { + return s + } + survived := len(s) + for i, addr := range s { + for _, del := range addrs { + if !addr.Addr.Equal(del) { + continue + } + survived-- + s[i] = s[survived] + } + } + if survived < 0 { + survived = 0 + } + return s[:survived] +} + func (ab *dsAddrBook) deleteAddrs(p peer.ID, addrs []ma.Multiaddr) (err error) { pr, err := ab.loadRecord(p, false, false) if err != nil { @@ -527,20 +599,8 @@ func (ab *dsAddrBook) deleteAddrs(p peer.ID, addrs []ma.Multiaddr) (err error) { pr.Lock() defer pr.Unlock() - // deletes addresses in place, and avoiding copies until we encounter the first deletion. - survived := 0 - for i, addr := range pr.Addrs { - for _, del := range addrs { - if addr.Addr.Equal(del) { - continue - } - if i != survived { - pr.Addrs[survived] = pr.Addrs[i] - } - survived++ - } - } - pr.Addrs = pr.Addrs[:survived] + pr.Addrs = deleteInPlace(pr.Addrs, addrs) + pr.SignedAddrs = deleteInPlace(pr.SignedAddrs, addrs) pr.dirty = true pr.clean() diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 4c5ce69..c9b1d44 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -19,10 +19,9 @@ import ( var log = logging.Logger("peerstore") type expiringAddr struct { - Addr ma.Multiaddr - TTL time.Duration - Expires time.Time - Certified bool + Addr ma.Multiaddr + TTL time.Duration + Expires time.Time } func (e *expiringAddr) ExpiredBy(t time.Time) bool { @@ -40,6 +39,17 @@ type addrSegment struct { addrs map[peer.ID]map[string]*expiringAddr } +// returns the address map for the given peer, creating it if it doesn't exist. +// Note: the caller MUST hold the lock before calling! +func (s *addrSegment) addrMap(p peer.ID) map[string]*expiringAddr { + m, found := s.addrs[p] + if !found { + m = make(map[string]*expiringAddr) + s.addrs[p] = m + } + return m +} + func (s *addrSegment) validAddrs(p peer.ID) []ma.Multiaddr { s.RLock() defer s.RUnlock() @@ -105,8 +115,10 @@ func (segments *addrSegments) peersWithAddrs() peer.IDSlice { var pids peer.IDSlice for _, s := range segments { s.RLock() - for pid, _ := range s.addrs { - pids = append(pids, pid) + for pid, amap := range s.addrs { + if amap != nil && len(amap) > 0 { + pids = append(pids, pid) + } } s.RUnlock() } @@ -190,6 +202,7 @@ func (mab *memoryAddrBook) gc() { mab.segments.gc() mab.signedSegments.gc() + // remove routing records for peers whose signed addrs have all been removed for _, p := range mab.signedSegments.peersWithAddrs() { if len(mab.CertifiedAddrs(p)) == 0 { delete(mab.signedRoutingStates, p) @@ -198,7 +211,19 @@ func (mab *memoryAddrBook) gc() { } func (mab *memoryAddrBook) PeersWithAddrs() peer.IDSlice { - return append(mab.signedSegments.peersWithAddrs(), mab.segments.peersWithAddrs()...) + // deduplicate, since the same peer could have both signed & unsigned addrs + s := peer.NewSet() + out := mab.signedSegments.peersWithAddrs() + for _, p := range out { + s.Add(p) + } + for _, p := range mab.segments.peersWithAddrs() { + if !s.Contains(p) { + out = append(out, p) + } + } + + return out } // AddAddr calls AddAddrs(p, []ma.Multiaddr{addr}, ttl) @@ -213,6 +238,8 @@ func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du mab.addAddrs(p, addrs, ttl, false) } +// AddCertifiedAddrs adds addresses from a routing.RoutingState record +// contained in the given SignedEnvelope. func (mab *memoryAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl time.Duration) error { state, err := routing.RoutingStateFromEnvelope(envelope) if err != nil { @@ -236,41 +263,57 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du return } - var s *addrSegment - if signed { - s = mab.signedSegments.get(p) - } else { - s = mab.segments.get(p) - } + s := mab.segments.get(p) s.Lock() defer s.Unlock() + amap := s.addrMap(p) + + s = mab.signedSegments.get(p) + s.Lock() + defer s.Unlock() + signedAddrMap := s.addrMap(p) - amap := s.addrs[p] - if amap == nil { - amap = make(map[string]*expiringAddr, len(addrs)) - s.addrs[p] = amap - } exp := time.Now().Add(ttl) + maxTTLAndExp := func(a *expiringAddr, t time.Duration, e time.Time) (time.Duration, time.Time) { + if a == nil { + return t, e + } + if a.TTL > ttl { + t = a.TTL + } + if a.Expires.After(exp) { + e = a.Expires + } + return t, e + } for _, addr := range addrs { if addr == nil { log.Warningf("was passed nil multiaddr for %s", p) continue } - asBytes := addr.Bytes() - a, found := amap[string(asBytes)] // won't allocate. - if !found { - // not found, save and announce it. - amap[string(asBytes)] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl} - mab.subManager.BroadcastAddr(p, addr) + k := string(addr.Bytes()) + + // find the highest TTL and Expiry time between + // existing records and function args + a, found := amap[k] // won't allocate. + b, foundSigned := signedAddrMap[k] + maxTTL, maxExp := maxTTLAndExp(a, ttl, exp) + maxTTL, maxExp = maxTTLAndExp(b, maxTTL, maxExp) + + entry := &expiringAddr{Addr: addr, Expires: maxExp, TTL: maxTTL} + + // if we're adding a signed addr, or if it was previously signed, + // make sure it's not also in the unsigned addr list. + if signed || foundSigned { + signedAddrMap[k] = entry + delete(amap, k) } else { - // Update expiration/TTL independently. - // We never want to reduce either. - if ttl > a.TTL { - a.TTL = ttl - } - if exp.After(a.Expires) { - a.Expires = exp - } + amap[k] = entry + } + + if !found && !foundSigned { + // not found, announce it. + mab.subManager.BroadcastAddr(p, addr) } } } @@ -283,15 +326,19 @@ func (mab *memoryAddrBook) SetAddr(p peer.ID, addr ma.Multiaddr, ttl time.Durati // SetAddrs sets the ttl on addresses. This clears any TTL there previously. // This is used when we receive the best estimate of the validity of an address. func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) { + mab.setAddrs(p, addrs, ttl) +} + +func (mab *memoryAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) { s := mab.segments.get(p) s.Lock() defer s.Unlock() + signedSegment := mab.signedSegments.get(p) + signedSegment.Lock() + defer signedSegment.Unlock() - amap := s.addrs[p] - if amap == nil { - amap = make(map[string]*expiringAddr, len(addrs)) - s.addrs[p] = amap - } + amap := s.addrMap(p) + signedAddrMap := signedSegment.addrMap(p) exp := time.Now().Add(ttl) for _, addr := range addrs { @@ -299,16 +346,30 @@ func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du log.Warningf("was passed nil multiaddr for %s", p) continue } + aBytes := addr.Bytes() + key := string(aBytes) + + // if the addr is in the signed addr set, update the ttl + // and ensure it's not in the unsigned set. otherwise, + // add to or update the unsigned set + mapToUpdate := amap + if _, signedExists := signedAddrMap[key]; signedExists { + mapToUpdate = signedAddrMap + delete(amap, key) + } // re-set all of them for new ttl. - aBytes := addr.Bytes() if ttl > 0 { - amap[string(aBytes)] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl} + mapToUpdate[key] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl} mab.subManager.BroadcastAddr(p, addr) } else { - delete(amap, string(aBytes)) + delete(mapToUpdate, key) } } + + if len(signedAddrMap) == 0 { + delete(mab.signedRoutingStates, p) + } } // UpdateAddrs updates the addresses associated with the given peer that have diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index 86e4ab5..eb2e541 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -375,6 +375,11 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { // we should get everything back from Addrs AssertAddressesEqual(t, allAddrs, m.Addrs(id)) + // PeersWithAddrs should return a single peer + if len(m.PeersWithAddrs()) != 1 { + t.Errorf("expected PeersWithAddrs to return 1, got %d", len(m.PeersWithAddrs())) + } + // adding a previously certified address via AddAddrs should not result // in duplicates m.AddAddrs(id, certifiedAddrs, time.Hour) @@ -391,7 +396,10 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { // update TTL on signed addrs to -1 to remove them. // the signed routing record should be deleted - m.UpdateAddrs(id, time.Hour, -1) + m.SetAddrs(id, certifiedAddrs, -1) + if len(m.CertifiedAddrs(id)) != 0 { + t.Error("expected zero certified addrs after setting TTL to -1") + } if m.SignedRoutingState(id) != nil { t.Error("expected signed routing record to be removed when addresses expire") } From 9d4e39c7a063429aaf8082e9f5294e769e8ac865 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Wed, 13 Nov 2019 12:15:46 -0500 Subject: [PATCH 10/39] better bounds checking when deleting addrs --- pstoreds/addr_book.go | 10 +++++++--- test/addr_book_suite.go | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 87ba39d..bc91346 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -571,6 +571,7 @@ func deleteInPlace(s []*pb.AddrBookRecord_AddrEntry, addrs []ma.Multiaddr) []*pb return s } survived := len(s) +Outer: for i, addr := range s { for _, del := range addrs { if !addr.Addr.Equal(del) { @@ -578,11 +579,14 @@ func deleteInPlace(s []*pb.AddrBookRecord_AddrEntry, addrs []ma.Multiaddr) []*pb } survived-- s[i] = s[survived] + // if there are no survivors, bail out + if survived == 0 { + break Outer + } + // we've already dealt with s[i], move to the next + continue Outer } } - if survived < 0 { - survived = 0 - } return s[:survived] } diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index eb2e541..dffd187 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -5,6 +5,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/routing" "github.com/libp2p/go-libp2p-core/test" + "github.com/multiformats/go-multiaddr" "testing" "time" @@ -179,6 +180,24 @@ func testSetNegativeTTLClears(m pstore.AddrBook) func(t *testing.T) { if len(m.Addrs(id)) != 0 { t.Error("expected empty address list after clearing all addresses") } + + // add half, but try to remove more than we added + m.SetAddrs(id, addrs[:50], time.Hour) + m.SetAddrs(id, addrs, -1) + if len(m.Addrs(id)) != 0 { + t.Error("expected empty address list after clearing all addresses") + } + + // try to remove the same addr multiple times + m.SetAddrs(id, addrs[:5], time.Hour) + repeated := make([]multiaddr.Multiaddr, 10) + for i := 0; i < len(repeated); i++ { + repeated[i] = addrs[0] + } + m.SetAddrs(id, repeated, -1) + if len(m.Addrs(id)) != 4 { + t.Errorf("expected 4 addrs after removing one, got %d", len(m.Addrs(id))) + } } } From c44655899d004f270e1123ac4605a657c8f1814f Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Wed, 13 Nov 2019 12:21:27 -0500 Subject: [PATCH 11/39] slight optimization --- pstoreds/addr_book.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index bc91346..d998596 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -578,11 +578,11 @@ Outer: continue } survived-- - s[i] = s[survived] // if there are no survivors, bail out if survived == 0 { break Outer } + s[i] = s[survived] // we've already dealt with s[i], move to the next continue Outer } From c821f29f997781ce8d48867ad3f9ef820fa3b9d4 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 19 Nov 2019 09:12:51 -0600 Subject: [PATCH 12/39] accept & return serialized envelopes --- pstoreds/addr_book.go | 44 ++++++++++++++++++++--------------------- pstoremem/addr_book.go | 25 +++++++++++++---------- test/addr_book_suite.go | 8 +++++--- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index d998596..6cdcf09 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -3,7 +3,6 @@ package pstoreds import ( "context" "fmt" - "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/routing" "sort" "sync" @@ -275,8 +274,8 @@ func (ab *dsAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio // AddCertifiedAddrs adds addresses from a routing.RoutingState record // contained in the given SignedEnvelope. -func (ab *dsAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl time.Duration) error { - state, err := routing.RoutingStateFromEnvelope(envelope) +func (ab *dsAddrBook) AddCertifiedAddrs(envelopeBytes []byte, ttl time.Duration) error { + state, err := routing.RoutingStateFromEnvelope(envelopeBytes) if err != nil { return err } @@ -293,7 +292,7 @@ func (ab *dsAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl tim return err } - return ab.storeRoutingState(state.PeerID, state.Seq, envelope) + return ab.storeRoutingState(state.PeerID, state.Seq, envelopeBytes) } func (ab *dsAddrBook) latestRoutingStateSeq(p peer.ID) uint64 { @@ -304,15 +303,11 @@ func (ab *dsAddrBook) latestRoutingStateSeq(p peer.ID) uint64 { return pr.CertifiedRecord.Seq } -func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelope *crypto.SignedEnvelope) error { +func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelopeBytes []byte) error { // reload record and add routing state // this has to be done after we add the addresses, since if // we try to flush a datastore record with no addresses, // it will just get deleted - envelopeBytes, err := envelope.Marshal() - if err != nil { - return err - } pr, err := ab.loadRecord(p, true, false) if err != nil { return err @@ -326,21 +321,24 @@ func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelope *crypto. return err } -func (ab *dsAddrBook) SignedRoutingState(p peer.ID) *crypto.SignedEnvelope { - pr, err := ab.loadRecord(p, true, false) - if err != nil { - log.Errorf("unable to load record for peer %s: %v", p.Pretty(), err) - return nil - } - if pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { - return nil - } - envelope, err := crypto.UnmarshalEnvelope(pr.CertifiedRecord.Raw) - if err != nil { - log.Errorf("unable to unmarshal stored signed routing record for peer %s: %v", p.Pretty(), err) - return nil +func (ab *dsAddrBook) SignedRoutingState(p peer.ID)[]byte { + return ab.SignedRoutingStates(p)[p] +} + +func (ab *dsAddrBook) SignedRoutingStates(peers ...peer.ID) map[peer.ID][]byte { + out := make(map[peer.ID][]byte, len(peers)) + for _, p := range peers { + pr, err := ab.loadRecord(p, true, false) + if err != nil { + log.Errorf("unable to load record for peer %s: %v", p.Pretty(), err) + continue + } + if pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { + continue + } + out[p] = pr.CertifiedRecord.Raw } - return envelope + return out } // SetAddr will add or update the TTL of an address in the AddrBook. diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index c9b1d44..8a3bb0c 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -7,7 +7,6 @@ import ( "time" logging "github.com/ipfs/go-log" - "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/routing" ma "github.com/multiformats/go-multiaddr" @@ -135,7 +134,7 @@ func (segments *addrSegments) clear(p peer.ID) { type certifiedRecord struct { Seq uint64 - Envelope *crypto.SignedEnvelope + EnvelopeBytes []byte } // memoryAddrBook manages addresses. @@ -240,8 +239,8 @@ func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // AddCertifiedAddrs adds addresses from a routing.RoutingState record // contained in the given SignedEnvelope. -func (mab *memoryAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, ttl time.Duration) error { - state, err := routing.RoutingStateFromEnvelope(envelope) +func (mab *memoryAddrBook) AddCertifiedAddrs(envelopeBytes []byte, ttl time.Duration) error { + state, err := routing.RoutingStateFromEnvelope(envelopeBytes) if err != nil { return err } @@ -252,7 +251,7 @@ func (mab *memoryAddrBook) AddCertifiedAddrs(envelope *crypto.SignedEnvelope, tt // TODO: should this be an error? return nil } - mab.signedRoutingStates[state.PeerID] = &certifiedRecord{Seq: state.Seq, Envelope: envelope} + mab.signedRoutingStates[state.PeerID] = &certifiedRecord{Seq: state.Seq, EnvelopeBytes: envelopeBytes} mab.addAddrs(state.PeerID, state.Multiaddrs(), ttl, true) return nil } @@ -398,12 +397,18 @@ func (mab *memoryAddrBook) CertifiedAddrs(p peer.ID) []ma.Multiaddr { return mab.signedSegments.get(p).validAddrs(p) } -func (mab *memoryAddrBook) SignedRoutingState(p peer.ID) *crypto.SignedEnvelope { - r := mab.signedRoutingStates[p] - if r != nil { - return r.Envelope +func (mab *memoryAddrBook) SignedRoutingState(p peer.ID) []byte { + return mab.SignedRoutingStates(p)[p] +} + +func (mab *memoryAddrBook) SignedRoutingStates(peers ...peer.ID) map[peer.ID][]byte { + out := make(map[peer.ID][]byte, len(peers)) + for _, p := range peers { + if r, ok := mab.signedRoutingStates[p]; ok { + out[p] = r.EnvelopeBytes + } } - return nil + return out } // ClearAddrs removes all previously stored addresses diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index dffd187..944bf08 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -378,9 +378,10 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { if err != nil { t.Errorf("error creating signed routing record: %v", err) } + envelopeBytes, err := envelope.Marshal() // add the signed record to addr book - err = m.AddCertifiedAddrs(envelope, time.Hour) + err = m.AddCertifiedAddrs(envelopeBytes, time.Hour) if err != nil { t.Errorf("error adding signed routing record to addrbook: %v", err) } @@ -408,8 +409,9 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { AssertAddressesEqual(t, certifiedAddrs, m.CertifiedAddrs(id)) // we should be able to retrieve the original envelope - envelope2 := m.SignedRoutingState(id) - if envelope2 == nil || !envelope.Equals(envelope2) { + envelopeBytes2 := m.SignedRoutingState(id) + envelope2, err := crypto.OpenEnvelope(envelopeBytes2, routing.StateEnvelopeDomain) + if envelope2 == nil || !envelope.Equal(envelope2) { t.Error("unable to retrieve signed routing record from addrbook") } From 924831e79e14d39a20dc8d3c41d876705cdf01a3 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 19 Nov 2019 09:34:11 -0600 Subject: [PATCH 13/39] import PR branch of go-libp2p-core --- go.mod | 3 ++- go.sum | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6979c9f..adf3932 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,8 @@ require ( github.com/ipfs/go-ds-leveldb v0.1.0 github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-buffer-pool v0.0.2 - github.com/libp2p/go-libp2p-core v0.2.4 + github.com/libp2p/go-libp2p-core v0.2.6-0.20191119151917-d86de054a1fd + github.com/libp2p/go-msgio v0.0.4 // indirect github.com/multiformats/go-base32 v0.0.3 github.com/multiformats/go-multiaddr v0.1.1 github.com/multiformats/go-multiaddr-fmt v0.1.0 diff --git a/go.sum b/go.sum index 57c3c5a..59784ff 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,7 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1 h1:F4k0TkTAZGLFzBOrVKDAvch6JZtuN4NHkfdcEZL50aI= @@ -95,8 +96,13 @@ github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= +github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-libp2p-core v0.2.4 h1:Et6ykkTwI6PU44tr8qUF9k43vP0aduMNniShAbUJJw8= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= +github.com/libp2p/go-libp2p-core v0.2.6-0.20191118152653-f5e22072f838 h1:ZWYUy+m0PmnsK/2NJdcCnGiJGgTiA0WSdxSJ58YHlJ4= +github.com/libp2p/go-libp2p-core v0.2.6-0.20191118152653-f5e22072f838/go.mod h1:xDyprN8hkMpX27XQ1bBnYtuSuaCUFvKm+Q6gltHZCHE= +github.com/libp2p/go-libp2p-core v0.2.6-0.20191119151917-d86de054a1fd h1:TIcqg9InsHZDnOB83VA+0sMI7cmn4TM4uls0ZrOqugw= +github.com/libp2p/go-libp2p-core v0.2.6-0.20191119151917-d86de054a1fd/go.mod h1:xDyprN8hkMpX27XQ1bBnYtuSuaCUFvKm+Q6gltHZCHE= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-openssl v0.0.3 h1:wjlG7HvQkt4Fq4cfH33Ivpwp0omaElYEi9z26qaIkIk= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= @@ -131,6 +137,7 @@ github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/e github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.1.1 h1:jFFKUuXTXv+3ARyHZi3XUqQO+YWMKgBdhEvuGRfnL6s= github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= +github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1 h1:HHwN1K12I+XllBCrqKnhX949Orn4oawPkegHMu2vDqQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= From 714dc3df960419893bcb824ec760f88f92d677b0 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 19 Nov 2019 09:36:51 -0600 Subject: [PATCH 14/39] go fmt --- pstoreds/addr_book.go | 2 +- pstoremem/addr_book.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 6cdcf09..648f383 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -321,7 +321,7 @@ func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelopeBytes []b return err } -func (ab *dsAddrBook) SignedRoutingState(p peer.ID)[]byte { +func (ab *dsAddrBook) SignedRoutingState(p peer.ID) []byte { return ab.SignedRoutingStates(p)[p] } diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 8a3bb0c..162b553 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -133,7 +133,7 @@ func (segments *addrSegments) clear(p peer.ID) { } type certifiedRecord struct { - Seq uint64 + Seq uint64 EnvelopeBytes []byte } From e5cfa72c470e9ed8187b296094598fed6718628d Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 19 Nov 2019 15:46:24 -0600 Subject: [PATCH 15/39] clear out old signed addrs when adding new routing state --- pstoreds/addr_book.go | 8 ++++++-- pstoremem/addr_book.go | 21 ++++++++++++++++----- test/addr_book_suite.go | 12 ++++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 648f383..565669c 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -478,7 +478,7 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio return nil } - removeFromUnsigned := make([]ma.Multiaddr, 0) + var removeFromUnsigned []ma.Multiaddr var added []*pb.AddrBookRecord_AddrEntry for _, incoming := range addrs { @@ -489,6 +489,9 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio // signed source. if it does exist, we update its TTL and // make sure it gets removed from the unsigned addr list existingSigned := updateExisting(pr.SignedAddrs, incoming) + if existingSigned != nil { + added = append(added, existingSigned) + } if signed || existingSigned != nil { removeFromUnsigned = append(removeFromUnsigned, incoming) } @@ -516,7 +519,8 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio } if signed { - pr.SignedAddrs = append(pr.SignedAddrs, added...) + // when adding signed addrs, we want to keep _only_ the incoming addrs + pr.SignedAddrs = added } else { pr.Addrs = append(pr.Addrs, added...) } diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 162b553..17c228e 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -267,10 +267,17 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du defer s.Unlock() amap := s.addrMap(p) - s = mab.signedSegments.get(p) - s.Lock() - defer s.Unlock() - signedAddrMap := s.addrMap(p) + signedSegment := mab.signedSegments.get(p) + signedSegment.Lock() + defer signedSegment.Unlock() + existingSignedAddrs := signedSegment.addrMap(p) + + // if we're adding signed addresses, we want _only_ the new addrs + // to be in the final map, so we make a new map to contain them + signedAddrMap := existingSignedAddrs + if signed { + signedAddrMap = make(map[string]*expiringAddr, len(addrs)) + } exp := time.Now().Add(ttl) maxTTLAndExp := func(a *expiringAddr, t time.Duration, e time.Time) (time.Duration, time.Time) { @@ -295,7 +302,7 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // find the highest TTL and Expiry time between // existing records and function args a, found := amap[k] // won't allocate. - b, foundSigned := signedAddrMap[k] + b, foundSigned := existingSignedAddrs[k] maxTTL, maxExp := maxTTLAndExp(a, ttl, exp) maxTTL, maxExp = maxTTLAndExp(b, maxTTL, maxExp) @@ -315,6 +322,10 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du mab.subManager.BroadcastAddr(p, addr) } } + + // replace existing signed addrs with new ones + // this is a noop if we're adding unsigned addrs + signedSegment.addrs[p] = signedAddrMap } // SetAddr calls mgr.SetAddrs(p, addr, ttl) diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index 944bf08..05a7261 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -415,6 +415,18 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { t.Error("unable to retrieve signed routing record from addrbook") } + // Adding a new envelope should clear existing certified addresses. + // Only the newly-added ones should remain + certifiedAddrs = certifiedAddrs[:3] + info = peer.AddrInfo{ID: id, Addrs: certifiedAddrs} + envelope, err = routing.RoutingStateFromAddrInfo(&info).ToSignedEnvelope(priv) + test.AssertNilError(t, err) + envelopeBytes, err = envelope.Marshal() + test.AssertNilError(t, err) + err = m.AddCertifiedAddrs(envelopeBytes, time.Hour) + test.AssertNilError(t, err) + AssertAddressesEqual(t, certifiedAddrs, m.CertifiedAddrs(id)) + // update TTL on signed addrs to -1 to remove them. // the signed routing record should be deleted m.SetAddrs(id, certifiedAddrs, -1) From 0c1916af8502e7c21b8666be857d8bee24405a96 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 19 Nov 2019 15:48:04 -0600 Subject: [PATCH 16/39] rm dead code --- pstoreds/addr_book.go | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 565669c..c3cb385 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -532,40 +532,6 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio return pr.flush(ab.ds) } -// deduplicateAddrs makes sure an address is only in the signed or unsigned addr list. -// if in both, keeps the signed addr & sets the TTL to whichever is greater -// Note! the caller MUST lock the addrsRecord before calling this -// TODO: refactor setAddrs & do this inline to avoid this extra nested loop -func deduplicateAddrs(pr *addrsRecord) { - if len(pr.SignedAddrs) == 0 { - return - } - - keepers := make([]*pb.AddrBookRecord_AddrEntry, 0, len(pr.Addrs)) - for _, unsigned := range pr.Addrs { - keepUnsigned := true - for _, signed := range pr.SignedAddrs { - if signed.Addr.Equal(unsigned.Addr) { - if unsigned.Expiry > signed.Expiry { - signed.Expiry = unsigned.Expiry - } - if unsigned.Ttl > signed.Ttl { - signed.Ttl = unsigned.Ttl - } - keepUnsigned = false - } - } - if keepUnsigned { - keepers = append(keepers, unsigned) - } - } - - if len(keepers) != len(pr.Addrs) { - pr.Addrs = keepers - pr.dirty = true - } -} - // deletes addresses in place, avoiding copies until we encounter the first deletion. // does not preserve order, but entries are re-sorted before flushing to disk anyway. func deleteInPlace(s []*pb.AddrBookRecord_AddrEntry, addrs []ma.Multiaddr) []*pb.AddrBookRecord_AddrEntry { From 5b63f9f4d27f0656df5989ac84398d501a6cae26 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 19 Nov 2019 15:49:46 -0600 Subject: [PATCH 17/39] doc comments --- pstoreds/addr_book.go | 8 ++++++++ pstoremem/addr_book.go | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index c3cb385..c5a3764 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -321,10 +321,18 @@ func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelopeBytes []b return err } +// SignedRoutingState returns a signed RoutingState record for the +// given peer id, if one exists in the peerstore. The record is +// returned as a byte slice containing a serialized SignedEnvelope. +// Returns nil if no routing state exists for the peer. func (ab *dsAddrBook) SignedRoutingState(p peer.ID) []byte { return ab.SignedRoutingStates(p)[p] } +// SignedRoutingStates returns signed RoutingState records for each of +// the given peer ids, if one exists in the peerstore. +// Returns a map of peer ids to serialized SignedEnvelope messages. If +// no routing state exists for a peer, their map entry will be nil. func (ab *dsAddrBook) SignedRoutingStates(peers ...peer.ID) map[peer.ID][]byte { out := make(map[peer.ID][]byte, len(peers)) for _, p := range peers { diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 17c228e..50331c9 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -408,10 +408,18 @@ func (mab *memoryAddrBook) CertifiedAddrs(p peer.ID) []ma.Multiaddr { return mab.signedSegments.get(p).validAddrs(p) } +// SignedRoutingState returns a signed RoutingState record for the +// given peer id, if one exists in the peerstore. The record is +// returned as a byte slice containing a serialized SignedEnvelope. +// Returns nil if no routing state exists for the peer. func (mab *memoryAddrBook) SignedRoutingState(p peer.ID) []byte { return mab.SignedRoutingStates(p)[p] } +// SignedRoutingStates returns signed RoutingState records for each of +// the given peer ids, if one exists in the peerstore. +// Returns a map of peer ids to serialized SignedEnvelope messages. If +// no routing state exists for a peer, their map entry will be nil. func (mab *memoryAddrBook) SignedRoutingStates(peers ...peer.ID) map[peer.ID][]byte { out := make(map[peer.ID][]byte, len(peers)) for _, p := range peers { From f36813497f5b090e53af94a4d495eeee3b0b6bd9 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 21 Nov 2019 12:00:39 -0600 Subject: [PATCH 18/39] track API changes in -core --- pstoreds/addr_book.go | 60 +++++++++++++++++------------------------ pstoremem/addr_book.go | 44 +++++++----------------------- test/addr_book_suite.go | 20 +++++--------- 3 files changed, 41 insertions(+), 83 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index c5a3764..ff61b9e 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -274,25 +274,20 @@ func (ab *dsAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio // AddCertifiedAddrs adds addresses from a routing.RoutingState record // contained in the given SignedEnvelope. -func (ab *dsAddrBook) AddCertifiedAddrs(envelopeBytes []byte, ttl time.Duration) error { - state, err := routing.RoutingStateFromEnvelope(envelopeBytes) - if err != nil { - return err - } - +func (ab *dsAddrBook) AddCertifiedAddrs(state *routing.SignedRoutingState, ttl time.Duration) error { // ensure that the seq number from envelope is >= any previously received seq no if ab.latestRoutingStateSeq(state.PeerID) >= state.Seq { // TODO: should this be an error? return nil } - addrs := cleanAddrs(state.Multiaddrs()) - err = ab.setAddrs(state.PeerID, addrs, ttl, ttlExtend, true) + addrs := cleanAddrs(state.Addrs) + err := ab.setAddrs(state.PeerID, addrs, ttl, ttlExtend, true) if err != nil { return err } - return ab.storeRoutingState(state.PeerID, state.Seq, envelopeBytes) + return ab.storeRoutingState(state.PeerID, state) } func (ab *dsAddrBook) latestRoutingStateSeq(p peer.ID) uint64 { @@ -303,7 +298,11 @@ func (ab *dsAddrBook) latestRoutingStateSeq(p peer.ID) uint64 { return pr.CertifiedRecord.Seq } -func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelopeBytes []byte) error { +func (ab *dsAddrBook) storeRoutingState(p peer.ID, state *routing.SignedRoutingState) error { + envelopeBytes, err := state.Marshal() + if err != nil { + return err + } // reload record and add routing state // this has to be done after we add the addresses, since if // we try to flush a datastore record with no addresses, @@ -313,7 +312,7 @@ func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelopeBytes []b return err } pr.CertifiedRecord = &pb.AddrBookRecord_CertifiedRecord{ - Seq: seq, + Seq: state.Seq, Raw: envelopeBytes, } pr.dirty = true @@ -322,31 +321,22 @@ func (ab *dsAddrBook) storeRoutingState(p peer.ID, seq uint64, envelopeBytes []b } // SignedRoutingState returns a signed RoutingState record for the -// given peer id, if one exists in the peerstore. The record is -// returned as a byte slice containing a serialized SignedEnvelope. -// Returns nil if no routing state exists for the peer. -func (ab *dsAddrBook) SignedRoutingState(p peer.ID) []byte { - return ab.SignedRoutingStates(p)[p] -} - -// SignedRoutingStates returns signed RoutingState records for each of -// the given peer ids, if one exists in the peerstore. -// Returns a map of peer ids to serialized SignedEnvelope messages. If -// no routing state exists for a peer, their map entry will be nil. -func (ab *dsAddrBook) SignedRoutingStates(peers ...peer.ID) map[peer.ID][]byte { - out := make(map[peer.ID][]byte, len(peers)) - for _, p := range peers { - pr, err := ab.loadRecord(p, true, false) - if err != nil { - log.Errorf("unable to load record for peer %s: %v", p.Pretty(), err) - continue - } - if pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { - continue - } - out[p] = pr.CertifiedRecord.Raw +// given peer id, if one exists in the peerstore. +func (ab *dsAddrBook) SignedRoutingState(p peer.ID) *routing.SignedRoutingState { + pr, err := ab.loadRecord(p, true, false) + if err != nil { + log.Errorf("unable to load record for peer %s: %v", p.Pretty(), err) + return nil + } + if pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { + return nil + } + state, err := routing.UnmarshalSignedRoutingState(pr.CertifiedRecord.Raw) + if err != nil { + log.Errorf("error unmarshaling stored SignedRoutingState record for peer %s: %v", p.Pretty(), err) + return nil } - return out + return state } // SetAddr will add or update the TTL of an address in the AddrBook. diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 50331c9..f3698fa 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -132,17 +132,12 @@ func (segments *addrSegments) clear(p peer.ID) { delete(s.addrs, p) } -type certifiedRecord struct { - Seq uint64 - EnvelopeBytes []byte -} - // memoryAddrBook manages addresses. type memoryAddrBook struct { segments addrSegments signedSegments addrSegments - signedRoutingStates map[peer.ID]*certifiedRecord + signedRoutingStates map[peer.ID]*routing.SignedRoutingState ctx context.Context cancel func() @@ -168,7 +163,7 @@ func NewAddrBook() pstore.AddrBook { subManager: NewAddrSubManager(), ctx: ctx, cancel: cancel, - signedRoutingStates: make(map[peer.ID]*certifiedRecord), + signedRoutingStates: make(map[peer.ID]*routing.SignedRoutingState), } go ab.background() @@ -239,20 +234,15 @@ func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // AddCertifiedAddrs adds addresses from a routing.RoutingState record // contained in the given SignedEnvelope. -func (mab *memoryAddrBook) AddCertifiedAddrs(envelopeBytes []byte, ttl time.Duration) error { - state, err := routing.RoutingStateFromEnvelope(envelopeBytes) - if err != nil { - return err - } - +func (mab *memoryAddrBook) AddCertifiedAddrs(state *routing.SignedRoutingState, ttl time.Duration) error { // ensure seq is greater than last received lastState, found := mab.signedRoutingStates[state.PeerID] if found && lastState.Seq >= state.Seq { // TODO: should this be an error? return nil } - mab.signedRoutingStates[state.PeerID] = &certifiedRecord{Seq: state.Seq, EnvelopeBytes: envelopeBytes} - mab.addAddrs(state.PeerID, state.Multiaddrs(), ttl, true) + mab.signedRoutingStates[state.PeerID] = state + mab.addAddrs(state.PeerID, state.Addrs, ttl, true) return nil } @@ -408,26 +398,10 @@ func (mab *memoryAddrBook) CertifiedAddrs(p peer.ID) []ma.Multiaddr { return mab.signedSegments.get(p).validAddrs(p) } -// SignedRoutingState returns a signed RoutingState record for the -// given peer id, if one exists in the peerstore. The record is -// returned as a byte slice containing a serialized SignedEnvelope. -// Returns nil if no routing state exists for the peer. -func (mab *memoryAddrBook) SignedRoutingState(p peer.ID) []byte { - return mab.SignedRoutingStates(p)[p] -} - -// SignedRoutingStates returns signed RoutingState records for each of -// the given peer ids, if one exists in the peerstore. -// Returns a map of peer ids to serialized SignedEnvelope messages. If -// no routing state exists for a peer, their map entry will be nil. -func (mab *memoryAddrBook) SignedRoutingStates(peers ...peer.ID) map[peer.ID][]byte { - out := make(map[peer.ID][]byte, len(peers)) - for _, p := range peers { - if r, ok := mab.signedRoutingStates[p]; ok { - out[p] = r.EnvelopeBytes - } - } - return out +// SignedRoutingState returns a SignedRoutingState record for the +// given peer id, if one exists in the peerstore. +func (mab *memoryAddrBook) SignedRoutingState(p peer.ID) *routing.SignedRoutingState { + return mab.signedRoutingStates[p] } // ClearAddrs removes all previously stored addresses diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index 05a7261..0cfe8d8 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -373,15 +373,13 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { allAddrs := GenerateAddrs(10) certifiedAddrs := allAddrs[:5] uncertifiedAddrs := allAddrs[5:] - info := peer.AddrInfo{ID: id, Addrs: certifiedAddrs} - envelope, err := routing.RoutingStateFromAddrInfo(&info).ToSignedEnvelope(priv) + signedState, err := routing.MakeSignedRoutingState(priv, certifiedAddrs) if err != nil { t.Errorf("error creating signed routing record: %v", err) } - envelopeBytes, err := envelope.Marshal() // add the signed record to addr book - err = m.AddCertifiedAddrs(envelopeBytes, time.Hour) + err = m.AddCertifiedAddrs(signedState, time.Hour) if err != nil { t.Errorf("error adding signed routing record to addrbook: %v", err) } @@ -408,22 +406,18 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { // and they should still be considered certified AssertAddressesEqual(t, certifiedAddrs, m.CertifiedAddrs(id)) - // we should be able to retrieve the original envelope - envelopeBytes2 := m.SignedRoutingState(id) - envelope2, err := crypto.OpenEnvelope(envelopeBytes2, routing.StateEnvelopeDomain) - if envelope2 == nil || !envelope.Equal(envelope2) { + // we should be able to retrieve the signed state + state2 := m.SignedRoutingState(id) + if state2 == nil || !signedState.Equal(state2) { t.Error("unable to retrieve signed routing record from addrbook") } // Adding a new envelope should clear existing certified addresses. // Only the newly-added ones should remain certifiedAddrs = certifiedAddrs[:3] - info = peer.AddrInfo{ID: id, Addrs: certifiedAddrs} - envelope, err = routing.RoutingStateFromAddrInfo(&info).ToSignedEnvelope(priv) + signedState, err = routing.MakeSignedRoutingState(priv, certifiedAddrs) test.AssertNilError(t, err) - envelopeBytes, err = envelope.Marshal() - test.AssertNilError(t, err) - err = m.AddCertifiedAddrs(envelopeBytes, time.Hour) + err = m.AddCertifiedAddrs(signedState, time.Hour) test.AssertNilError(t, err) AssertAddressesEqual(t, certifiedAddrs, m.CertifiedAddrs(id)) From 50b0a8d4d5d153e8ead67b38aa5f8999b7072bb7 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 21 Nov 2019 12:01:45 -0600 Subject: [PATCH 19/39] bump -core dep to new PR commit --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index adf3932..949a746 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ipfs/go-ds-leveldb v0.1.0 github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-buffer-pool v0.0.2 - github.com/libp2p/go-libp2p-core v0.2.6-0.20191119151917-d86de054a1fd + github.com/libp2p/go-libp2p-core v0.2.6-0.20191121175514-5fa975301271 github.com/libp2p/go-msgio v0.0.4 // indirect github.com/multiformats/go-base32 v0.0.3 github.com/multiformats/go-multiaddr v0.1.1 diff --git a/go.sum b/go.sum index 59784ff..271ef15 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,8 @@ github.com/libp2p/go-libp2p-core v0.2.6-0.20191118152653-f5e22072f838 h1:ZWYUy+m github.com/libp2p/go-libp2p-core v0.2.6-0.20191118152653-f5e22072f838/go.mod h1:xDyprN8hkMpX27XQ1bBnYtuSuaCUFvKm+Q6gltHZCHE= github.com/libp2p/go-libp2p-core v0.2.6-0.20191119151917-d86de054a1fd h1:TIcqg9InsHZDnOB83VA+0sMI7cmn4TM4uls0ZrOqugw= github.com/libp2p/go-libp2p-core v0.2.6-0.20191119151917-d86de054a1fd/go.mod h1:xDyprN8hkMpX27XQ1bBnYtuSuaCUFvKm+Q6gltHZCHE= +github.com/libp2p/go-libp2p-core v0.2.6-0.20191121175514-5fa975301271 h1:4zW1wrEii7L5t+e9AF8zyLSc+mibnmfBcfMWA6ItyXk= +github.com/libp2p/go-libp2p-core v0.2.6-0.20191121175514-5fa975301271/go.mod h1:xDyprN8hkMpX27XQ1bBnYtuSuaCUFvKm+Q6gltHZCHE= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-openssl v0.0.3 h1:wjlG7HvQkt4Fq4cfH33Ivpwp0omaElYEi9z26qaIkIk= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= From ebf8ecbd2677f7b571fcc3657f7b84717138ee40 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 22 Nov 2019 11:43:05 -0600 Subject: [PATCH 20/39] put signed and unsigned addrs in same segment --- pstoremem/addr_book.go | 270 +++++++++++++++++++---------------------- 1 file changed, 126 insertions(+), 144 deletions(-) diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index f3698fa..04fbb12 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -36,106 +36,17 @@ type addrSegment struct { // space unused. storing the *values* directly in the map will // drastically increase the space waste. In our case, by 6x. addrs map[peer.ID]map[string]*expiringAddr -} - -// returns the address map for the given peer, creating it if it doesn't exist. -// Note: the caller MUST hold the lock before calling! -func (s *addrSegment) addrMap(p peer.ID) map[string]*expiringAddr { - m, found := s.addrs[p] - if !found { - m = make(map[string]*expiringAddr) - s.addrs[p] = m - } - return m -} - -func (s *addrSegment) validAddrs(p peer.ID) []ma.Multiaddr { - s.RLock() - defer s.RUnlock() - - amap, found := s.addrs[p] - if !found { - return nil - } - - now := time.Now() - good := make([]ma.Multiaddr, 0, len(amap)) - for _, m := range amap { - if !m.ExpiredBy(now) { - good = append(good, m.Addr) - } - } - return good -} - -func (s *addrSegment) updateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.Duration) { - s.Lock() - defer s.Unlock() - - amap, found := s.addrs[p] - if !found { - return - } - - exp := time.Now().Add(newTTL) - for k, addr := range amap { - if oldTTL == addr.TTL { - addr.TTL = newTTL - addr.Expires = exp - amap[k] = addr - } - } + signedAddrs map[peer.ID]map[string]*expiringAddr } func (segments *addrSegments) get(p peer.ID) *addrSegment { return segments[byte(p[len(p)-1])] } -func (segments *addrSegments) gc() { - now := time.Now() - for _, s := range segments { - s.Lock() - for p, amap := range s.addrs { - for k, addr := range amap { - if addr.ExpiredBy(now) { - delete(amap, k) - } - } - if len(amap) == 0 { - delete(s.addrs, p) - } - } - s.Unlock() - } -} - -func (segments *addrSegments) peersWithAddrs() peer.IDSlice { - var pids peer.IDSlice - for _, s := range segments { - s.RLock() - for pid, amap := range s.addrs { - if amap != nil && len(amap) > 0 { - pids = append(pids, pid) - } - } - s.RUnlock() - } - return pids -} - -func (segments *addrSegments) clear(p peer.ID) { - s := segments.get(p) - s.Lock() - defer s.Unlock() - - delete(s.addrs, p) -} - // memoryAddrBook manages addresses. type memoryAddrBook struct { - segments addrSegments - signedSegments addrSegments + segments addrSegments signedRoutingStates map[peer.ID]*routing.SignedRoutingState @@ -150,16 +61,15 @@ var _ pstore.AddrBook = (*memoryAddrBook)(nil) func NewAddrBook() pstore.AddrBook { ctx, cancel := context.WithCancel(context.Background()) - makeSegments := func() (ret addrSegments) { - for i, _ := range ret { - ret[i] = &addrSegment{addrs: make(map[peer.ID]map[string]*expiringAddr)} - } - return ret - } - ab := &memoryAddrBook{ - segments: makeSegments(), - signedSegments: makeSegments(), + segments: func() (ret addrSegments) { + for i, _ := range ret { + ret[i] = &addrSegment{ + addrs: make(map[peer.ID]map[string]*expiringAddr), + signedAddrs: make(map[peer.ID]map[string]*expiringAddr)} + } + return ret + }(), subManager: NewAddrSubManager(), ctx: ctx, cancel: cancel, @@ -193,31 +103,52 @@ func (mab *memoryAddrBook) Close() error { // gc garbage collects the in-memory address book. func (mab *memoryAddrBook) gc() { - mab.segments.gc() - mab.signedSegments.gc() - - // remove routing records for peers whose signed addrs have all been removed - for _, p := range mab.signedSegments.peersWithAddrs() { - if len(mab.CertifiedAddrs(p)) == 0 { + now := time.Now() + collect := func(addrs map[peer.ID]map[string]*expiringAddr) []peer.ID { + var collectedPeers []peer.ID + for p, amap := range addrs { + for k, addr := range amap { + if addr.ExpiredBy(now) { + delete(amap, k) + } + } + if len(amap) == 0 { + delete(addrs, p) + collectedPeers = append(collectedPeers, p) + } + } + return collectedPeers + } + for _, s := range mab.segments { + s.Lock() + collect(s.addrs) + collected := collect(s.signedAddrs) + // remove routing records for peers whose signed addrs have all been removed + for _, p := range collected { delete(mab.signedRoutingStates, p) } + s.Unlock() } } func (mab *memoryAddrBook) PeersWithAddrs() peer.IDSlice { // deduplicate, since the same peer could have both signed & unsigned addrs - s := peer.NewSet() - out := mab.signedSegments.peersWithAddrs() - for _, p := range out { - s.Add(p) - } - for _, p := range mab.segments.peersWithAddrs() { - if !s.Contains(p) { - out = append(out, p) + pidSet := peer.NewSet() + for _, s := range mab.segments { + s.RLock() + for pid, amap := range s.addrs { + if amap != nil && len(amap) > 0 { + pidSet.Add(pid) + } + } + for pid, amap := range s.signedAddrs { + if amap != nil && len(amap) > 0 { + pidSet.Add(pid) + } } + s.RUnlock() } - - return out + return pidSet.Peers() } // AddAddr calls AddAddrs(p, []ma.Multiaddr{addr}, ttl) @@ -255,12 +186,16 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du s := mab.segments.get(p) s.Lock() defer s.Unlock() - amap := s.addrMap(p) - - signedSegment := mab.signedSegments.get(p) - signedSegment.Lock() - defer signedSegment.Unlock() - existingSignedAddrs := signedSegment.addrMap(p) + amap, ok := s.addrs[p] + if !ok { + amap = make(map[string]*expiringAddr) + s.addrs[p] = amap + } + existingSignedAddrs, ok := s.signedAddrs[p] + if !ok { + existingSignedAddrs = make(map[string]*expiringAddr) + s.signedAddrs[p] = existingSignedAddrs + } // if we're adding signed addresses, we want _only_ the new addrs // to be in the final map, so we make a new map to contain them @@ -315,7 +250,7 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // replace existing signed addrs with new ones // this is a noop if we're adding unsigned addrs - signedSegment.addrs[p] = signedAddrMap + s.signedAddrs[p] = signedAddrMap } // SetAddr calls mgr.SetAddrs(p, addr, ttl) @@ -326,19 +261,20 @@ func (mab *memoryAddrBook) SetAddr(p peer.ID, addr ma.Multiaddr, ttl time.Durati // SetAddrs sets the ttl on addresses. This clears any TTL there previously. // This is used when we receive the best estimate of the validity of an address. func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) { - mab.setAddrs(p, addrs, ttl) -} - -func (mab *memoryAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) { s := mab.segments.get(p) s.Lock() defer s.Unlock() - signedSegment := mab.signedSegments.get(p) - signedSegment.Lock() - defer signedSegment.Unlock() - amap := s.addrMap(p) - signedAddrMap := signedSegment.addrMap(p) + amap, ok := s.addrs[p] + if !ok { + amap = make(map[string]*expiringAddr) + s.addrs[p] = amap + } + signedAddrMap, ok := s.signedAddrs[p] + if !ok { + signedAddrMap = make(map[string]*expiringAddr) + s.signedAddrs[p] = signedAddrMap + } exp := time.Now().Add(ttl) for _, addr := range addrs { @@ -367,6 +303,7 @@ func (mab *memoryAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du } } + // if we've expired all the signed addresses for a peer, remove their signed routing state record if len(signedAddrMap) == 0 { delete(mab.signedRoutingStates, p) } @@ -375,27 +312,68 @@ func (mab *memoryAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // UpdateAddrs updates the addresses associated with the given peer that have // the given oldTTL to have the given newTTL. func (mab *memoryAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.Duration) { - mab.segments.get(p).updateAddrs(p, oldTTL, newTTL) - mab.signedSegments.get(p).updateAddrs(p, oldTTL, newTTL) - - // updating may have caused all our certified addrs to expire. - // if so, remove the signed record we sourced them from. - if len(mab.CertifiedAddrs(p)) == 0 { - delete(mab.signedRoutingStates, p) + s := mab.segments.get(p) + s.Lock() + defer s.Unlock() + exp := time.Now().Add(newTTL) + update := func(amap map[string]*expiringAddr) { + for k, a := range amap { + if oldTTL == a.TTL { + a.TTL = newTTL + a.Expires = exp + amap[k] = a + } + } + } + amap, found := s.addrs[p] + if found { + update(amap) + } + signedAddrMap, found := s.signedAddrs[p] + if found { + update(signedAddrMap) + // updating may have caused all our certified addrs to expire. + // if so, remove the signed record we sourced them from. + if len(signedAddrMap) == 0 { + delete(mab.signedRoutingStates, p) + } } } // Addrs returns all known (and valid) addresses for a given peer func (mab *memoryAddrBook) Addrs(p peer.ID) []ma.Multiaddr { + s := mab.segments.get(p) + s.RLock() + defer s.RUnlock() + return append( - mab.signedSegments.get(p).validAddrs(p), - mab.segments.get(p).validAddrs(p)...) + validAddrs(s.signedAddrs[p]), + validAddrs(s.addrs[p])...) } // CertifiedAddrs returns all known (and valid) addressed that have been // certified by the given peer. func (mab *memoryAddrBook) CertifiedAddrs(p peer.ID) []ma.Multiaddr { - return mab.signedSegments.get(p).validAddrs(p) + s := mab.segments.get(p) + s.RLock() + defer s.RUnlock() + + return validAddrs(s.signedAddrs[p]) +} + +func validAddrs(amap map[string]*expiringAddr) []ma.Multiaddr { + now := time.Now() + good := make([]ma.Multiaddr, 0, len(amap)) + if amap == nil { + return good + } + for _, m := range amap { + if !m.ExpiredBy(now) { + good = append(good, m.Addr) + } + } + + return good } // SignedRoutingState returns a SignedRoutingState record for the @@ -406,8 +384,12 @@ func (mab *memoryAddrBook) SignedRoutingState(p peer.ID) *routing.SignedRoutingS // ClearAddrs removes all previously stored addresses func (mab *memoryAddrBook) ClearAddrs(p peer.ID) { - mab.segments.clear(p) - mab.signedSegments.clear(p) + s := mab.segments.get(p) + s.Lock() + defer s.Unlock() + + delete(s.addrs, p) + delete(s.signedAddrs, p) } // AddrStream returns a channel on which all new addresses discovered for a From 0f9538856f8533ff8718a91e1b482c2df1b4d39f Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 22 Nov 2019 12:06:40 -0600 Subject: [PATCH 21/39] move signedRoutingStates inside addrSegment --- pstoremem/addr_book.go | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 04fbb12..6e2106a 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -38,6 +38,8 @@ type addrSegment struct { addrs map[peer.ID]map[string]*expiringAddr signedAddrs map[peer.ID]map[string]*expiringAddr + + signedRoutingStates map[peer.ID]*routing.SignedRoutingState } func (segments *addrSegments) get(p peer.ID) *addrSegment { @@ -48,8 +50,6 @@ func (segments *addrSegments) get(p peer.ID) *addrSegment { type memoryAddrBook struct { segments addrSegments - signedRoutingStates map[peer.ID]*routing.SignedRoutingState - ctx context.Context cancel func() @@ -65,15 +65,15 @@ func NewAddrBook() pstore.AddrBook { segments: func() (ret addrSegments) { for i, _ := range ret { ret[i] = &addrSegment{ - addrs: make(map[peer.ID]map[string]*expiringAddr), - signedAddrs: make(map[peer.ID]map[string]*expiringAddr)} + addrs: make(map[peer.ID]map[string]*expiringAddr), + signedAddrs: make(map[peer.ID]map[string]*expiringAddr), + signedRoutingStates: make(map[peer.ID]*routing.SignedRoutingState)} } return ret }(), - subManager: NewAddrSubManager(), - ctx: ctx, - cancel: cancel, - signedRoutingStates: make(map[peer.ID]*routing.SignedRoutingState), + subManager: NewAddrSubManager(), + ctx: ctx, + cancel: cancel, } go ab.background() @@ -125,7 +125,7 @@ func (mab *memoryAddrBook) gc() { collected := collect(s.signedAddrs) // remove routing records for peers whose signed addrs have all been removed for _, p := range collected { - delete(mab.signedRoutingStates, p) + delete(s.signedRoutingStates, p) } s.Unlock() } @@ -167,12 +167,15 @@ func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // contained in the given SignedEnvelope. func (mab *memoryAddrBook) AddCertifiedAddrs(state *routing.SignedRoutingState, ttl time.Duration) error { // ensure seq is greater than last received - lastState, found := mab.signedRoutingStates[state.PeerID] + s := mab.segments.get(state.PeerID) + s.Lock() + lastState, found := s.signedRoutingStates[state.PeerID] if found && lastState.Seq >= state.Seq { // TODO: should this be an error? return nil } - mab.signedRoutingStates[state.PeerID] = state + s.signedRoutingStates[state.PeerID] = state + s.Unlock() mab.addAddrs(state.PeerID, state.Addrs, ttl, true) return nil } @@ -305,7 +308,7 @@ func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // if we've expired all the signed addresses for a peer, remove their signed routing state record if len(signedAddrMap) == 0 { - delete(mab.signedRoutingStates, p) + delete(s.signedRoutingStates, p) } } @@ -335,7 +338,7 @@ func (mab *memoryAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL t // updating may have caused all our certified addrs to expire. // if so, remove the signed record we sourced them from. if len(signedAddrMap) == 0 { - delete(mab.signedRoutingStates, p) + delete(s.signedRoutingStates, p) } } } @@ -379,7 +382,10 @@ func validAddrs(amap map[string]*expiringAddr) []ma.Multiaddr { // SignedRoutingState returns a SignedRoutingState record for the // given peer id, if one exists in the peerstore. func (mab *memoryAddrBook) SignedRoutingState(p peer.ID) *routing.SignedRoutingState { - return mab.signedRoutingStates[p] + s := mab.segments.get(p) + s.RLock() + defer s.RUnlock() + return s.signedRoutingStates[p] } // ClearAddrs removes all previously stored addresses @@ -390,6 +396,7 @@ func (mab *memoryAddrBook) ClearAddrs(p peer.ID) { delete(s.addrs, p) delete(s.signedAddrs, p) + delete(s.signedRoutingStates, p) } // AddrStream returns a channel on which all new addresses discovered for a From 02e9e9677ec2427404c6993f6423ce5cdc160441 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 13 Dec 2019 10:29:57 -0500 Subject: [PATCH 22/39] adopt CertifiedAddrBook interface - removes CertifiedAddrs method from addr book implementations - asserts AddrBooks are also CertifiedAddrBooks - ensures we either have certified addrs or uncertified, but not both - updates tests for CertifiedAddrs --- pstoreds/addr_book.go | 40 ++++++++++++++-------------------------- pstoremem/addr_book.go | 35 +++++++++++++++++------------------ test/addr_book_suite.go | 40 +++++++++++++++++++--------------------- 3 files changed, 50 insertions(+), 65 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index ff61b9e..2f21c13 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -162,6 +162,7 @@ type dsAddrBook struct { } var _ pstore.AddrBook = (*dsAddrBook)(nil) +var _ pstore.CertifiedAddrBook = (*dsAddrBook)(nil) // NewAddrBook initializes a new datastore-backed address book. It serves as a drop-in replacement for pstoremem // (memory-backed peerstore), and works with any datastore implementing the ds.Batching interface. @@ -277,7 +278,6 @@ func (ab *dsAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio func (ab *dsAddrBook) AddCertifiedAddrs(state *routing.SignedRoutingState, ttl time.Duration) error { // ensure that the seq number from envelope is >= any previously received seq no if ab.latestRoutingStateSeq(state.PeerID) >= state.Seq { - // TODO: should this be an error? return nil } @@ -383,16 +383,6 @@ func (ab *dsAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.D // Addrs returns all of the non-expired addresses for a given peer. func (ab *dsAddrBook) Addrs(p peer.ID) []ma.Multiaddr { - return ab.addrs(p, true) -} - -// CertifiedAddrs returns all of the non-expired address that have been -// certified by the given peer. -func (ab *dsAddrBook) CertifiedAddrs(p peer.ID) []ma.Multiaddr { - return ab.addrs(p, false) -} - -func (ab *dsAddrBook) addrs(p peer.ID, includeUncertified bool) []ma.Multiaddr { pr, err := ab.loadRecord(p, true, true) if err != nil { log.Warning("failed to load peerstore entry for peer %v while querying addrs, err: %v", p, err) @@ -402,14 +392,12 @@ func (ab *dsAddrBook) addrs(p peer.ID, includeUncertified bool) []ma.Multiaddr { pr.RLock() defer pr.RUnlock() - addrs := make([]ma.Multiaddr, 0, len(pr.Addrs)) + var addrs []ma.Multiaddr for _, a := range pr.SignedAddrs { addrs = append(addrs, a.Addr) } - if includeUncertified { - for _, a := range pr.Addrs { - addrs = append(addrs, a.Addr) - } + for _, a := range pr.Addrs { + addrs = append(addrs, a.Addr) } return addrs } @@ -476,7 +464,6 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio return nil } - var removeFromUnsigned []ma.Multiaddr var added []*pb.AddrBookRecord_AddrEntry for _, incoming := range addrs { @@ -484,15 +471,11 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio // we always check if the incoming addr is already in the // signed addr list, even if we're not adding it from a - // signed source. if it does exist, we update its TTL and - // make sure it gets removed from the unsigned addr list + // signed source. if it does exist, we update its TTL existingSigned := updateExisting(pr.SignedAddrs, incoming) if existingSigned != nil { added = append(added, existingSigned) } - if signed || existingSigned != nil { - removeFromUnsigned = append(removeFromUnsigned, incoming) - } // if we're adding a signed addr that already existed in the // unsigned set, we move the existing unsigned entry over @@ -519,12 +502,17 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio if signed { // when adding signed addrs, we want to keep _only_ the incoming addrs pr.SignedAddrs = added + pr.Addrs = make([]*pb.AddrBookRecord_AddrEntry, 0) } else { - pr.Addrs = append(pr.Addrs, added...) + // if the incoming addrs are unsigned, we only keep them if + // no signed addrs exist for the peer + if len(pr.SignedAddrs) == 0 { + pr.Addrs = append(pr.Addrs, added...) + } else { + pr.Addrs = make([]*pb.AddrBookRecord_AddrEntry, 0) + } } - pr.Addrs = deleteInPlace(pr.Addrs, removeFromUnsigned) - pr.dirty = true pr.clean() return pr.flush(ab.ds) @@ -562,7 +550,7 @@ func (ab *dsAddrBook) deleteAddrs(p peer.ID, addrs []ma.Multiaddr) (err error) { return fmt.Errorf("failed to load peerstore entry for peer %v while deleting addrs, err: %v", p, err) } - if pr.Addrs == nil { + if pr.Addrs == nil && pr.SignedAddrs == nil { return nil } diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 6e2106a..8bdb6dc 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -57,6 +57,7 @@ type memoryAddrBook struct { } var _ pstore.AddrBook = (*memoryAddrBook)(nil) +var _ pstore.CertifiedAddrBook = (*memoryAddrBook)(nil) func NewAddrBook() pstore.AddrBook { ctx, cancel := context.WithCancel(context.Background()) @@ -171,11 +172,11 @@ func (mab *memoryAddrBook) AddCertifiedAddrs(state *routing.SignedRoutingState, s.Lock() lastState, found := s.signedRoutingStates[state.PeerID] if found && lastState.Seq >= state.Seq { - // TODO: should this be an error? + s.Unlock() return nil } s.signedRoutingStates[state.PeerID] = state - s.Unlock() + s.Unlock() // need to release the lock, since addAddrs will try to take it mab.addAddrs(state.PeerID, state.Addrs, ttl, true) return nil } @@ -189,10 +190,10 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du s := mab.segments.get(p) s.Lock() defer s.Unlock() - amap, ok := s.addrs[p] + existingUnsignedAddrs, ok := s.addrs[p] if !ok { - amap = make(map[string]*expiringAddr) - s.addrs[p] = amap + existingUnsignedAddrs = make(map[string]*expiringAddr) + s.addrs[p] = existingUnsignedAddrs } existingSignedAddrs, ok := s.signedAddrs[p] if !ok { @@ -200,11 +201,18 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du s.signedAddrs[p] = existingSignedAddrs } + // if we already have signed addrs, ignore attempts to add unsigned addrs + if !signed && len(existingSignedAddrs) != 0 { + return + } + // if we're adding signed addresses, we want _only_ the new addrs // to be in the final map, so we make a new map to contain them signedAddrMap := existingSignedAddrs + unsignedAddrMap := existingUnsignedAddrs if signed { signedAddrMap = make(map[string]*expiringAddr, len(addrs)) + unsignedAddrMap = make(map[string]*expiringAddr, len(addrs)) } exp := time.Now().Add(ttl) @@ -229,7 +237,7 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // find the highest TTL and Expiry time between // existing records and function args - a, found := amap[k] // won't allocate. + a, found := existingUnsignedAddrs[k] // won't allocate. b, foundSigned := existingSignedAddrs[k] maxTTL, maxExp := maxTTLAndExp(a, ttl, exp) maxTTL, maxExp = maxTTLAndExp(b, maxTTL, maxExp) @@ -240,9 +248,9 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // make sure it's not also in the unsigned addr list. if signed || foundSigned { signedAddrMap[k] = entry - delete(amap, k) + delete(existingUnsignedAddrs, k) } else { - amap[k] = entry + existingUnsignedAddrs[k] = entry } if !found && !foundSigned { @@ -254,6 +262,7 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // replace existing signed addrs with new ones // this is a noop if we're adding unsigned addrs s.signedAddrs[p] = signedAddrMap + s.addrs[p] = unsignedAddrMap } // SetAddr calls mgr.SetAddrs(p, addr, ttl) @@ -354,16 +363,6 @@ func (mab *memoryAddrBook) Addrs(p peer.ID) []ma.Multiaddr { validAddrs(s.addrs[p])...) } -// CertifiedAddrs returns all known (and valid) addressed that have been -// certified by the given peer. -func (mab *memoryAddrBook) CertifiedAddrs(p peer.ID) []ma.Multiaddr { - s := mab.segments.get(p) - s.RLock() - defer s.RUnlock() - - return validAddrs(s.signedAddrs[p]) -} - func validAddrs(amap map[string]*expiringAddr) []ma.Multiaddr { now := time.Now() good := make([]ma.Multiaddr, 0, len(amap)) diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index 0cfe8d8..8ea3ff5 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -364,6 +364,8 @@ func testPeersWithAddrs(m pstore.AddrBook) func(t *testing.T) { func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { return func(t *testing.T) { + cab := m.(pstore.CertifiedAddrBook) + priv, _, err := test.RandTestKeyPair(crypto.Ed25519, 256) if err != nil { t.Errorf("error generating testing keys: %v", err) @@ -378,36 +380,32 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { t.Errorf("error creating signed routing record: %v", err) } + // add a few non-certified addrs + m.AddAddrs(id, uncertifiedAddrs, time.Hour) + + // make sure they're present + AssertAddressesEqual(t, uncertifiedAddrs, m.Addrs(id)) + // add the signed record to addr book - err = m.AddCertifiedAddrs(signedState, time.Hour) + err = cab.AddCertifiedAddrs(signedState, time.Hour) if err != nil { t.Errorf("error adding signed routing record to addrbook: %v", err) } - // add a few non-certified addrs - m.AddAddrs(id, uncertifiedAddrs, time.Hour) - - // we should get only certified addrs back from CertifiedAddrs - AssertAddressesEqual(t, certifiedAddrs, m.CertifiedAddrs(id)) - - // we should get everything back from Addrs - AssertAddressesEqual(t, allAddrs, m.Addrs(id)) + // the non-certified addrs should be gone & we should get only certified addrs back from Addrs + AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) // PeersWithAddrs should return a single peer if len(m.PeersWithAddrs()) != 1 { t.Errorf("expected PeersWithAddrs to return 1, got %d", len(m.PeersWithAddrs())) } - // adding a previously certified address via AddAddrs should not result - // in duplicates - m.AddAddrs(id, certifiedAddrs, time.Hour) - AssertAddressesEqual(t, allAddrs, m.Addrs(id)) - - // and they should still be considered certified - AssertAddressesEqual(t, certifiedAddrs, m.CertifiedAddrs(id)) + // once certified addrs exist, trying to add non-certified addrs should have no effect + m.AddAddrs(id, uncertifiedAddrs, time.Hour) + AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) // we should be able to retrieve the signed state - state2 := m.SignedRoutingState(id) + state2 := cab.SignedRoutingState(id) if state2 == nil || !signedState.Equal(state2) { t.Error("unable to retrieve signed routing record from addrbook") } @@ -417,17 +415,17 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { certifiedAddrs = certifiedAddrs[:3] signedState, err = routing.MakeSignedRoutingState(priv, certifiedAddrs) test.AssertNilError(t, err) - err = m.AddCertifiedAddrs(signedState, time.Hour) + err = cab.AddCertifiedAddrs(signedState, time.Hour) test.AssertNilError(t, err) - AssertAddressesEqual(t, certifiedAddrs, m.CertifiedAddrs(id)) + AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) // update TTL on signed addrs to -1 to remove them. // the signed routing record should be deleted m.SetAddrs(id, certifiedAddrs, -1) - if len(m.CertifiedAddrs(id)) != 0 { + if len(m.Addrs(id)) != 0 { t.Error("expected zero certified addrs after setting TTL to -1") } - if m.SignedRoutingState(id) != nil { + if cab.SignedRoutingState(id) != nil { t.Error("expected signed routing record to be removed when addresses expire") } } From 1c176d574e3199b72162fbb254f55846c20812ce Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 6 Jan 2020 15:57:27 -0500 Subject: [PATCH 23/39] test that peerstore is a CertifiedAddrBook --- test/peerstore_suite.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/peerstore_suite.go b/test/peerstore_suite.go index a0ae326..8a62962 100644 --- a/test/peerstore_suite.go +++ b/test/peerstore_suite.go @@ -23,6 +23,7 @@ var peerstoreSuite = map[string]func(pstore.Peerstore) func(*testing.T){ "PeerstoreProtoStore": testPeerstoreProtoStore, "BasicPeerstore": testBasicPeerstore, "Metadata": testMetadata, + "CertifiedAddrBook": testCertifiedAddrBook, } type PeerstoreFactory func() (pstore.Peerstore, func()) @@ -357,6 +358,15 @@ func testMetadata(ps pstore.Peerstore) func(t *testing.T) { } } +func testCertifiedAddrBook(ps pstore.Peerstore) func(*testing.T) { + return func(t *testing.T) { + _, ok := ps.(pstore.CertifiedAddrBook) + if !ok { + t.Error("expected peerstore to implement CertifiedAddrBook interface") + } + } +} + func getAddrs(t *testing.T, n int) []ma.Multiaddr { var addrs []ma.Multiaddr for i := 0; i < n; i++ { From db7fc6596cda6a913e4716c53ad58b2fa1929355 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 6 Jan 2020 15:58:22 -0500 Subject: [PATCH 24/39] return concrete types from constructor funcs --- pstoreds/keybook.go | 2 +- pstoreds/metadata.go | 2 +- pstoreds/protobook.go | 2 +- pstoremem/addr_book.go | 2 +- pstoremem/keybook.go | 2 +- pstoremem/metadata.go | 2 +- pstoremem/protobook.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pstoreds/keybook.go b/pstoreds/keybook.go index 64b73a3..35021b2 100644 --- a/pstoreds/keybook.go +++ b/pstoreds/keybook.go @@ -28,7 +28,7 @@ type dsKeyBook struct { var _ pstore.KeyBook = (*dsKeyBook)(nil) -func NewKeyBook(_ context.Context, store ds.Datastore, _ Options) (pstore.KeyBook, error) { +func NewKeyBook(_ context.Context, store ds.Datastore, _ Options) (*dsKeyBook, error) { return &dsKeyBook{store}, nil } diff --git a/pstoreds/metadata.go b/pstoreds/metadata.go index f54f382..4d285af 100644 --- a/pstoreds/metadata.go +++ b/pstoreds/metadata.go @@ -36,7 +36,7 @@ func init() { // See `init()` to learn which types are registered by default. Modules wishing to store // values of other types will need to `gob.Register()` them explicitly, or else callers // will receive runtime errors. -func NewPeerMetadata(_ context.Context, store ds.Datastore, _ Options) (pstore.PeerMetadata, error) { +func NewPeerMetadata(_ context.Context, store ds.Datastore, _ Options) (*dsPeerMetadata, error) { return &dsPeerMetadata{store}, nil } diff --git a/pstoreds/protobook.go b/pstoreds/protobook.go index fc084d6..27e70f5 100644 --- a/pstoreds/protobook.go +++ b/pstoreds/protobook.go @@ -26,7 +26,7 @@ type dsProtoBook struct { var _ pstore.ProtoBook = (*dsProtoBook)(nil) -func NewProtoBook(meta pstore.PeerMetadata) pstore.ProtoBook { +func NewProtoBook(meta pstore.PeerMetadata) *dsProtoBook { return &dsProtoBook{ meta: meta, segments: func() (ret protoSegments) { diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 8bdb6dc..1b1b902 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -59,7 +59,7 @@ type memoryAddrBook struct { var _ pstore.AddrBook = (*memoryAddrBook)(nil) var _ pstore.CertifiedAddrBook = (*memoryAddrBook)(nil) -func NewAddrBook() pstore.AddrBook { +func NewAddrBook() *memoryAddrBook { ctx, cancel := context.WithCancel(context.Background()) ab := &memoryAddrBook{ diff --git a/pstoremem/keybook.go b/pstoremem/keybook.go index f6ab84d..79f2550 100644 --- a/pstoremem/keybook.go +++ b/pstoremem/keybook.go @@ -19,7 +19,7 @@ type memoryKeyBook struct { var _ pstore.KeyBook = (*memoryKeyBook)(nil) // noop new, but in the future we may want to do some init work. -func NewKeyBook() pstore.KeyBook { +func NewKeyBook() *memoryKeyBook { return &memoryKeyBook{ pks: map[peer.ID]ic.PubKey{}, sks: map[peer.ID]ic.PrivKey{}, diff --git a/pstoremem/metadata.go b/pstoremem/metadata.go index 821c2b6..7ded769 100644 --- a/pstoremem/metadata.go +++ b/pstoremem/metadata.go @@ -27,7 +27,7 @@ type memoryPeerMetadata struct { var _ pstore.PeerMetadata = (*memoryPeerMetadata)(nil) -func NewPeerMetadata() pstore.PeerMetadata { +func NewPeerMetadata() *memoryPeerMetadata { return &memoryPeerMetadata{ ds: make(map[metakey]interface{}), interned: make(map[string]interface{}), diff --git a/pstoremem/protobook.go b/pstoremem/protobook.go index d627c6c..04d8ec4 100644 --- a/pstoremem/protobook.go +++ b/pstoremem/protobook.go @@ -28,7 +28,7 @@ type memoryProtoBook struct { var _ pstore.ProtoBook = (*memoryProtoBook)(nil) -func NewProtoBook() pstore.ProtoBook { +func NewProtoBook() *memoryProtoBook { return &memoryProtoBook{ interned: make(map[string]string, 256), segments: func() (ret protoSegments) { From 55e6abc54a35ca5bc0b2097777127d6feb396263 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 6 Jan 2020 15:59:15 -0500 Subject: [PATCH 25/39] embed concrete types into peerstore structs --- peerstore.go | 1 + pstoreds/peerstore.go | 65 ++++++++++++++++++++++++++++++++++-- pstoremem/peerstore.go | 75 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 132 insertions(+), 9 deletions(-) diff --git a/peerstore.go b/peerstore.go index 3331b4a..278a5f8 100644 --- a/peerstore.go +++ b/peerstore.go @@ -21,6 +21,7 @@ type peerstore struct { // NewPeerstore creates a data structure that stores peer data, backed by the // supplied implementations of KeyBook, AddrBook and PeerMetadata. +// Deprecated: use pstoreds.NewPeerstore or peerstoremem.NewPeerstore instead. func NewPeerstore(kb pstore.KeyBook, ab pstore.AddrBook, pb pstore.ProtoBook, md pstore.PeerMetadata) pstore.Peerstore { return &peerstore{ KeyBook: kb, diff --git a/pstoreds/peerstore.go b/pstoreds/peerstore.go index 99b381d..3c8aa0f 100644 --- a/pstoreds/peerstore.go +++ b/pstoreds/peerstore.go @@ -2,6 +2,8 @@ package pstoreds import ( "context" + "fmt" + "io" "time" base32 "github.com/multiformats/go-base32" @@ -47,8 +49,17 @@ func DefaultOpts() Options { } } +type pstoreds struct { + peerstore.Metrics + + dsKeyBook + dsAddrBook + dsProtoBook + dsPeerMetadata +} + // NewPeerstore creates a peerstore backed by the provided persistent datastore. -func NewPeerstore(ctx context.Context, store ds.Batching, opts Options) (peerstore.Peerstore, error) { +func NewPeerstore(ctx context.Context, store ds.Batching, opts Options) (*pstoreds, error) { addrBook, err := NewAddrBook(ctx, store, opts) if err != nil { return nil, err @@ -66,7 +77,13 @@ func NewPeerstore(ctx context.Context, store ds.Batching, opts Options) (peersto protoBook := NewProtoBook(peerMetadata) - ps := pstore.NewPeerstore(keyBook, addrBook, protoBook, peerMetadata) + ps := &pstoreds{ + Metrics: pstore.NewMetrics(), + dsKeyBook: *keyBook, + dsAddrBook: *addrBook, + dsPeerMetadata: *peerMetadata, + dsProtoBook: *protoBook, + } return ps, nil } @@ -103,3 +120,47 @@ func uniquePeerIds(ds ds.Datastore, prefix ds.Key, extractor func(result query.R } return ids, nil } + +func (ps *pstoreds) Close() (err error) { + var errs []error + weakClose := func(name string, c interface{}) { + if cl, ok := c.(io.Closer); ok { + if err = cl.Close(); err != nil { + errs = append(errs, fmt.Errorf("%s error: %s", name, err)) + } + } + } + + weakClose("keybook", ps.dsKeyBook) + weakClose("addressbook", ps.dsAddrBook) + weakClose("protobook", ps.dsProtoBook) + weakClose("peermetadata", ps.dsPeerMetadata) + + if len(errs) > 0 { + return fmt.Errorf("failed while closing peerstore; err(s): %q", errs) + } + return nil +} + +func (ps *pstoreds) Peers() peer.IDSlice { + set := map[peer.ID]struct{}{} + for _, p := range ps.PeersWithKeys() { + set[p] = struct{}{} + } + for _, p := range ps.PeersWithAddrs() { + set[p] = struct{}{} + } + + pps := make(peer.IDSlice, 0, len(set)) + for p := range set { + pps = append(pps, p) + } + return pps +} + +func (ps *pstoreds) PeerInfo(p peer.ID) peer.AddrInfo { + return peer.AddrInfo{ + ID: p, + Addrs: ps.dsAddrBook.Addrs(p), + } +} diff --git a/pstoremem/peerstore.go b/pstoremem/peerstore.go index c7cbd67..f16887d 100644 --- a/pstoremem/peerstore.go +++ b/pstoremem/peerstore.go @@ -1,12 +1,73 @@ package pstoremem -import pstore "github.com/libp2p/go-libp2p-peerstore" +import ( + "fmt" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peerstore" + pstore "github.com/libp2p/go-libp2p-peerstore" + "io" +) + +type pstoremem struct { + peerstore.Metrics + + memoryKeyBook + memoryAddrBook + memoryProtoBook + memoryPeerMetadata +} // NewPeerstore creates an in-memory threadsafe collection of peers. -func NewPeerstore() pstore.Peerstore { - return pstore.NewPeerstore( - NewKeyBook(), - NewAddrBook(), - NewProtoBook(), - NewPeerMetadata()) +func NewPeerstore() *pstoremem { + return &pstoremem{ + Metrics: pstore.NewMetrics(), + memoryKeyBook: *NewKeyBook(), + memoryAddrBook: *NewAddrBook(), + memoryProtoBook: *NewProtoBook(), + memoryPeerMetadata: *NewPeerMetadata(), + } +} + +func (ps *pstoremem) Close() (err error) { + var errs []error + weakClose := func(name string, c interface{}) { + if cl, ok := c.(io.Closer); ok { + if err = cl.Close(); err != nil { + errs = append(errs, fmt.Errorf("%s error: %s", name, err)) + } + } + } + + weakClose("keybook", ps.memoryKeyBook) + weakClose("addressbook", ps.memoryAddrBook) + weakClose("protobook", ps.memoryProtoBook) + weakClose("peermetadata", ps.memoryPeerMetadata) + + if len(errs) > 0 { + return fmt.Errorf("failed while closing peerstore; err(s): %q", errs) + } + return nil +} + +func (ps *pstoremem) Peers() peer.IDSlice { + set := map[peer.ID]struct{}{} + for _, p := range ps.PeersWithKeys() { + set[p] = struct{}{} + } + for _, p := range ps.PeersWithAddrs() { + set[p] = struct{}{} + } + + pps := make(peer.IDSlice, 0, len(set)) + for p := range set { + pps = append(pps, p) + } + return pps +} + +func (ps *pstoremem) PeerInfo(p peer.ID) peer.AddrInfo { + return peer.AddrInfo{ + ID: p, + Addrs: ps.memoryAddrBook.Addrs(p), + } } From 8c8e310db686e223eff7b58a9d460ddfd968bcd1 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 7 Jan 2020 12:53:19 -0500 Subject: [PATCH 26/39] update to reflect name changes in -core --- pb/pstore.pb.go | 251 ++++++++++++++++++++-------------------- pb/pstore.proto | 9 +- pstoreds/addr_book.go | 39 ++++--- pstoremem/addr_book.go | 59 ++++++---- test/addr_book_suite.go | 29 +++-- 5 files changed, 208 insertions(+), 179 deletions(-) diff --git a/pb/pstore.pb.go b/pb/pstore.pb.go index b398e54..aa4e253 100644 --- a/pb/pstore.pb.go +++ b/pb/pstore.pb.go @@ -9,6 +9,7 @@ import ( proto "github.com/gogo/protobuf/proto" io "io" math "math" + math_bits "math/bits" ) // Reference imports to suppress errors if they are not otherwise used. @@ -20,17 +21,16 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package // AddrBookRecord represents a record for a peer in the address book. type AddrBookRecord struct { // The peer ID. Id *ProtoPeerID `protobuf:"bytes,1,opt,name=id,proto3,customtype=ProtoPeerID" json:"id,omitempty"` // The multiaddresses. This is a sorted list where element 0 expires the soonest. - Addrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,2,rep,name=addrs,proto3" json:"addrs,omitempty"` - // Multiaddresses that were sourced from a signed RoutingState record. + Addrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,2,rep,name=addrs,proto3" json:"addrs,omitempty"` SignedAddrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,3,rep,name=signedAddrs,proto3" json:"signedAddrs,omitempty"` - // The most recently received signed RoutingState record. + // The most recently received signed PeerRecord. CertifiedRecord *AddrBookRecord_CertifiedRecord `protobuf:"bytes,4,opt,name=certifiedRecord,proto3" json:"certifiedRecord,omitempty"` } @@ -48,7 +48,7 @@ func (m *AddrBookRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro return xxx_messageInfo_AddrBookRecord.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -111,7 +111,7 @@ func (m *AddrBookRecord_AddrEntry) XXX_Marshal(b []byte, deterministic bool) ([] return xxx_messageInfo_AddrBookRecord_AddrEntry.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -144,12 +144,12 @@ func (m *AddrBookRecord_AddrEntry) GetTtl() int64 { return 0 } -// CertifiedRecord contains a serialized RoutingState record used to +// CertifiedRecord contains a serialized signed PeerRecord used to // populate the signedAddrs list. type AddrBookRecord_CertifiedRecord struct { - // The Seq counter from the RoutingState record + // The Seq counter from the signed PeerRecord envelope Seq uint64 `protobuf:"varint,1,opt,name=seq,proto3" json:"seq,omitempty"` - // The serialized bytes of the SignedEnvelope containing the RoutingState record. + // The serialized bytes of the SignedEnvelope containing the PeerRecord. Raw []byte `protobuf:"bytes,2,opt,name=raw,proto3" json:"raw,omitempty"` } @@ -167,7 +167,7 @@ func (m *AddrBookRecord_CertifiedRecord) XXX_Marshal(b []byte, deterministic boo return xxx_messageInfo_AddrBookRecord_CertifiedRecord.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -236,7 +236,7 @@ var fileDescriptor_f96873690e08a98f = []byte{ func (m *AddrBookRecord) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -244,61 +244,74 @@ func (m *AddrBookRecord) Marshal() (dAtA []byte, err error) { } func (m *AddrBookRecord) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *AddrBookRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Id != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintPstore(dAtA, i, uint64(m.Id.Size())) - n1, err := m.Id.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n1 - } - if len(m.Addrs) > 0 { - for _, msg := range m.Addrs { - dAtA[i] = 0x12 - i++ - i = encodeVarintPstore(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) + if m.CertifiedRecord != nil { + { + size, err := m.CertifiedRecord.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } - i += n + i -= size + i = encodeVarintPstore(dAtA, i, uint64(size)) } + i-- + dAtA[i] = 0x22 } if len(m.SignedAddrs) > 0 { - for _, msg := range m.SignedAddrs { + for iNdEx := len(m.SignedAddrs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.SignedAddrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPstore(dAtA, i, uint64(size)) + } + i-- dAtA[i] = 0x1a - i++ - i = encodeVarintPstore(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + } + } + if len(m.Addrs) > 0 { + for iNdEx := len(m.Addrs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Addrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPstore(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x12 } } - if m.CertifiedRecord != nil { - dAtA[i] = 0x22 - i++ - i = encodeVarintPstore(dAtA, i, uint64(m.CertifiedRecord.Size())) - n2, err := m.CertifiedRecord.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.Id != nil { + { + size := m.Id.Size() + i -= size + if _, err := m.Id.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + i = encodeVarintPstore(dAtA, i, uint64(size)) } - i += n2 + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *AddrBookRecord_AddrEntry) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -306,37 +319,44 @@ func (m *AddrBookRecord_AddrEntry) Marshal() (dAtA []byte, err error) { } func (m *AddrBookRecord_AddrEntry) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *AddrBookRecord_AddrEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Addr != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintPstore(dAtA, i, uint64(m.Addr.Size())) - n3, err := m.Addr.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n3 + if m.Ttl != 0 { + i = encodeVarintPstore(dAtA, i, uint64(m.Ttl)) + i-- + dAtA[i] = 0x18 } if m.Expiry != 0 { - dAtA[i] = 0x10 - i++ i = encodeVarintPstore(dAtA, i, uint64(m.Expiry)) + i-- + dAtA[i] = 0x10 } - if m.Ttl != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPstore(dAtA, i, uint64(m.Ttl)) + if m.Addr != nil { + { + size := m.Addr.Size() + i -= size + if _, err := m.Addr.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + i = encodeVarintPstore(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *AddrBookRecord_CertifiedRecord) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -344,51 +364,59 @@ func (m *AddrBookRecord_CertifiedRecord) Marshal() (dAtA []byte, err error) { } func (m *AddrBookRecord_CertifiedRecord) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *AddrBookRecord_CertifiedRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Seq != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintPstore(dAtA, i, uint64(m.Seq)) - } if len(m.Raw) > 0 { - dAtA[i] = 0x12 - i++ + i -= len(m.Raw) + copy(dAtA[i:], m.Raw) i = encodeVarintPstore(dAtA, i, uint64(len(m.Raw))) - i += copy(dAtA[i:], m.Raw) + i-- + dAtA[i] = 0x12 } - return i, nil + if m.Seq != 0 { + i = encodeVarintPstore(dAtA, i, uint64(m.Seq)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil } func encodeVarintPstore(dAtA []byte, offset int, v uint64) int { + offset -= sovPstore(v) + base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) - return offset + 1 + return base } func NewPopulatedAddrBookRecord(r randyPstore, easy bool) *AddrBookRecord { this := &AddrBookRecord{} this.Id = NewPopulatedProtoPeerID(r) - if r.Intn(10) != 0 { + if r.Intn(5) != 0 { v1 := r.Intn(5) this.Addrs = make([]*AddrBookRecord_AddrEntry, v1) for i := 0; i < v1; i++ { this.Addrs[i] = NewPopulatedAddrBookRecord_AddrEntry(r, easy) } } - if r.Intn(10) != 0 { + if r.Intn(5) != 0 { v2 := r.Intn(5) this.SignedAddrs = make([]*AddrBookRecord_AddrEntry, v2) for i := 0; i < v2; i++ { this.SignedAddrs[i] = NewPopulatedAddrBookRecord_AddrEntry(r, easy) } } - if r.Intn(10) != 0 { + if r.Intn(5) != 0 { this.CertifiedRecord = NewPopulatedAddrBookRecord_CertifiedRecord(r, easy) } if !easy && r.Intn(10) != 0 { @@ -562,14 +590,7 @@ func (m *AddrBookRecord_CertifiedRecord) Size() (n int) { } func sovPstore(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n + return (math_bits.Len64(x|1) + 6) / 7 } func sozPstore(x uint64) (n int) { return sovPstore(uint64((x << 1) ^ uint64((int64(x) >> 63)))) @@ -1001,6 +1022,7 @@ func (m *AddrBookRecord_CertifiedRecord) Unmarshal(dAtA []byte) error { func skipPstore(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 + depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -1032,10 +1054,8 @@ func skipPstore(dAtA []byte) (n int, err error) { break } } - return iNdEx, nil case 1: iNdEx += 8 - return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -1056,55 +1076,30 @@ func skipPstore(dAtA []byte) (n int, err error) { return 0, ErrInvalidLengthPstore } iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthPstore - } - return iNdEx, nil case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPstore - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipPstore(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthPstore - } - } - return iNdEx, nil + depth++ case 4: - return iNdEx, nil + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupPstore + } + depth-- case 5: iNdEx += 4 - return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } + if iNdEx < 0 { + return 0, ErrInvalidLengthPstore + } + if depth == 0 { + return iNdEx, nil + } } - panic("unreachable") + return 0, io.ErrUnexpectedEOF } var ( - ErrInvalidLengthPstore = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowPstore = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthPstore = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowPstore = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupPstore = fmt.Errorf("proto: unexpected end of group") ) diff --git a/pb/pstore.proto b/pb/pstore.proto index e0724e2..2a683d7 100644 --- a/pb/pstore.proto +++ b/pb/pstore.proto @@ -14,10 +14,9 @@ message AddrBookRecord { // The multiaddresses. This is a sorted list where element 0 expires the soonest. repeated AddrEntry addrs = 2; - // Multiaddresses that were sourced from a signed RoutingState record. repeated AddrEntry signedAddrs = 3; - // The most recently received signed RoutingState record. + // The most recently received signed PeerRecord. CertifiedRecord certifiedRecord = 4; // AddrEntry represents a single multiaddress. @@ -31,13 +30,13 @@ message AddrBookRecord { int64 ttl = 3; } - // CertifiedRecord contains a serialized RoutingState record used to + // CertifiedRecord contains a serialized signed PeerRecord used to // populate the signedAddrs list. message CertifiedRecord { - // The Seq counter from the RoutingState record + // The Seq counter from the signed PeerRecord envelope uint64 seq = 1; - // The serialized bytes of the SignedEnvelope containing the RoutingState record. + // The serialized bytes of the SignedEnvelope containing the PeerRecord. bytes raw = 2; } } diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 2f21c13..515e1ea 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -3,7 +3,7 @@ package pstoreds import ( "context" "fmt" - "github.com/libp2p/go-libp2p-core/routing" + "github.com/libp2p/go-libp2p-core/record" "sort" "sync" "time" @@ -118,6 +118,10 @@ func (r *addrsRecord) clean() (chgd bool) { r.Addrs = removeExpired(r.Addrs, now) r.SignedAddrs = removeExpired(r.SignedAddrs, now) + if len(r.SignedAddrs) == 0 { + r.CertifiedRecord = nil + } + return r.dirty || len(r.Addrs) != addrsLen || len(r.SignedAddrs) != signedAddrsLen } @@ -275,22 +279,26 @@ func (ab *dsAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio // AddCertifiedAddrs adds addresses from a routing.RoutingState record // contained in the given SignedEnvelope. -func (ab *dsAddrBook) AddCertifiedAddrs(state *routing.SignedRoutingState, ttl time.Duration) error { +func (ab *dsAddrBook) AddCertifiedAddrs(recordEnvelope *record.SignedEnvelope, ttl time.Duration) error { + rec, err := peer.PeerRecordFromSignedEnvelope(recordEnvelope) + if err != nil { + return err + } // ensure that the seq number from envelope is >= any previously received seq no - if ab.latestRoutingStateSeq(state.PeerID) >= state.Seq { + if ab.latestPeerRecordSeq(rec.PeerID) >= recordEnvelope.Seq { return nil } - addrs := cleanAddrs(state.Addrs) - err := ab.setAddrs(state.PeerID, addrs, ttl, ttlExtend, true) + addrs := cleanAddrs(rec.Addrs) + err = ab.setAddrs(rec.PeerID, addrs, ttl, ttlExtend, true) if err != nil { return err } - return ab.storeRoutingState(state.PeerID, state) + return ab.storeSignedPeerRecord(rec.PeerID, recordEnvelope) } -func (ab *dsAddrBook) latestRoutingStateSeq(p peer.ID) uint64 { +func (ab *dsAddrBook) latestPeerRecordSeq(p peer.ID) uint64 { pr, err := ab.loadRecord(p, true, false) if err != nil || pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { return 0 @@ -298,8 +306,8 @@ func (ab *dsAddrBook) latestRoutingStateSeq(p peer.ID) uint64 { return pr.CertifiedRecord.Seq } -func (ab *dsAddrBook) storeRoutingState(p peer.ID, state *routing.SignedRoutingState) error { - envelopeBytes, err := state.Marshal() +func (ab *dsAddrBook) storeSignedPeerRecord(p peer.ID, envelope *record.SignedEnvelope) error { + envelopeBytes, err := envelope.Marshal() if err != nil { return err } @@ -312,7 +320,7 @@ func (ab *dsAddrBook) storeRoutingState(p peer.ID, state *routing.SignedRoutingS return err } pr.CertifiedRecord = &pb.AddrBookRecord_CertifiedRecord{ - Seq: state.Seq, + Seq: envelope.Seq, Raw: envelopeBytes, } pr.dirty = true @@ -320,9 +328,10 @@ func (ab *dsAddrBook) storeRoutingState(p peer.ID, state *routing.SignedRoutingS return err } -// SignedRoutingState returns a signed RoutingState record for the -// given peer id, if one exists in the peerstore. -func (ab *dsAddrBook) SignedRoutingState(p peer.ID) *routing.SignedRoutingState { +// SignedPeerRecord returns a SignedEnvelope containing a PeerRecord for the +// given peer id, if one exists. +// Returns nil if no signed PeerRecord exists for the peer. +func (ab *dsAddrBook) SignedPeerRecord(p peer.ID) *record.SignedEnvelope { pr, err := ab.loadRecord(p, true, false) if err != nil { log.Errorf("unable to load record for peer %s: %v", p.Pretty(), err) @@ -331,9 +340,9 @@ func (ab *dsAddrBook) SignedRoutingState(p peer.ID) *routing.SignedRoutingState if pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { return nil } - state, err := routing.UnmarshalSignedRoutingState(pr.CertifiedRecord.Raw) + state, err := record.ConsumeEnvelope(pr.CertifiedRecord.Raw, peer.PeerRecordEnvelopeDomain) if err != nil { - log.Errorf("error unmarshaling stored SignedRoutingState record for peer %s: %v", p.Pretty(), err) + log.Errorf("error unmarshaling stored signed peer record for peer %s: %v", p.Pretty(), err) return nil } return state diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 1b1b902..9c575b7 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -2,13 +2,13 @@ package pstoremem import ( "context" + "github.com/libp2p/go-libp2p-core/record" "sort" "sync" "time" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/routing" ma "github.com/multiformats/go-multiaddr" pstore "github.com/libp2p/go-libp2p-core/peerstore" @@ -39,7 +39,7 @@ type addrSegment struct { signedAddrs map[peer.ID]map[string]*expiringAddr - signedRoutingStates map[peer.ID]*routing.SignedRoutingState + signedPeerRecords map[peer.ID]*record.SignedEnvelope } func (segments *addrSegments) get(p peer.ID) *addrSegment { @@ -66,9 +66,9 @@ func NewAddrBook() *memoryAddrBook { segments: func() (ret addrSegments) { for i, _ := range ret { ret[i] = &addrSegment{ - addrs: make(map[peer.ID]map[string]*expiringAddr), - signedAddrs: make(map[peer.ID]map[string]*expiringAddr), - signedRoutingStates: make(map[peer.ID]*routing.SignedRoutingState)} + addrs: make(map[peer.ID]map[string]*expiringAddr), + signedAddrs: make(map[peer.ID]map[string]*expiringAddr), + signedPeerRecords: make(map[peer.ID]*record.SignedEnvelope)} } return ret }(), @@ -126,7 +126,7 @@ func (mab *memoryAddrBook) gc() { collected := collect(s.signedAddrs) // remove routing records for peers whose signed addrs have all been removed for _, p := range collected { - delete(s.signedRoutingStates, p) + delete(s.signedPeerRecords, p) } s.Unlock() } @@ -164,20 +164,23 @@ func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du mab.addAddrs(p, addrs, ttl, false) } -// AddCertifiedAddrs adds addresses from a routing.RoutingState record -// contained in the given SignedEnvelope. -func (mab *memoryAddrBook) AddCertifiedAddrs(state *routing.SignedRoutingState, ttl time.Duration) error { +// AddCertifiedAddrs adds addresses from a signed peer.PeerRecord contained in a SignedEnvelope. +func (mab *memoryAddrBook) AddCertifiedAddrs(recordEnvelope *record.SignedEnvelope, ttl time.Duration) error { + rec, err := peer.PeerRecordFromSignedEnvelope(recordEnvelope) + if err != nil { + return err + } // ensure seq is greater than last received - s := mab.segments.get(state.PeerID) + s := mab.segments.get(rec.PeerID) s.Lock() - lastState, found := s.signedRoutingStates[state.PeerID] - if found && lastState.Seq >= state.Seq { + lastState, found := s.signedPeerRecords[rec.PeerID] + if found && lastState.Seq >= recordEnvelope.Seq { s.Unlock() return nil } - s.signedRoutingStates[state.PeerID] = state + s.signedPeerRecords[rec.PeerID] = recordEnvelope s.Unlock() // need to release the lock, since addAddrs will try to take it - mab.addAddrs(state.PeerID, state.Addrs, ttl, true) + mab.addAddrs(rec.PeerID, rec.Addrs, ttl, true) return nil } @@ -263,6 +266,11 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // this is a noop if we're adding unsigned addrs s.signedAddrs[p] = signedAddrMap s.addrs[p] = unsignedAddrMap + + // if we've expired all the signed addresses for a peer, remove their signed routing state record + if len(signedAddrMap) == 0 { + delete(s.signedPeerRecords, p) + } } // SetAddr calls mgr.SetAddrs(p, addr, ttl) @@ -317,7 +325,7 @@ func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du // if we've expired all the signed addresses for a peer, remove their signed routing state record if len(signedAddrMap) == 0 { - delete(s.signedRoutingStates, p) + delete(s.signedPeerRecords, p) } } @@ -347,7 +355,7 @@ func (mab *memoryAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL t // updating may have caused all our certified addrs to expire. // if so, remove the signed record we sourced them from. if len(signedAddrMap) == 0 { - delete(s.signedRoutingStates, p) + delete(s.signedPeerRecords, p) } } } @@ -378,13 +386,22 @@ func validAddrs(amap map[string]*expiringAddr) []ma.Multiaddr { return good } -// SignedRoutingState returns a SignedRoutingState record for the -// given peer id, if one exists in the peerstore. -func (mab *memoryAddrBook) SignedRoutingState(p peer.ID) *routing.SignedRoutingState { +// SignedPeerRecord returns a SignedEnvelope containing a PeerRecord for the +// given peer id, if one exists. +// Returns nil if no signed PeerRecord exists for the peer. +func (mab *memoryAddrBook) SignedPeerRecord(p peer.ID) *record.SignedEnvelope { s := mab.segments.get(p) s.RLock() defer s.RUnlock() - return s.signedRoutingStates[p] + + // although the signed record gets garbage collected when all addrs inside it are expired, + // we may be in between the expiration time and the GC interval + // so, we check to see if we have any valid signed addrs before returning the record + if len(validAddrs(s.signedAddrs[p])) == 0 { + return nil + } + + return s.signedPeerRecords[p] } // ClearAddrs removes all previously stored addresses @@ -395,7 +412,7 @@ func (mab *memoryAddrBook) ClearAddrs(p peer.ID) { delete(s.addrs, p) delete(s.signedAddrs, p) - delete(s.signedRoutingStates, p) + delete(s.signedPeerRecords, p) } // AddrStream returns a channel on which all new addresses discovered for a diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index 8ea3ff5..572716e 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -3,7 +3,6 @@ package test import ( "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/routing" "github.com/libp2p/go-libp2p-core/test" "github.com/multiformats/go-multiaddr" "testing" @@ -375,7 +374,7 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { allAddrs := GenerateAddrs(10) certifiedAddrs := allAddrs[:5] uncertifiedAddrs := allAddrs[5:] - signedState, err := routing.MakeSignedRoutingState(priv, certifiedAddrs) + signedRec, err := peer.NewPeerRecord(id, certifiedAddrs).Sign(priv) if err != nil { t.Errorf("error creating signed routing record: %v", err) } @@ -387,7 +386,7 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { AssertAddressesEqual(t, uncertifiedAddrs, m.Addrs(id)) // add the signed record to addr book - err = cab.AddCertifiedAddrs(signedState, time.Hour) + err = cab.AddCertifiedAddrs(signedRec, time.Hour) if err != nil { t.Errorf("error adding signed routing record to addrbook: %v", err) } @@ -404,18 +403,18 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { m.AddAddrs(id, uncertifiedAddrs, time.Hour) AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) - // we should be able to retrieve the signed state - state2 := cab.SignedRoutingState(id) - if state2 == nil || !signedState.Equal(state2) { + // we should be able to retrieve the signed peer record + rec2 := cab.SignedPeerRecord(id) + if rec2 == nil || !signedRec.Equal(rec2) { t.Error("unable to retrieve signed routing record from addrbook") } // Adding a new envelope should clear existing certified addresses. // Only the newly-added ones should remain certifiedAddrs = certifiedAddrs[:3] - signedState, err = routing.MakeSignedRoutingState(priv, certifiedAddrs) + signedRec, err = peer.NewPeerRecord(id, certifiedAddrs).Sign(priv) test.AssertNilError(t, err) - err = cab.AddCertifiedAddrs(signedState, time.Hour) + err = cab.AddCertifiedAddrs(signedRec, time.Hour) test.AssertNilError(t, err) AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) @@ -425,8 +424,18 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { if len(m.Addrs(id)) != 0 { t.Error("expected zero certified addrs after setting TTL to -1") } - if cab.SignedRoutingState(id) != nil { - t.Error("expected signed routing record to be removed when addresses expire") + if cab.SignedPeerRecord(id) != nil { + t.Error("expected signed peer record to be removed when addresses expire") + } + + // Test that natural TTL expiration clears signed peer records + err = cab.AddCertifiedAddrs(signedRec, time.Second) + test.AssertNilError(t, err) + AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) + + time.Sleep(2 * time.Second) + if cab.SignedPeerRecord(id) != nil { + t.Error("expected signed peer record to be removed when addresses expire") } } } From 57a7872ade1d06216058119fafee276ac08206ef Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 10 Jan 2020 12:18:24 -0500 Subject: [PATCH 27/39] bump -core dep to new PR commit --- go.mod | 11 ++++++++--- go.sum | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 949a746..436756a 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,20 @@ require ( github.com/ipfs/go-ds-leveldb v0.1.0 github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-buffer-pool v0.0.2 - github.com/libp2p/go-libp2p-core v0.2.6-0.20191121175514-5fa975301271 + github.com/libp2p/go-libp2p-core v0.3.1-0.20200107184944-460979ea35ac github.com/libp2p/go-msgio v0.0.4 // indirect + github.com/mailru/easyjson v0.7.0 // indirect github.com/multiformats/go-base32 v0.0.3 - github.com/multiformats/go-multiaddr v0.1.1 + github.com/multiformats/go-multiaddr v0.2.0 github.com/multiformats/go-multiaddr-fmt v0.1.0 github.com/multiformats/go-multiaddr-net v0.1.1 - github.com/multiformats/go-multihash v0.0.8 + github.com/multiformats/go-multihash v0.0.10 + github.com/multiformats/go-varint v0.0.2 // indirect github.com/pkg/errors v0.8.1 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 + golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 // indirect + golang.org/x/sys v0.0.0-20200107162124-548cf772de50 // indirect + golang.org/x/tools v0.0.0-20200110142700-428f1ab0ca03 // indirect ) go 1.13 diff --git a/go.sum b/go.sum index 271ef15..a930b70 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= @@ -62,6 +64,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1 h1:F4k0TkTAZGLFzBOrVKDAvch6JZtuN4NHkfdcEZL50aI= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= @@ -105,12 +109,18 @@ github.com/libp2p/go-libp2p-core v0.2.6-0.20191119151917-d86de054a1fd h1:TIcqg9I github.com/libp2p/go-libp2p-core v0.2.6-0.20191119151917-d86de054a1fd/go.mod h1:xDyprN8hkMpX27XQ1bBnYtuSuaCUFvKm+Q6gltHZCHE= github.com/libp2p/go-libp2p-core v0.2.6-0.20191121175514-5fa975301271 h1:4zW1wrEii7L5t+e9AF8zyLSc+mibnmfBcfMWA6ItyXk= github.com/libp2p/go-libp2p-core v0.2.6-0.20191121175514-5fa975301271/go.mod h1:xDyprN8hkMpX27XQ1bBnYtuSuaCUFvKm+Q6gltHZCHE= +github.com/libp2p/go-libp2p-core v0.3.1-0.20200107184944-460979ea35ac h1:cxNslmoGCKMUPegFZWKrHnv49/Cy1InSf52nlASoMqs= +github.com/libp2p/go-libp2p-core v0.3.1-0.20200107184944-460979ea35ac/go.mod h1:KJbT3ekTPccG9QUSa5vc/0QVZXPE9+3TVlyqESRQic0= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-openssl v0.0.3 h1:wjlG7HvQkt4Fq4cfH33Ivpwp0omaElYEi9z26qaIkIk= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= +github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -130,11 +140,15 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.1.1 h1:rVAztJYMhCQ7vEFr8FvxW3mS+HF2eY/oPbOMeS0ZDnE= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= +github.com/multiformats/go-multiaddr v0.2.0 h1:lR52sFwcTCuQb6bTfnXF6zA2XfyYvyd+5a9qECv/J90= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.1.1 h1:jFFKUuXTXv+3ARyHZi3XUqQO+YWMKgBdhEvuGRfnL6s= @@ -145,6 +159,12 @@ github.com/multiformats/go-multihash v0.0.1 h1:HHwN1K12I+XllBCrqKnhX949Orn4oawPk github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.8 h1:wrYcW5yxSi3dU07n5jnuS5PrNwyHy0zRHGVoUugWvXg= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-varint v0.0.1 h1:TR/0rdQtnNxuN2IhiB639xC3tWM4IUi7DkTBVTdGW/M= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.2 h1:6sUvyh2YHpJCb8RZ6eYzj6iJQ4+chWYmyIHxszqlPTA= +github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -177,6 +197,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -187,18 +208,23 @@ github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1 github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 h1:nVJ3guKA9qdkEQ3TUdXI9QSINo2CUPM/cySEvw2w8I0= +golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -215,6 +241,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6Zh golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -228,6 +255,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50 h1:YvQ10rzcqWXLlJZ3XCUoO25savxmscf4+SC+ZqiCHhA= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -240,7 +269,10 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20200110142700-428f1ab0ca03 h1:iPq9R0Zx+m39Vtm+v/7imzPGK2FOWXJFeIe1EBvHew4= +golang.org/x/tools v0.0.0-20200110142700-428f1ab0ca03/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= From 32e09dbf67befa12e43c400990490e0cbff12636 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Fri, 10 Jan 2020 12:28:52 -0500 Subject: [PATCH 28/39] use go 1.13.x on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5163d69..a156d3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ os: language: go go: - - 1.11.x + - 1.13.x env: global: From 084bc23850efde3b60aa1cc3a0bdef2c7b3a14e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 15 Jan 2020 17:50:53 +0000 Subject: [PATCH 29/39] add a TODO. --- pstoreds/addr_book.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 515e1ea..65ec4b6 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -401,6 +401,7 @@ func (ab *dsAddrBook) Addrs(p peer.ID) []ma.Multiaddr { pr.RLock() defer pr.RUnlock() + // TODO: presize this slice. var addrs []ma.Multiaddr for _, a := range pr.SignedAddrs { addrs = append(addrs, a.Addr) From 2e7d7b9be5345c9dac86f30f390540d3c6fa8450 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 21 Jan 2020 11:26:02 -0500 Subject: [PATCH 30/39] pre-allocate addr slice --- pstoreds/addr_book.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 65ec4b6..96cd65f 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -401,8 +401,11 @@ func (ab *dsAddrBook) Addrs(p peer.ID) []ma.Multiaddr { pr.RLock() defer pr.RUnlock() - // TODO: presize this slice. - var addrs []ma.Multiaddr + size := len(pr.SignedAddrs) + if len(pr.Addrs) > size { + size = len(pr.Addrs) + } + addrs := make([]ma.Multiaddr, size) for _, a := range pr.SignedAddrs { addrs = append(addrs, a.Addr) } From 1f4670bb6c343f2a545d552dc2f7689c7007b81a Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 21 Jan 2020 11:22:23 -0500 Subject: [PATCH 31/39] track changes in -core PR --- go.mod | 8 ++++---- go.sum | 8 ++++++++ pstoreds/addr_book.go | 28 ++++++++++++++++----------- pstoremem/addr_book.go | 43 +++++++++++++++++++++++++++++------------ test/addr_book_suite.go | 22 +++++++++++++-------- 5 files changed, 74 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index 436756a..7e650b5 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ipfs/go-ds-leveldb v0.1.0 github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-buffer-pool v0.0.2 - github.com/libp2p/go-libp2p-core v0.3.1-0.20200107184944-460979ea35ac + github.com/libp2p/go-libp2p-core v0.3.1-0.20200121152518-5a523a102629 github.com/libp2p/go-msgio v0.0.4 // indirect github.com/mailru/easyjson v0.7.0 // indirect github.com/multiformats/go-base32 v0.0.3 @@ -19,9 +19,9 @@ require ( github.com/multiformats/go-varint v0.0.2 // indirect github.com/pkg/errors v0.8.1 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 - golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 // indirect - golang.org/x/sys v0.0.0-20200107162124-548cf772de50 // indirect - golang.org/x/tools v0.0.0-20200110142700-428f1ab0ca03 // indirect + golang.org/x/crypto v0.0.0-20200117160349-530e935923ad // indirect + golang.org/x/sys v0.0.0-20200121082415-34d275377bf9 // indirect + golang.org/x/tools v0.0.0-20200121145236-13c74800b9d2 // indirect ) go 1.13 diff --git a/go.sum b/go.sum index a930b70..a40982e 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,8 @@ github.com/libp2p/go-libp2p-core v0.2.6-0.20191121175514-5fa975301271 h1:4zW1wrE github.com/libp2p/go-libp2p-core v0.2.6-0.20191121175514-5fa975301271/go.mod h1:xDyprN8hkMpX27XQ1bBnYtuSuaCUFvKm+Q6gltHZCHE= github.com/libp2p/go-libp2p-core v0.3.1-0.20200107184944-460979ea35ac h1:cxNslmoGCKMUPegFZWKrHnv49/Cy1InSf52nlASoMqs= github.com/libp2p/go-libp2p-core v0.3.1-0.20200107184944-460979ea35ac/go.mod h1:KJbT3ekTPccG9QUSa5vc/0QVZXPE9+3TVlyqESRQic0= +github.com/libp2p/go-libp2p-core v0.3.1-0.20200121152518-5a523a102629 h1:U0mnCQcq57Y67jSgC4DtNpYcyFen91pWnxT4dsZ2M1Q= +github.com/libp2p/go-libp2p-core v0.3.1-0.20200121152518-5a523a102629/go.mod h1:KJbT3ekTPccG9QUSa5vc/0QVZXPE9+3TVlyqESRQic0= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-openssl v0.0.3 h1:wjlG7HvQkt4Fq4cfH33Ivpwp0omaElYEi9z26qaIkIk= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= @@ -220,6 +222,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 h1:nVJ3guKA9qdkEQ3TUdXI9QSINo2CUPM/cySEvw2w8I0= golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad h1:Jh8cai0fqIK+f6nG0UgPW5wFk8wmiMhM3AyciDBdtQg= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -257,6 +261,8 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSF golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50 h1:YvQ10rzcqWXLlJZ3XCUoO25savxmscf4+SC+ZqiCHhA= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200121082415-34d275377bf9 h1:N19i1HjUnR7TF7rMt8O4p3dLvqvmYyzB6ifMFmrbY50= +golang.org/x/sys v0.0.0-20200121082415-34d275377bf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -271,6 +277,8 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20200110142700-428f1ab0ca03 h1:iPq9R0Zx+m39Vtm+v/7imzPGK2FOWXJFeIe1EBvHew4= golang.org/x/tools v0.0.0-20200110142700-428f1ab0ca03/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200121145236-13c74800b9d2 h1:pdlZPiMD2o4d/4PpHsPwGv7VwPb9UxXWORqFHS7X3bQ= +golang.org/x/tools v0.0.0-20200121145236-13c74800b9d2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 96cd65f..576127f 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -277,15 +277,21 @@ func (ab *dsAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio ab.setAddrs(p, addrs, ttl, ttlExtend, false) } -// AddCertifiedAddrs adds addresses from a routing.RoutingState record -// contained in the given SignedEnvelope. -func (ab *dsAddrBook) AddCertifiedAddrs(recordEnvelope *record.SignedEnvelope, ttl time.Duration) error { - rec, err := peer.PeerRecordFromSignedEnvelope(recordEnvelope) +// ProcessPeerRecord adds addresses from a signed peer.PeerRecord (contained in +// a routing.Envelope), which will expire after the given TTL. +// See https://godoc.org/github.com/libp2p/go-libp2p-core/peerstore#CertifiedAddrBook for more details. +func (ab *dsAddrBook) ProcessPeerRecord(recordEnvelope *record.Envelope, ttl time.Duration) error { + r, err := recordEnvelope.Record() if err != nil { return err } + rec, ok := r.(*peer.PeerRecord) + if !ok { + return fmt.Errorf("envelope did not contain PeerRecord") + } + // ensure that the seq number from envelope is >= any previously received seq no - if ab.latestPeerRecordSeq(rec.PeerID) >= recordEnvelope.Seq { + if ab.latestPeerRecordSeq(rec.PeerID) >= rec.Seq { return nil } @@ -295,7 +301,7 @@ func (ab *dsAddrBook) AddCertifiedAddrs(recordEnvelope *record.SignedEnvelope, t return err } - return ab.storeSignedPeerRecord(rec.PeerID, recordEnvelope) + return ab.storeSignedPeerRecord(rec.PeerID, recordEnvelope, rec) } func (ab *dsAddrBook) latestPeerRecordSeq(p peer.ID) uint64 { @@ -306,7 +312,7 @@ func (ab *dsAddrBook) latestPeerRecordSeq(p peer.ID) uint64 { return pr.CertifiedRecord.Seq } -func (ab *dsAddrBook) storeSignedPeerRecord(p peer.ID, envelope *record.SignedEnvelope) error { +func (ab *dsAddrBook) storeSignedPeerRecord(p peer.ID, envelope *record.Envelope, rec *peer.PeerRecord) error { envelopeBytes, err := envelope.Marshal() if err != nil { return err @@ -320,7 +326,7 @@ func (ab *dsAddrBook) storeSignedPeerRecord(p peer.ID, envelope *record.SignedEn return err } pr.CertifiedRecord = &pb.AddrBookRecord_CertifiedRecord{ - Seq: envelope.Seq, + Seq: rec.Seq, Raw: envelopeBytes, } pr.dirty = true @@ -328,10 +334,10 @@ func (ab *dsAddrBook) storeSignedPeerRecord(p peer.ID, envelope *record.SignedEn return err } -// SignedPeerRecord returns a SignedEnvelope containing a PeerRecord for the +// GetPeerRecord returns a record.Envelope containing a peer.PeerRecord for the // given peer id, if one exists. // Returns nil if no signed PeerRecord exists for the peer. -func (ab *dsAddrBook) SignedPeerRecord(p peer.ID) *record.SignedEnvelope { +func (ab *dsAddrBook) GetPeerRecord(p peer.ID) *record.Envelope { pr, err := ab.loadRecord(p, true, false) if err != nil { log.Errorf("unable to load record for peer %s: %v", p.Pretty(), err) @@ -340,7 +346,7 @@ func (ab *dsAddrBook) SignedPeerRecord(p peer.ID) *record.SignedEnvelope { if pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { return nil } - state, err := record.ConsumeEnvelope(pr.CertifiedRecord.Raw, peer.PeerRecordEnvelopeDomain) + state, _, err := record.ConsumeEnvelope(pr.CertifiedRecord.Raw, peer.PeerRecordEnvelopeDomain) if err != nil { log.Errorf("error unmarshaling stored signed peer record for peer %s: %v", p.Pretty(), err) return nil diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 9c575b7..0296b48 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -2,16 +2,17 @@ package pstoremem import ( "context" - "github.com/libp2p/go-libp2p-core/record" + "fmt" "sort" "sync" "time" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/peer" + pstore "github.com/libp2p/go-libp2p-core/peerstore" + "github.com/libp2p/go-libp2p-core/record" ma "github.com/multiformats/go-multiaddr" - pstore "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-peerstore/addr" ) @@ -27,6 +28,11 @@ func (e *expiringAddr) ExpiredBy(t time.Time) bool { return t.After(e.Expires) } +type peerRecordState struct { + Envelope *record.Envelope + Seq uint64 +} + type addrSegments [256]*addrSegment type addrSegment struct { @@ -39,7 +45,7 @@ type addrSegment struct { signedAddrs map[peer.ID]map[string]*expiringAddr - signedPeerRecords map[peer.ID]*record.SignedEnvelope + signedPeerRecords map[peer.ID]*peerRecordState } func (segments *addrSegments) get(p peer.ID) *addrSegment { @@ -68,7 +74,7 @@ func NewAddrBook() *memoryAddrBook { ret[i] = &addrSegment{ addrs: make(map[peer.ID]map[string]*expiringAddr), signedAddrs: make(map[peer.ID]map[string]*expiringAddr), - signedPeerRecords: make(map[peer.ID]*record.SignedEnvelope)} + signedPeerRecords: make(map[peer.ID]*peerRecordState)} } return ret }(), @@ -164,21 +170,30 @@ func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du mab.addAddrs(p, addrs, ttl, false) } -// AddCertifiedAddrs adds addresses from a signed peer.PeerRecord contained in a SignedEnvelope. -func (mab *memoryAddrBook) AddCertifiedAddrs(recordEnvelope *record.SignedEnvelope, ttl time.Duration) error { - rec, err := peer.PeerRecordFromSignedEnvelope(recordEnvelope) +// ProcessPeerRecord adds addresses from a signed peer.PeerRecord (contained in +// a routing.Envelope), which will expire after the given TTL. +// See https://godoc.org/github.com/libp2p/go-libp2p-core/peerstore#CertifiedAddrBook for more details. +func (mab *memoryAddrBook) ProcessPeerRecord(recordEnvelope *record.Envelope, ttl time.Duration) error { + r, err := recordEnvelope.Record() if err != nil { return err } + rec, ok := r.(*peer.PeerRecord) + if !ok { + return fmt.Errorf("unable to process envelope: not a PeerRecord") + } // ensure seq is greater than last received s := mab.segments.get(rec.PeerID) s.Lock() lastState, found := s.signedPeerRecords[rec.PeerID] - if found && lastState.Seq >= recordEnvelope.Seq { + if found && lastState.Seq >= rec.Seq { s.Unlock() return nil } - s.signedPeerRecords[rec.PeerID] = recordEnvelope + s.signedPeerRecords[rec.PeerID] = &peerRecordState{ + Envelope: recordEnvelope, + Seq: rec.Seq, + } s.Unlock() // need to release the lock, since addAddrs will try to take it mab.addAddrs(rec.PeerID, rec.Addrs, ttl, true) return nil @@ -386,10 +401,10 @@ func validAddrs(amap map[string]*expiringAddr) []ma.Multiaddr { return good } -// SignedPeerRecord returns a SignedEnvelope containing a PeerRecord for the +// GetPeerRecord returns a Envelope containing a PeerRecord for the // given peer id, if one exists. // Returns nil if no signed PeerRecord exists for the peer. -func (mab *memoryAddrBook) SignedPeerRecord(p peer.ID) *record.SignedEnvelope { +func (mab *memoryAddrBook) GetPeerRecord(p peer.ID) *record.Envelope { s := mab.segments.get(p) s.RLock() defer s.RUnlock() @@ -401,7 +416,11 @@ func (mab *memoryAddrBook) SignedPeerRecord(p peer.ID) *record.SignedEnvelope { return nil } - return s.signedPeerRecords[p] + state := s.signedPeerRecords[p] + if state == nil { + return nil + } + return state.Envelope } // ClearAddrs removes all previously stored addresses diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index 572716e..b59a0d1 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -374,7 +374,10 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { allAddrs := GenerateAddrs(10) certifiedAddrs := allAddrs[:5] uncertifiedAddrs := allAddrs[5:] - signedRec, err := peer.NewPeerRecord(id, certifiedAddrs).Sign(priv) + rec := peer.NewPeerRecord() + rec.PeerID = id + rec.Addrs = certifiedAddrs + signedRec, err := rec.Sign(priv) if err != nil { t.Errorf("error creating signed routing record: %v", err) } @@ -386,7 +389,7 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { AssertAddressesEqual(t, uncertifiedAddrs, m.Addrs(id)) // add the signed record to addr book - err = cab.AddCertifiedAddrs(signedRec, time.Hour) + err = cab.ProcessPeerRecord(signedRec, time.Hour) if err != nil { t.Errorf("error adding signed routing record to addrbook: %v", err) } @@ -404,7 +407,7 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) // we should be able to retrieve the signed peer record - rec2 := cab.SignedPeerRecord(id) + rec2 := cab.GetPeerRecord(id) if rec2 == nil || !signedRec.Equal(rec2) { t.Error("unable to retrieve signed routing record from addrbook") } @@ -412,9 +415,12 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { // Adding a new envelope should clear existing certified addresses. // Only the newly-added ones should remain certifiedAddrs = certifiedAddrs[:3] - signedRec, err = peer.NewPeerRecord(id, certifiedAddrs).Sign(priv) + rec = peer.NewPeerRecord() + rec.PeerID = id + rec.Addrs = certifiedAddrs + signedRec, err = rec.Sign(priv) test.AssertNilError(t, err) - err = cab.AddCertifiedAddrs(signedRec, time.Hour) + err = cab.ProcessPeerRecord(signedRec, time.Hour) test.AssertNilError(t, err) AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) @@ -424,17 +430,17 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { if len(m.Addrs(id)) != 0 { t.Error("expected zero certified addrs after setting TTL to -1") } - if cab.SignedPeerRecord(id) != nil { + if cab.GetPeerRecord(id) != nil { t.Error("expected signed peer record to be removed when addresses expire") } // Test that natural TTL expiration clears signed peer records - err = cab.AddCertifiedAddrs(signedRec, time.Second) + err = cab.ProcessPeerRecord(signedRec, time.Second) test.AssertNilError(t, err) AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) time.Sleep(2 * time.Second) - if cab.SignedPeerRecord(id) != nil { + if cab.GetPeerRecord(id) != nil { t.Error("expected signed peer record to be removed when addresses expire") } } From 7a4e88340104a6274f119da0f391d5279cdc3197 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Wed, 22 Jan 2020 14:43:43 -0500 Subject: [PATCH 32/39] fix slice allocation --- pstoreds/addr_book.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 576127f..4170180 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -411,7 +411,7 @@ func (ab *dsAddrBook) Addrs(p peer.ID) []ma.Multiaddr { if len(pr.Addrs) > size { size = len(pr.Addrs) } - addrs := make([]ma.Multiaddr, size) + addrs := make([]ma.Multiaddr, 0, size) for _, a := range pr.SignedAddrs { addrs = append(addrs, a.Addr) } From 1bda3bad742fc3fc176b39616865aa7f8bbb771a Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 4 Feb 2020 14:48:06 -0500 Subject: [PATCH 33/39] simplify addr book implementation since signed and unsigned addrs are now mutually exclusive, there's no benefit to keeping two lists of addresses. --- pb/pstore.pb.go | 132 ++++++++------------------------- pb/pstore.proto | 2 - pstoreds/addr_book.go | 96 +++++++----------------- pstoremem/addr_book.go | 161 +++++++++++++---------------------------- 4 files changed, 109 insertions(+), 282 deletions(-) diff --git a/pb/pstore.pb.go b/pb/pstore.pb.go index aa4e253..7e18cb3 100644 --- a/pb/pstore.pb.go +++ b/pb/pstore.pb.go @@ -28,8 +28,7 @@ type AddrBookRecord struct { // The peer ID. Id *ProtoPeerID `protobuf:"bytes,1,opt,name=id,proto3,customtype=ProtoPeerID" json:"id,omitempty"` // The multiaddresses. This is a sorted list where element 0 expires the soonest. - Addrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,2,rep,name=addrs,proto3" json:"addrs,omitempty"` - SignedAddrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,3,rep,name=signedAddrs,proto3" json:"signedAddrs,omitempty"` + Addrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,2,rep,name=addrs,proto3" json:"addrs,omitempty"` // The most recently received signed PeerRecord. CertifiedRecord *AddrBookRecord_CertifiedRecord `protobuf:"bytes,4,opt,name=certifiedRecord,proto3" json:"certifiedRecord,omitempty"` } @@ -74,13 +73,6 @@ func (m *AddrBookRecord) GetAddrs() []*AddrBookRecord_AddrEntry { return nil } -func (m *AddrBookRecord) GetSignedAddrs() []*AddrBookRecord_AddrEntry { - if m != nil { - return m.SignedAddrs - } - return nil -} - func (m *AddrBookRecord) GetCertifiedRecord() *AddrBookRecord_CertifiedRecord { if m != nil { return m.CertifiedRecord @@ -209,28 +201,27 @@ func init() { func init() { proto.RegisterFile("pstore.proto", fileDescriptor_f96873690e08a98f) } var fileDescriptor_f96873690e08a98f = []byte{ - // 331 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x91, 0xb1, 0x4f, 0xc2, 0x40, - 0x14, 0xc6, 0xb9, 0x16, 0x49, 0xb8, 0xa2, 0x98, 0x1b, 0x4c, 0xc3, 0x70, 0x54, 0x5d, 0xea, 0x60, - 0x49, 0x30, 0x0e, 0x8e, 0xa0, 0x0c, 0x6e, 0xe4, 0x5c, 0x5c, 0x29, 0x77, 0xd4, 0x8b, 0xca, 0xe1, - 0xf5, 0x88, 0xf2, 0x5f, 0xb8, 0xfa, 0xdf, 0x38, 0x3a, 0x32, 0x1a, 0x06, 0xa2, 0xe5, 0x9f, 0x70, - 0x34, 0xf7, 0x8a, 0x44, 0x48, 0x4c, 0xdc, 0xbe, 0xef, 0xf5, 0xfb, 0x7e, 0xaf, 0x79, 0x87, 0x2b, - 0xa3, 0xd4, 0x28, 0x2d, 0xa2, 0x91, 0x56, 0x46, 0x91, 0xf2, 0x8f, 0x8b, 0x6b, 0xc7, 0x89, 0x34, - 0x37, 0xe3, 0x38, 0xea, 0xab, 0xfb, 0x46, 0xa2, 0x12, 0xd5, 0x80, 0x44, 0x3c, 0x1e, 0x80, 0x03, - 0x03, 0x2a, 0x6f, 0x1e, 0xbc, 0xb8, 0x78, 0xa7, 0xc5, 0xb9, 0x6e, 0x2b, 0x75, 0xcb, 0x44, 0x5f, - 0x69, 0x4e, 0xea, 0xd8, 0x91, 0xdc, 0x47, 0x01, 0x0a, 0x2b, 0xed, 0xea, 0x6c, 0x5e, 0xf7, 0xba, - 0x36, 0xd9, 0x15, 0x42, 0x5f, 0x5e, 0x30, 0x47, 0x72, 0x72, 0x86, 0xb7, 0x7a, 0x9c, 0xeb, 0xd4, - 0x77, 0x02, 0x37, 0xf4, 0x9a, 0x87, 0xd1, 0x6a, 0x7b, 0xb4, 0x8e, 0x02, 0xdb, 0x19, 0x1a, 0x3d, - 0x61, 0x79, 0x83, 0x74, 0xb0, 0x97, 0xca, 0x64, 0x28, 0x78, 0x0b, 0x00, 0xee, 0xff, 0x01, 0xbf, - 0x7b, 0xe4, 0x0a, 0x57, 0xfb, 0x42, 0x1b, 0x39, 0x90, 0x82, 0xe7, 0x49, 0xbf, 0x18, 0xa0, 0xd0, - 0x6b, 0x1e, 0xfd, 0x8d, 0x3a, 0x5f, 0x2f, 0xb0, 0x4d, 0x42, 0xed, 0x1a, 0x97, 0x57, 0xeb, 0xc8, - 0x3e, 0x2e, 0xda, 0x3f, 0x5e, 0x9e, 0x61, 0x7b, 0x36, 0xaf, 0x97, 0xe1, 0x0c, 0x36, 0xc1, 0xe0, - 0x13, 0xd9, 0xc3, 0x25, 0xf1, 0x34, 0x92, 0x7a, 0xe2, 0x3b, 0x01, 0x0a, 0x5d, 0xb6, 0x74, 0x64, - 0x17, 0xbb, 0xc6, 0xdc, 0xf9, 0x2e, 0x0c, 0xad, 0xac, 0x9d, 0xe2, 0xea, 0xc6, 0x76, 0x1b, 0x4a, - 0xc5, 0x03, 0xe0, 0x8b, 0xcc, 0x4a, 0x3b, 0xd1, 0xbd, 0x47, 0x60, 0x55, 0x98, 0x95, 0xed, 0xe0, - 0xeb, 0x93, 0xa2, 0xd7, 0x8c, 0xa2, 0xb7, 0x8c, 0xa2, 0x69, 0x46, 0xd1, 0x47, 0x46, 0xd1, 0xf3, - 0x82, 0x16, 0xa6, 0x0b, 0x5a, 0x78, 0x5f, 0xd0, 0x42, 0x5c, 0x82, 0x47, 0x3c, 0xf9, 0x0e, 0x00, - 0x00, 0xff, 0xff, 0x26, 0xc5, 0x77, 0x15, 0x0e, 0x02, 0x00, 0x00, + // 315 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x90, 0xbf, 0x4e, 0xf3, 0x30, + 0x14, 0xc5, 0xeb, 0xa4, 0x5f, 0xa5, 0xba, 0xf9, 0x08, 0xf2, 0x80, 0xa2, 0x0c, 0x4e, 0x80, 0x25, + 0x0c, 0xa4, 0x52, 0x11, 0x03, 0x23, 0x01, 0x06, 0xb6, 0xca, 0x2c, 0xac, 0x4d, 0xec, 0x06, 0x8b, + 0x3f, 0x0e, 0x8e, 0x2b, 0xe8, 0x5b, 0xf0, 0x48, 0x8c, 0x8c, 0x15, 0x13, 0xea, 0x50, 0x41, 0xf2, + 0x12, 0x8c, 0xc8, 0x4e, 0xa9, 0xd4, 0x4a, 0x6c, 0xe7, 0x5c, 0x9f, 0xfb, 0x3b, 0x57, 0x86, 0x4e, + 0x51, 0x2a, 0x21, 0x59, 0x5c, 0x48, 0xa1, 0x04, 0xea, 0xfe, 0xba, 0xd4, 0x3f, 0xcc, 0xb9, 0xba, + 0x99, 0xa4, 0x71, 0x26, 0xee, 0xfb, 0xb9, 0xc8, 0x45, 0xdf, 0x24, 0xd2, 0xc9, 0xd8, 0x38, 0x63, + 0x8c, 0x6a, 0x36, 0xf7, 0xde, 0x2d, 0xb8, 0x75, 0x4a, 0xa9, 0x4c, 0x84, 0xb8, 0x25, 0x2c, 0x13, + 0x92, 0xa2, 0x00, 0x5a, 0x9c, 0x7a, 0x20, 0x04, 0x91, 0x93, 0xb8, 0xf3, 0x45, 0xd0, 0x1b, 0xea, + 0xe4, 0x90, 0x31, 0x79, 0x79, 0x4e, 0x2c, 0x4e, 0xd1, 0x09, 0xfc, 0x37, 0xa2, 0x54, 0x96, 0x9e, + 0x15, 0xda, 0x51, 0x6f, 0xb0, 0x1f, 0xaf, 0xda, 0xe3, 0x75, 0x94, 0xb1, 0x17, 0x0f, 0x4a, 0x4e, + 0x49, 0xb3, 0x81, 0xae, 0xa0, 0x9b, 0x31, 0xa9, 0xf8, 0x98, 0x33, 0xda, 0x64, 0xbc, 0x76, 0x08, + 0xa2, 0xde, 0xe0, 0xe0, 0x6f, 0xc8, 0xd9, 0xfa, 0x02, 0xd9, 0x24, 0xf8, 0xd7, 0xb0, 0xbb, 0x2a, + 0x42, 0xbb, 0xb0, 0xad, 0xab, 0x96, 0xf7, 0xff, 0x9f, 0x2f, 0x82, 0xae, 0xb9, 0x5f, 0x27, 0x88, + 0x79, 0x42, 0x3b, 0xb0, 0xc3, 0x9e, 0x0b, 0x2e, 0xa7, 0x9e, 0x15, 0x82, 0xc8, 0x26, 0x4b, 0x87, + 0xb6, 0xa1, 0xad, 0xd4, 0x9d, 0x67, 0x9b, 0xa1, 0x96, 0xfe, 0x31, 0x74, 0x37, 0xda, 0x75, 0xa8, + 0x64, 0x8f, 0x06, 0xdf, 0x26, 0x5a, 0xea, 0x89, 0x1c, 0x3d, 0x19, 0x96, 0x43, 0xb4, 0x4c, 0xc2, + 0xef, 0x2f, 0x0c, 0x5e, 0x2b, 0x0c, 0xde, 0x2a, 0x0c, 0x66, 0x15, 0x06, 0x9f, 0x15, 0x06, 0x2f, + 0x35, 0x6e, 0xcd, 0x6a, 0xdc, 0xfa, 0xa8, 0x71, 0x2b, 0xed, 0x98, 0xdf, 0x3f, 0xfa, 0x09, 0x00, + 0x00, 0xff, 0xff, 0x1e, 0x8d, 0xf1, 0x8a, 0xc7, 0x01, 0x00, 0x00, } func (m *AddrBookRecord) Marshal() (dAtA []byte, err error) { @@ -265,20 +256,6 @@ func (m *AddrBookRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x22 } - if len(m.SignedAddrs) > 0 { - for iNdEx := len(m.SignedAddrs) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.SignedAddrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPstore(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a - } - } if len(m.Addrs) > 0 { for iNdEx := len(m.Addrs) - 1; iNdEx >= 0; iNdEx-- { { @@ -409,13 +386,6 @@ func NewPopulatedAddrBookRecord(r randyPstore, easy bool) *AddrBookRecord { this.Addrs[i] = NewPopulatedAddrBookRecord_AddrEntry(r, easy) } } - if r.Intn(5) != 0 { - v2 := r.Intn(5) - this.SignedAddrs = make([]*AddrBookRecord_AddrEntry, v2) - for i := 0; i < v2; i++ { - this.SignedAddrs[i] = NewPopulatedAddrBookRecord_AddrEntry(r, easy) - } - } if r.Intn(5) != 0 { this.CertifiedRecord = NewPopulatedAddrBookRecord_CertifiedRecord(r, easy) } @@ -443,9 +413,9 @@ func NewPopulatedAddrBookRecord_AddrEntry(r randyPstore, easy bool) *AddrBookRec func NewPopulatedAddrBookRecord_CertifiedRecord(r randyPstore, easy bool) *AddrBookRecord_CertifiedRecord { this := &AddrBookRecord_CertifiedRecord{} this.Seq = uint64(uint64(r.Uint32())) - v3 := r.Intn(100) - this.Raw = make([]byte, v3) - for i := 0; i < v3; i++ { + v2 := r.Intn(100) + this.Raw = make([]byte, v2) + for i := 0; i < v2; i++ { this.Raw[i] = byte(r.Intn(256)) } if !easy && r.Intn(10) != 0 { @@ -472,9 +442,9 @@ func randUTF8RunePstore(r randyPstore) rune { return rune(ru + 61) } func randStringPstore(r randyPstore) string { - v4 := r.Intn(100) - tmps := make([]rune, v4) - for i := 0; i < v4; i++ { + v3 := r.Intn(100) + tmps := make([]rune, v3) + for i := 0; i < v3; i++ { tmps[i] = randUTF8RunePstore(r) } return string(tmps) @@ -496,11 +466,11 @@ func randFieldPstore(dAtA []byte, r randyPstore, fieldNumber int, wire int) []by switch wire { case 0: dAtA = encodeVarintPopulatePstore(dAtA, uint64(key)) - v5 := r.Int63() + v4 := r.Int63() if r.Intn(2) == 0 { - v5 *= -1 + v4 *= -1 } - dAtA = encodeVarintPopulatePstore(dAtA, uint64(v5)) + dAtA = encodeVarintPopulatePstore(dAtA, uint64(v4)) case 1: dAtA = encodeVarintPopulatePstore(dAtA, uint64(key)) dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) @@ -541,12 +511,6 @@ func (m *AddrBookRecord) Size() (n int) { n += 1 + l + sovPstore(uint64(l)) } } - if len(m.SignedAddrs) > 0 { - for _, e := range m.SignedAddrs { - l = e.Size() - n += 1 + l + sovPstore(uint64(l)) - } - } if m.CertifiedRecord != nil { l = m.CertifiedRecord.Size() n += 1 + l + sovPstore(uint64(l)) @@ -693,40 +657,6 @@ func (m *AddrBookRecord) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SignedAddrs", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPstore - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPstore - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPstore - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.SignedAddrs = append(m.SignedAddrs, &AddrBookRecord_AddrEntry{}) - if err := m.SignedAddrs[len(m.SignedAddrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field CertifiedRecord", wireType) diff --git a/pb/pstore.proto b/pb/pstore.proto index 2a683d7..f0a5b53 100644 --- a/pb/pstore.proto +++ b/pb/pstore.proto @@ -14,8 +14,6 @@ message AddrBookRecord { // The multiaddresses. This is a sorted list where element 0 expires the soonest. repeated AddrEntry addrs = 2; - repeated AddrEntry signedAddrs = 3; - // The most recently received signed PeerRecord. CertifiedRecord certifiedRecord = 4; diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 4170180..4e687fb 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -49,11 +49,7 @@ type addrsRecord struct { func (r *addrsRecord) flush(write ds.Write) (err error) { key := addrBookBase.ChildString(b32.RawStdEncoding.EncodeToString([]byte(r.Id.ID))) - if len(r.SignedAddrs) == 0 { - r.CertifiedRecord = nil - } - - if len(r.Addrs) == 0 && len(r.SignedAddrs) == 0 { + if len(r.Addrs) == 0 { if err = write.Delete(key); err == nil { r.dirty = false } @@ -90,14 +86,13 @@ func (r *addrsRecord) flush(write ds.Write) (err error) { func (r *addrsRecord) clean() (chgd bool) { now := time.Now().Unix() addrsLen := len(r.Addrs) - signedAddrsLen := len(r.SignedAddrs) if !r.dirty && !r.hasExpiredAddrs(now) { // record is not dirty, and we have no expired entries to purge. return false } - if addrsLen == 0 && signedAddrsLen == 0 { + if addrsLen == 0 { // this is a ghost record; let's signal it has to be written. // flush() will take care of doing the deletion. return true @@ -109,29 +104,15 @@ func (r *addrsRecord) clean() (chgd bool) { }) } - if r.dirty && signedAddrsLen > 1 { - sort.Slice(r.Addrs, func(i, j int) bool { - return r.Addrs[i].Expiry < r.Addrs[j].Expiry - }) - } - r.Addrs = removeExpired(r.Addrs, now) - r.SignedAddrs = removeExpired(r.SignedAddrs, now) - - if len(r.SignedAddrs) == 0 { - r.CertifiedRecord = nil - } - return r.dirty || len(r.Addrs) != addrsLen || len(r.SignedAddrs) != signedAddrsLen + return r.dirty || len(r.Addrs) != addrsLen } func (r *addrsRecord) hasExpiredAddrs(now int64) bool { if len(r.Addrs) > 0 && r.Addrs[0].Expiry <= now { return true } - if len(r.SignedAddrs) > 0 && r.SignedAddrs[0].Expiry <= now { - return true - } return false } @@ -290,7 +271,7 @@ func (ab *dsAddrBook) ProcessPeerRecord(recordEnvelope *record.Envelope, ttl tim return fmt.Errorf("envelope did not contain PeerRecord") } - // ensure that the seq number from envelope is >= any previously received seq no + // ensure that the seq number from envelope is > any previously received seq no if ab.latestPeerRecordSeq(rec.PeerID) >= rec.Seq { return nil } @@ -306,7 +287,7 @@ func (ab *dsAddrBook) ProcessPeerRecord(recordEnvelope *record.Envelope, ttl tim func (ab *dsAddrBook) latestPeerRecordSeq(p peer.ID) uint64 { pr, err := ab.loadRecord(p, true, false) - if err != nil || pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { + if err != nil || len(pr.Addrs) == 0 || pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { return 0 } return pr.CertifiedRecord.Seq @@ -343,7 +324,7 @@ func (ab *dsAddrBook) GetPeerRecord(p peer.ID) *record.Envelope { log.Errorf("unable to load record for peer %s: %v", p.Pretty(), err) return nil } - if pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 { + if pr.CertifiedRecord == nil || len(pr.CertifiedRecord.Raw) == 0 || len(pr.Addrs) == 0 { return nil } state, _, err := record.ConsumeEnvelope(pr.CertifiedRecord.Raw, peer.PeerRecordEnvelopeDomain) @@ -382,8 +363,7 @@ func (ab *dsAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.D defer pr.Unlock() newExp := time.Now().Add(newTTL).Unix() - allAddrs := append(pr.Addrs, pr.SignedAddrs...) - for _, entry := range allAddrs { + for _, entry := range pr.Addrs { if entry.Ttl != int64(oldTTL) { continue } @@ -407,16 +387,9 @@ func (ab *dsAddrBook) Addrs(p peer.ID) []ma.Multiaddr { pr.RLock() defer pr.RUnlock() - size := len(pr.SignedAddrs) - if len(pr.Addrs) > size { - size = len(pr.Addrs) - } - addrs := make([]ma.Multiaddr, 0, size) - for _, a := range pr.SignedAddrs { - addrs = append(addrs, a.Addr) - } - for _, a := range pr.Addrs { - addrs = append(addrs, a.Addr) + addrs := make([]ma.Multiaddr, len(pr.Addrs)) + for i, a := range pr.Addrs { + addrs[i] = a.Addr } return addrs } @@ -458,6 +431,11 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio pr.Lock() defer pr.Unlock() + // if we have a signed PeerRecord, ignore attempts to add unsigned addrs + if !signed && pr.CertifiedRecord != nil { + return nil + } + newExp := time.Now().Add(ttl).Unix() updateExisting := func(entryList []*pb.AddrBookRecord_AddrEntry, incoming ma.Multiaddr) *pb.AddrBookRecord_AddrEntry { for _, have := range entryList { @@ -483,34 +461,22 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio return nil } - var added []*pb.AddrBookRecord_AddrEntry - + var entries []*pb.AddrBookRecord_AddrEntry for _, incoming := range addrs { - existingUnsigned := updateExisting(pr.Addrs, incoming) - - // we always check if the incoming addr is already in the - // signed addr list, even if we're not adding it from a - // signed source. if it does exist, we update its TTL - existingSigned := updateExisting(pr.SignedAddrs, incoming) - if existingSigned != nil { - added = append(added, existingSigned) - } + existingEntry := updateExisting(pr.Addrs, incoming) - // if we're adding a signed addr that already existed in the - // unsigned set, we move the existing unsigned entry over - // with the updated TTL, but we don't broadcast it as new - if signed && existingSigned == nil && existingUnsigned != nil { - added = append(added, existingUnsigned) - } - - // new addr, add & broadcast - if existingUnsigned == nil && existingSigned == nil { + if existingEntry != nil { + if signed { + entries = append(entries, existingEntry) + } + } else { + // new addr, add & broadcast entry := &pb.AddrBookRecord_AddrEntry{ Addr: &pb.ProtoAddr{Multiaddr: incoming}, Ttl: int64(ttl), Expiry: newExp, } - added = append(added, entry) + entries = append(entries, entry) // note: there's a minor chance that writing the record will fail, in which case we would've broadcast // the addresses without persisting them. This is very unlikely and not much of an issue. @@ -520,16 +486,9 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio if signed { // when adding signed addrs, we want to keep _only_ the incoming addrs - pr.SignedAddrs = added - pr.Addrs = make([]*pb.AddrBookRecord_AddrEntry, 0) + pr.Addrs = entries } else { - // if the incoming addrs are unsigned, we only keep them if - // no signed addrs exist for the peer - if len(pr.SignedAddrs) == 0 { - pr.Addrs = append(pr.Addrs, added...) - } else { - pr.Addrs = make([]*pb.AddrBookRecord_AddrEntry, 0) - } + pr.Addrs = append(pr.Addrs, entries...) } pr.dirty = true @@ -569,7 +528,7 @@ func (ab *dsAddrBook) deleteAddrs(p peer.ID, addrs []ma.Multiaddr) (err error) { return fmt.Errorf("failed to load peerstore entry for peer %v while deleting addrs, err: %v", p, err) } - if pr.Addrs == nil && pr.SignedAddrs == nil { + if pr.Addrs == nil { return nil } @@ -577,7 +536,6 @@ func (ab *dsAddrBook) deleteAddrs(p peer.ID, addrs []ma.Multiaddr) (err error) { defer pr.Unlock() pr.Addrs = deleteInPlace(pr.Addrs, addrs) - pr.SignedAddrs = deleteInPlace(pr.SignedAddrs, addrs) pr.dirty = true pr.clean() diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 0296b48..e73ab9c 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -43,8 +43,6 @@ type addrSegment struct { // drastically increase the space waste. In our case, by 6x. addrs map[peer.ID]map[string]*expiringAddr - signedAddrs map[peer.ID]map[string]*expiringAddr - signedPeerRecords map[peer.ID]*peerRecordState } @@ -73,7 +71,6 @@ func NewAddrBook() *memoryAddrBook { for i, _ := range ret { ret[i] = &addrSegment{ addrs: make(map[peer.ID]map[string]*expiringAddr), - signedAddrs: make(map[peer.ID]map[string]*expiringAddr), signedPeerRecords: make(map[peer.ID]*peerRecordState)} } return ret @@ -111,27 +108,22 @@ func (mab *memoryAddrBook) Close() error { // gc garbage collects the in-memory address book. func (mab *memoryAddrBook) gc() { now := time.Now() - collect := func(addrs map[peer.ID]map[string]*expiringAddr) []peer.ID { + for _, s := range mab.segments { + s.Lock() var collectedPeers []peer.ID - for p, amap := range addrs { + for p, amap := range s.addrs { for k, addr := range amap { if addr.ExpiredBy(now) { delete(amap, k) } } if len(amap) == 0 { - delete(addrs, p) + delete(s.addrs, p) collectedPeers = append(collectedPeers, p) } } - return collectedPeers - } - for _, s := range mab.segments { - s.Lock() - collect(s.addrs) - collected := collect(s.signedAddrs) - // remove routing records for peers whose signed addrs have all been removed - for _, p := range collected { + // remove signed records for peers whose signed addrs have all been removed + for _, p := range collectedPeers { delete(s.signedPeerRecords, p) } s.Unlock() @@ -148,11 +140,6 @@ func (mab *memoryAddrBook) PeersWithAddrs() peer.IDSlice { pidSet.Add(pid) } } - for pid, amap := range s.signedAddrs { - if amap != nil && len(amap) > 0 { - pidSet.Add(pid) - } - } s.RUnlock() } return pidSet.Peers() @@ -167,11 +154,16 @@ func (mab *memoryAddrBook) AddAddr(p peer.ID, addr ma.Multiaddr, ttl time.Durati // (time-to-live), after which the address is no longer valid. // This function never reduces the TTL or expiration of an address. func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) { + // if we have a valid peer record, ignore unsigned addrs + peerRec := mab.GetPeerRecord(p) + if peerRec != nil { + return + } mab.addAddrs(p, addrs, ttl, false) } // ProcessPeerRecord adds addresses from a signed peer.PeerRecord (contained in -// a routing.Envelope), which will expire after the given TTL. +// a record.Envelope), which will expire after the given TTL. // See https://godoc.org/github.com/libp2p/go-libp2p-core/peerstore#CertifiedAddrBook for more details. func (mab *memoryAddrBook) ProcessPeerRecord(recordEnvelope *record.Envelope, ttl time.Duration) error { r, err := recordEnvelope.Record() @@ -208,82 +200,55 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du s := mab.segments.get(p) s.Lock() defer s.Unlock() - existingUnsignedAddrs, ok := s.addrs[p] - if !ok { - existingUnsignedAddrs = make(map[string]*expiringAddr) - s.addrs[p] = existingUnsignedAddrs - } - existingSignedAddrs, ok := s.signedAddrs[p] - if !ok { - existingSignedAddrs = make(map[string]*expiringAddr) - s.signedAddrs[p] = existingSignedAddrs - } - // if we already have signed addrs, ignore attempts to add unsigned addrs - if !signed && len(existingSignedAddrs) != 0 { - return - } - - // if we're adding signed addresses, we want _only_ the new addrs - // to be in the final map, so we make a new map to contain them - signedAddrMap := existingSignedAddrs - unsignedAddrMap := existingUnsignedAddrs - if signed { - signedAddrMap = make(map[string]*expiringAddr, len(addrs)) - unsignedAddrMap = make(map[string]*expiringAddr, len(addrs)) + amap, ok := s.addrs[p] + if !ok { + amap = make(map[string]*expiringAddr) + s.addrs[p] = amap } exp := time.Now().Add(ttl) - maxTTLAndExp := func(a *expiringAddr, t time.Duration, e time.Time) (time.Duration, time.Time) { - if a == nil { - return t, e - } - if a.TTL > ttl { - t = a.TTL - } - if a.Expires.After(exp) { - e = a.Expires - } - return t, e - } + addrSet := make(map[string]struct{}, len(addrs)) for _, addr := range addrs { if addr == nil { log.Warningf("was passed nil multiaddr for %s", p) continue } k := string(addr.Bytes()) + addrSet[k] = struct{}{} // find the highest TTL and Expiry time between // existing records and function args - a, found := existingUnsignedAddrs[k] // won't allocate. - b, foundSigned := existingSignedAddrs[k] - maxTTL, maxExp := maxTTLAndExp(a, ttl, exp) - maxTTL, maxExp = maxTTLAndExp(b, maxTTL, maxExp) - - entry := &expiringAddr{Addr: addr, Expires: maxExp, TTL: maxTTL} - - // if we're adding a signed addr, or if it was previously signed, - // make sure it's not also in the unsigned addr list. - if signed || foundSigned { - signedAddrMap[k] = entry - delete(existingUnsignedAddrs, k) - } else { - existingUnsignedAddrs[k] = entry - } + a, found := amap[k] // won't allocate. - if !found && !foundSigned { + if !found { // not found, announce it. + entry := &expiringAddr{Addr: addr, Expires: exp, TTL: ttl} + amap[k] = entry mab.subManager.BroadcastAddr(p, addr) + } else { + // update ttl & exp to whichever is greater between new and existing entry + if ttl > a.TTL { + a.TTL = ttl + } + if exp.After(a.Expires) { + a.Expires = exp + } } } - // replace existing signed addrs with new ones - // this is a noop if we're adding unsigned addrs - s.signedAddrs[p] = signedAddrMap - s.addrs[p] = unsignedAddrMap + // when adding signed addrs, make sure only the addrs from the input list remain. + if signed { + for k := range amap { + _, ok := addrSet[k] + if !ok { + delete(amap, k) + } + } + } // if we've expired all the signed addresses for a peer, remove their signed routing state record - if len(signedAddrMap) == 0 { + if len(addrs) == 0 { delete(s.signedPeerRecords, p) } } @@ -305,11 +270,6 @@ func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du amap = make(map[string]*expiringAddr) s.addrs[p] = amap } - signedAddrMap, ok := s.signedAddrs[p] - if !ok { - signedAddrMap = make(map[string]*expiringAddr) - s.signedAddrs[p] = signedAddrMap - } exp := time.Now().Add(ttl) for _, addr := range addrs { @@ -320,26 +280,17 @@ func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du aBytes := addr.Bytes() key := string(aBytes) - // if the addr is in the signed addr set, update the ttl - // and ensure it's not in the unsigned set. otherwise, - // add to or update the unsigned set - mapToUpdate := amap - if _, signedExists := signedAddrMap[key]; signedExists { - mapToUpdate = signedAddrMap - delete(amap, key) - } - // re-set all of them for new ttl. if ttl > 0 { - mapToUpdate[key] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl} + amap[key] = &expiringAddr{Addr: addr, Expires: exp, TTL: ttl} mab.subManager.BroadcastAddr(p, addr) } else { - delete(mapToUpdate, key) + delete(amap, key) } } // if we've expired all the signed addresses for a peer, remove their signed routing state record - if len(signedAddrMap) == 0 { + if len(amap) == 0 { delete(s.signedPeerRecords, p) } } @@ -351,7 +302,8 @@ func (mab *memoryAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL t s.Lock() defer s.Unlock() exp := time.Now().Add(newTTL) - update := func(amap map[string]*expiringAddr) { + amap, found := s.addrs[p] + if found { for k, a := range amap { if oldTTL == a.TTL { a.TTL = newTTL @@ -360,18 +312,10 @@ func (mab *memoryAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL t } } } - amap, found := s.addrs[p] - if found { - update(amap) - } - signedAddrMap, found := s.signedAddrs[p] - if found { - update(signedAddrMap) - // updating may have caused all our certified addrs to expire. - // if so, remove the signed record we sourced them from. - if len(signedAddrMap) == 0 { - delete(s.signedPeerRecords, p) - } + + // if we've expired all the signed addresses for a peer, remove their signed routing state record + if len(amap) == 0 { + delete(s.signedPeerRecords, p) } } @@ -381,9 +325,7 @@ func (mab *memoryAddrBook) Addrs(p peer.ID) []ma.Multiaddr { s.RLock() defer s.RUnlock() - return append( - validAddrs(s.signedAddrs[p]), - validAddrs(s.addrs[p])...) + return validAddrs(s.addrs[p]) } func validAddrs(amap map[string]*expiringAddr) []ma.Multiaddr { @@ -412,7 +354,7 @@ func (mab *memoryAddrBook) GetPeerRecord(p peer.ID) *record.Envelope { // although the signed record gets garbage collected when all addrs inside it are expired, // we may be in between the expiration time and the GC interval // so, we check to see if we have any valid signed addrs before returning the record - if len(validAddrs(s.signedAddrs[p])) == 0 { + if len(validAddrs(s.addrs[p])) == 0 { return nil } @@ -430,7 +372,6 @@ func (mab *memoryAddrBook) ClearAddrs(p peer.ID) { defer s.Unlock() delete(s.addrs, p) - delete(s.signedAddrs, p) delete(s.signedPeerRecords, p) } From 1ef508a3b277678e922649695a09f03c7a89fb43 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 4 Feb 2020 14:50:42 -0500 Subject: [PATCH 34/39] track API changes from -core --- go.mod | 11 +++++------ go.sum | 17 +++++++++++++++++ pstoreds/addr_book.go | 18 +++++++++++------- pstoremem/addr_book.go | 12 ++++++------ test/addr_book_suite.go | 20 +++++++++++++++----- 5 files changed, 54 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 2c60244..3ec652f 100644 --- a/go.mod +++ b/go.mod @@ -8,20 +8,19 @@ require ( github.com/ipfs/go-ds-leveldb v0.4.0 github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-buffer-pool v0.0.2 - github.com/libp2p/go-libp2p-core v0.3.1-0.20200121152518-5a523a102629 + github.com/libp2p/go-libp2p-core v0.3.1-0.20200204201746-48fdd0fe436b github.com/libp2p/go-msgio v0.0.4 // indirect github.com/mailru/easyjson v0.7.0 // indirect github.com/multiformats/go-base32 v0.0.3 github.com/multiformats/go-multiaddr v0.2.0 github.com/multiformats/go-multiaddr-fmt v0.1.0 github.com/multiformats/go-multiaddr-net v0.1.1 - github.com/multiformats/go-multihash v0.0.10 - github.com/multiformats/go-varint v0.0.2 // indirect + github.com/multiformats/go-multihash v0.0.13 github.com/pkg/errors v0.9.1 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 - golang.org/x/crypto v0.0.0-20200117160349-530e935923ad // indirect - golang.org/x/sys v0.0.0-20200121082415-34d275377bf9 // indirect - golang.org/x/tools v0.0.0-20200121145236-13c74800b9d2 // indirect + golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 // indirect + golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 // indirect + golang.org/x/tools v0.0.0-20200204192400-7124308813f3 // indirect ) go 1.13 diff --git a/go.sum b/go.sum index e87ef4d..cd67199 100644 --- a/go.sum +++ b/go.sum @@ -85,10 +85,14 @@ github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoR github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-libp2p-core v0.3.1-0.20200121152518-5a523a102629 h1:U0mnCQcq57Y67jSgC4DtNpYcyFen91pWnxT4dsZ2M1Q= github.com/libp2p/go-libp2p-core v0.3.1-0.20200121152518-5a523a102629/go.mod h1:KJbT3ekTPccG9QUSa5vc/0QVZXPE9+3TVlyqESRQic0= +github.com/libp2p/go-libp2p-core v0.3.1-0.20200204201746-48fdd0fe436b h1:dStuLsQQG74FsC+frL1GDJ2J4KzFQPwySoMJ3Sihscc= +github.com/libp2p/go-libp2p-core v0.3.1-0.20200204201746-48fdd0fe436b/go.mod h1:xblDXGika41jyMQImcF5SzEmM3mRJlXtm7Xa/9QOjFg= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= +github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= @@ -125,9 +129,13 @@ github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKT github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2 h1:6sUvyh2YHpJCb8RZ6eYzj6iJQ4+chWYmyIHxszqlPTA= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -140,7 +148,9 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smola/gocompat v0.2.0 h1:6b1oIMlUXIpz//VKEDzPVBK8KG7beVwmHIUEBIs/Pns= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -175,6 +185,8 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200117160349-530e935923ad h1:Jh8cai0fqIK+f6nG0UgPW5wFk8wmiMhM3AyciDBdtQg= golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -206,6 +218,8 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200121082415-34d275377bf9 h1:N19i1HjUnR7TF7rMt8O4p3dLvqvmYyzB6ifMFmrbY50= golang.org/x/sys v0.0.0-20200121082415-34d275377bf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -215,7 +229,10 @@ golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20200121145236-13c74800b9d2 h1:pdlZPiMD2o4d/4PpHsPwGv7VwPb9UxXWORqFHS7X3bQ= golang.org/x/tools v0.0.0-20200121145236-13c74800b9d2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204192400-7124308813f3 h1:Ms82wn6YK4ZycO6Bxyh0kxX3gFFVGo79CCuc52xgcys= +golang.org/x/tools v0.0.0-20200204192400-7124308813f3/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 4e687fb..59b8712 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -258,31 +258,35 @@ func (ab *dsAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio ab.setAddrs(p, addrs, ttl, ttlExtend, false) } -// ProcessPeerRecord adds addresses from a signed peer.PeerRecord (contained in +// ConsumePeerRecord adds addresses from a signed peer.PeerRecord (contained in // a routing.Envelope), which will expire after the given TTL. // See https://godoc.org/github.com/libp2p/go-libp2p-core/peerstore#CertifiedAddrBook for more details. -func (ab *dsAddrBook) ProcessPeerRecord(recordEnvelope *record.Envelope, ttl time.Duration) error { +func (ab *dsAddrBook) ConsumePeerRecord(recordEnvelope *record.Envelope, ttl time.Duration) (bool, error) { r, err := recordEnvelope.Record() if err != nil { - return err + return false, err } rec, ok := r.(*peer.PeerRecord) if !ok { - return fmt.Errorf("envelope did not contain PeerRecord") + return false, fmt.Errorf("envelope did not contain PeerRecord") } // ensure that the seq number from envelope is > any previously received seq no if ab.latestPeerRecordSeq(rec.PeerID) >= rec.Seq { - return nil + return false, nil } addrs := cleanAddrs(rec.Addrs) err = ab.setAddrs(rec.PeerID, addrs, ttl, ttlExtend, true) if err != nil { - return err + return false, err } - return ab.storeSignedPeerRecord(rec.PeerID, recordEnvelope, rec) + err = ab.storeSignedPeerRecord(rec.PeerID, recordEnvelope, rec) + if err != nil { + return false, err + } + return true, nil } func (ab *dsAddrBook) latestPeerRecordSeq(p peer.ID) uint64 { diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index e73ab9c..970cd8f 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -162,17 +162,17 @@ func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du mab.addAddrs(p, addrs, ttl, false) } -// ProcessPeerRecord adds addresses from a signed peer.PeerRecord (contained in +// ConsumePeerRecord adds addresses from a signed peer.PeerRecord (contained in // a record.Envelope), which will expire after the given TTL. // See https://godoc.org/github.com/libp2p/go-libp2p-core/peerstore#CertifiedAddrBook for more details. -func (mab *memoryAddrBook) ProcessPeerRecord(recordEnvelope *record.Envelope, ttl time.Duration) error { +func (mab *memoryAddrBook) ConsumePeerRecord(recordEnvelope *record.Envelope, ttl time.Duration) (bool, error) { r, err := recordEnvelope.Record() if err != nil { - return err + return false, err } rec, ok := r.(*peer.PeerRecord) if !ok { - return fmt.Errorf("unable to process envelope: not a PeerRecord") + return false, fmt.Errorf("unable to process envelope: not a PeerRecord") } // ensure seq is greater than last received s := mab.segments.get(rec.PeerID) @@ -180,7 +180,7 @@ func (mab *memoryAddrBook) ProcessPeerRecord(recordEnvelope *record.Envelope, tt lastState, found := s.signedPeerRecords[rec.PeerID] if found && lastState.Seq >= rec.Seq { s.Unlock() - return nil + return false, nil } s.signedPeerRecords[rec.PeerID] = &peerRecordState{ Envelope: recordEnvelope, @@ -188,7 +188,7 @@ func (mab *memoryAddrBook) ProcessPeerRecord(recordEnvelope *record.Envelope, tt } s.Unlock() // need to release the lock, since addAddrs will try to take it mab.addAddrs(rec.PeerID, rec.Addrs, ttl, true) - return nil + return true, nil } func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration, signed bool) { diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index b59a0d1..b04097e 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -3,6 +3,7 @@ package test import ( "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/record" "github.com/libp2p/go-libp2p-core/test" "github.com/multiformats/go-multiaddr" "testing" @@ -377,7 +378,7 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { rec := peer.NewPeerRecord() rec.PeerID = id rec.Addrs = certifiedAddrs - signedRec, err := rec.Sign(priv) + signedRec, err := record.Seal(rec, priv) if err != nil { t.Errorf("error creating signed routing record: %v", err) } @@ -389,7 +390,7 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { AssertAddressesEqual(t, uncertifiedAddrs, m.Addrs(id)) // add the signed record to addr book - err = cab.ProcessPeerRecord(signedRec, time.Hour) + _, err = cab.ConsumePeerRecord(signedRec, time.Hour) if err != nil { t.Errorf("error adding signed routing record to addrbook: %v", err) } @@ -402,6 +403,15 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { t.Errorf("expected PeersWithAddrs to return 1, got %d", len(m.PeersWithAddrs())) } + // adding the same peer record again should result in the record being ignored + accepted, err := cab.ConsumePeerRecord(signedRec, time.Hour) + if accepted { + t.Error("Expected record with duplicate sequence number to be ignored") + } + if err != nil { + t.Errorf("Expected record with duplicate sequence number to be ignored without error, got err: %s", err) + } + // once certified addrs exist, trying to add non-certified addrs should have no effect m.AddAddrs(id, uncertifiedAddrs, time.Hour) AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) @@ -418,9 +428,9 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { rec = peer.NewPeerRecord() rec.PeerID = id rec.Addrs = certifiedAddrs - signedRec, err = rec.Sign(priv) + signedRec, err = record.Seal(rec, priv) test.AssertNilError(t, err) - err = cab.ProcessPeerRecord(signedRec, time.Hour) + _, err = cab.ConsumePeerRecord(signedRec, time.Hour) test.AssertNilError(t, err) AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) @@ -435,7 +445,7 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { } // Test that natural TTL expiration clears signed peer records - err = cab.ProcessPeerRecord(signedRec, time.Second) + _, err = cab.ConsumePeerRecord(signedRec, time.Second) test.AssertNilError(t, err) AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) From 681e2e514112354892efc39025b7242a3f4d9b0d Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 10 Feb 2020 13:09:45 -0500 Subject: [PATCH 35/39] validate that PeerRecord was signed by correct key --- pstoreds/addr_book.go | 3 +++ pstoremem/addr_book.go | 4 ++++ test/addr_book_suite.go | 11 +++++++++++ 3 files changed, 18 insertions(+) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 59b8712..6892266 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -270,6 +270,9 @@ func (ab *dsAddrBook) ConsumePeerRecord(recordEnvelope *record.Envelope, ttl tim if !ok { return false, fmt.Errorf("envelope did not contain PeerRecord") } + if !rec.PeerID.MatchesPublicKey(recordEnvelope.PublicKey) { + return false, fmt.Errorf("signing key does not match PeerID in PeerRecord") + } // ensure that the seq number from envelope is > any previously received seq no if ab.latestPeerRecordSeq(rec.PeerID) >= rec.Seq { diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index 970cd8f..ad17395 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -174,6 +174,10 @@ func (mab *memoryAddrBook) ConsumePeerRecord(recordEnvelope *record.Envelope, tt if !ok { return false, fmt.Errorf("unable to process envelope: not a PeerRecord") } + if !rec.PeerID.MatchesPublicKey(recordEnvelope.PublicKey) { + return false, fmt.Errorf("signing key does not match PeerID in PeerRecord") + } + // ensure seq is greater than last received s := mab.segments.get(rec.PeerID) s.Lock() diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index b04097e..1499a09 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -453,5 +453,16 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { if cab.GetPeerRecord(id) != nil { t.Error("expected signed peer record to be removed when addresses expire") } + + // adding a peer record that's signed with the wrong key should fail + priv2, _, err := test.RandTestKeyPair(crypto.Ed25519, 256) + test.AssertNilError(t, err) + env, err := record.Seal(rec, priv2) + test.AssertNilError(t, err) + + accepted, err = cab.ConsumePeerRecord(env, time.Second) + if accepted || err == nil { + t.Error("expected adding a PeerRecord that's signed with the wrong key to fail") + } } } From fc8dd0be2a172fa45e45704d490f3c469f3a4874 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 10 Feb 2020 13:11:49 -0500 Subject: [PATCH 36/39] fix doc comment --- pstoreds/addr_book.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index 6892266..bbad22d 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -259,7 +259,7 @@ func (ab *dsAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio } // ConsumePeerRecord adds addresses from a signed peer.PeerRecord (contained in -// a routing.Envelope), which will expire after the given TTL. +// a record.Envelope), which will expire after the given TTL. // See https://godoc.org/github.com/libp2p/go-libp2p-core/peerstore#CertifiedAddrBook for more details. func (ab *dsAddrBook) ConsumePeerRecord(recordEnvelope *record.Envelope, ttl time.Duration) (bool, error) { r, err := recordEnvelope.Record() From 07a4aea08659f78f26b88c3de988a93ffea3c56d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 10 Feb 2020 20:14:40 +0000 Subject: [PATCH 37/39] go mod tidy. --- go.mod | 4 ---- go.sum | 37 ++++++++++++++++--------------------- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 3ec652f..ded4ee4 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,6 @@ require ( github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-buffer-pool v0.0.2 github.com/libp2p/go-libp2p-core v0.3.1-0.20200204201746-48fdd0fe436b - github.com/libp2p/go-msgio v0.0.4 // indirect - github.com/mailru/easyjson v0.7.0 // indirect github.com/multiformats/go-base32 v0.0.3 github.com/multiformats/go-multiaddr v0.2.0 github.com/multiformats/go-multiaddr-fmt v0.1.0 @@ -18,9 +16,7 @@ require ( github.com/multiformats/go-multihash v0.0.13 github.com/pkg/errors v0.9.1 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 - golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 // indirect golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 // indirect - golang.org/x/tools v0.0.0-20200204192400-7124308813f3 // indirect ) go 1.13 diff --git a/go.sum b/go.sum index cd67199..5027f3d 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= @@ -28,7 +29,9 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= @@ -49,6 +52,7 @@ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmv github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= @@ -76,24 +80,21 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= -github.com/libp2p/go-libp2p-core v0.3.1-0.20200121152518-5a523a102629 h1:U0mnCQcq57Y67jSgC4DtNpYcyFen91pWnxT4dsZ2M1Q= -github.com/libp2p/go-libp2p-core v0.3.1-0.20200121152518-5a523a102629/go.mod h1:KJbT3ekTPccG9QUSa5vc/0QVZXPE9+3TVlyqESRQic0= github.com/libp2p/go-libp2p-core v0.3.1-0.20200204201746-48fdd0fe436b h1:dStuLsQQG74FsC+frL1GDJ2J4KzFQPwySoMJ3Sihscc= github.com/libp2p/go-libp2p-core v0.3.1-0.20200204201746-48fdd0fe436b/go.mod h1:xblDXGika41jyMQImcF5SzEmM3mRJlXtm7Xa/9QOjFg= -github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -132,12 +133,12 @@ github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpK github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.2 h1:6sUvyh2YHpJCb8RZ6eYzj6iJQ4+chWYmyIHxszqlPTA= -github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -145,6 +146,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -165,6 +167,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -181,17 +184,12 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad h1:Jh8cai0fqIK+f6nG0UgPW5wFk8wmiMhM3AyciDBdtQg= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w= -golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -205,7 +203,6 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -216,11 +213,10 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200121082415-34d275377bf9 h1:N19i1HjUnR7TF7rMt8O4p3dLvqvmYyzB6ifMFmrbY50= -golang.org/x/sys v0.0.0-20200121082415-34d275377bf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -229,12 +225,7 @@ golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20200121145236-13c74800b9d2 h1:pdlZPiMD2o4d/4PpHsPwGv7VwPb9UxXWORqFHS7X3bQ= -golang.org/x/tools v0.0.0-20200121145236-13c74800b9d2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204192400-7124308813f3 h1:Ms82wn6YK4ZycO6Bxyh0kxX3gFFVGo79CCuc52xgcys= -golang.org/x/tools v0.0.0-20200204192400-7124308813f3/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -242,11 +233,15 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From ffe5f0c0d36b36ac3b4cb5c5acbf847927a5c982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 10 Feb 2020 20:51:58 +0000 Subject: [PATCH 38/39] fix some nits. --- pb/pstore.pb.go | 49 +++++++++++++++++++++++++------------------------ pb/pstore.proto | 2 +- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/pb/pstore.pb.go b/pb/pstore.pb.go index 7e18cb3..c7d9980 100644 --- a/pb/pstore.pb.go +++ b/pb/pstore.pb.go @@ -30,7 +30,7 @@ type AddrBookRecord struct { // The multiaddresses. This is a sorted list where element 0 expires the soonest. Addrs []*AddrBookRecord_AddrEntry `protobuf:"bytes,2,rep,name=addrs,proto3" json:"addrs,omitempty"` // The most recently received signed PeerRecord. - CertifiedRecord *AddrBookRecord_CertifiedRecord `protobuf:"bytes,4,opt,name=certifiedRecord,proto3" json:"certifiedRecord,omitempty"` + CertifiedRecord *AddrBookRecord_CertifiedRecord `protobuf:"bytes,3,opt,name=certified_record,json=certifiedRecord,proto3" json:"certified_record,omitempty"` } func (m *AddrBookRecord) Reset() { *m = AddrBookRecord{} } @@ -201,27 +201,28 @@ func init() { func init() { proto.RegisterFile("pstore.proto", fileDescriptor_f96873690e08a98f) } var fileDescriptor_f96873690e08a98f = []byte{ - // 315 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x90, 0xbf, 0x4e, 0xf3, 0x30, - 0x14, 0xc5, 0xeb, 0xa4, 0x5f, 0xa5, 0xba, 0xf9, 0x08, 0xf2, 0x80, 0xa2, 0x0c, 0x4e, 0x80, 0x25, - 0x0c, 0xa4, 0x52, 0x11, 0x03, 0x23, 0x01, 0x06, 0xb6, 0xca, 0x2c, 0xac, 0x4d, 0xec, 0x06, 0x8b, - 0x3f, 0x0e, 0x8e, 0x2b, 0xe8, 0x5b, 0xf0, 0x48, 0x8c, 0x8c, 0x15, 0x13, 0xea, 0x50, 0x41, 0xf2, - 0x12, 0x8c, 0xc8, 0x4e, 0xa9, 0xd4, 0x4a, 0x6c, 0xe7, 0x5c, 0x9f, 0xfb, 0x3b, 0x57, 0x86, 0x4e, - 0x51, 0x2a, 0x21, 0x59, 0x5c, 0x48, 0xa1, 0x04, 0xea, 0xfe, 0xba, 0xd4, 0x3f, 0xcc, 0xb9, 0xba, - 0x99, 0xa4, 0x71, 0x26, 0xee, 0xfb, 0xb9, 0xc8, 0x45, 0xdf, 0x24, 0xd2, 0xc9, 0xd8, 0x38, 0x63, - 0x8c, 0x6a, 0x36, 0xf7, 0xde, 0x2d, 0xb8, 0x75, 0x4a, 0xa9, 0x4c, 0x84, 0xb8, 0x25, 0x2c, 0x13, - 0x92, 0xa2, 0x00, 0x5a, 0x9c, 0x7a, 0x20, 0x04, 0x91, 0x93, 0xb8, 0xf3, 0x45, 0xd0, 0x1b, 0xea, - 0xe4, 0x90, 0x31, 0x79, 0x79, 0x4e, 0x2c, 0x4e, 0xd1, 0x09, 0xfc, 0x37, 0xa2, 0x54, 0x96, 0x9e, - 0x15, 0xda, 0x51, 0x6f, 0xb0, 0x1f, 0xaf, 0xda, 0xe3, 0x75, 0x94, 0xb1, 0x17, 0x0f, 0x4a, 0x4e, - 0x49, 0xb3, 0x81, 0xae, 0xa0, 0x9b, 0x31, 0xa9, 0xf8, 0x98, 0x33, 0xda, 0x64, 0xbc, 0x76, 0x08, - 0xa2, 0xde, 0xe0, 0xe0, 0x6f, 0xc8, 0xd9, 0xfa, 0x02, 0xd9, 0x24, 0xf8, 0xd7, 0xb0, 0xbb, 0x2a, - 0x42, 0xbb, 0xb0, 0xad, 0xab, 0x96, 0xf7, 0xff, 0x9f, 0x2f, 0x82, 0xae, 0xb9, 0x5f, 0x27, 0x88, - 0x79, 0x42, 0x3b, 0xb0, 0xc3, 0x9e, 0x0b, 0x2e, 0xa7, 0x9e, 0x15, 0x82, 0xc8, 0x26, 0x4b, 0x87, - 0xb6, 0xa1, 0xad, 0xd4, 0x9d, 0x67, 0x9b, 0xa1, 0x96, 0xfe, 0x31, 0x74, 0x37, 0xda, 0x75, 0xa8, - 0x64, 0x8f, 0x06, 0xdf, 0x26, 0x5a, 0xea, 0x89, 0x1c, 0x3d, 0x19, 0x96, 0x43, 0xb4, 0x4c, 0xc2, - 0xef, 0x2f, 0x0c, 0x5e, 0x2b, 0x0c, 0xde, 0x2a, 0x0c, 0x66, 0x15, 0x06, 0x9f, 0x15, 0x06, 0x2f, - 0x35, 0x6e, 0xcd, 0x6a, 0xdc, 0xfa, 0xa8, 0x71, 0x2b, 0xed, 0x98, 0xdf, 0x3f, 0xfa, 0x09, 0x00, - 0x00, 0xff, 0xff, 0x1e, 0x8d, 0xf1, 0x8a, 0xc7, 0x01, 0x00, 0x00, + // 322 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x28, 0x2e, 0xc9, + 0x2f, 0x4a, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x84, 0xf1, 0x92, 0xa4, 0x74, 0xd3, + 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, 0xf5, 0xd3, 0xf3, 0xd3, 0xf3, 0xf5, 0xc1, + 0x2a, 0x92, 0x4a, 0xd3, 0xc0, 0x3c, 0x30, 0x07, 0xcc, 0x82, 0xe8, 0x54, 0xba, 0xcc, 0xc4, 0xc5, + 0xe7, 0x98, 0x92, 0x52, 0xe4, 0x94, 0x9f, 0x9f, 0x1d, 0x94, 0x9a, 0x9c, 0x5f, 0x94, 0x22, 0x24, + 0xcf, 0xc5, 0x94, 0x99, 0x22, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0xe3, 0xc4, 0x7f, 0xeb, 0x9e, 0x3c, + 0x77, 0x00, 0x48, 0x65, 0x40, 0x6a, 0x6a, 0x91, 0xa7, 0x4b, 0x10, 0x53, 0x66, 0x8a, 0x90, 0x25, + 0x17, 0x6b, 0x62, 0x4a, 0x4a, 0x51, 0xb1, 0x04, 0x93, 0x02, 0xb3, 0x06, 0xb7, 0x91, 0xb2, 0x1e, + 0xdc, 0x76, 0x3d, 0x54, 0xa3, 0xc0, 0x5c, 0xd7, 0xbc, 0x92, 0xa2, 0xca, 0x20, 0x88, 0x0e, 0xa1, + 0x10, 0x2e, 0x81, 0xe4, 0xd4, 0xa2, 0x92, 0xcc, 0xb4, 0xcc, 0xd4, 0x94, 0xf8, 0x22, 0xb0, 0x22, + 0x09, 0x66, 0x05, 0x46, 0x0d, 0x6e, 0x23, 0x4d, 0xdc, 0xa6, 0x38, 0xc3, 0x74, 0x40, 0xf8, 0x41, + 0xfc, 0xc9, 0xa8, 0x02, 0x52, 0x11, 0x5c, 0x9c, 0x70, 0x9b, 0x84, 0x14, 0xb9, 0x58, 0x40, 0x76, + 0x41, 0x3d, 0xc0, 0x7b, 0xeb, 0x9e, 0x3c, 0x27, 0xd8, 0x03, 0x20, 0x15, 0x41, 0x60, 0x29, 0x21, + 0x31, 0x2e, 0xb6, 0xd4, 0x8a, 0x82, 0xcc, 0xa2, 0x4a, 0x09, 0x26, 0x05, 0x46, 0x0d, 0xe6, 0x20, + 0x28, 0x4f, 0x48, 0x80, 0x8b, 0xb9, 0xa4, 0x24, 0x07, 0xec, 0x20, 0xe6, 0x20, 0x10, 0x53, 0xca, + 0x94, 0x8b, 0x1f, 0xcd, 0x76, 0x90, 0xa2, 0xe2, 0xd4, 0x42, 0xb0, 0xf1, 0x2c, 0x41, 0x20, 0x26, + 0x48, 0xa4, 0x28, 0xb1, 0x1c, 0x6c, 0x16, 0x4f, 0x10, 0x88, 0xe9, 0xa4, 0xf0, 0xe3, 0xa1, 0x1c, + 0xe3, 0x81, 0x47, 0x72, 0x8c, 0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x78, 0x24, 0xc7, 0xf8, 0xe0, 0x91, + 0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, 0x17, 0x1e, 0xcb, 0x31, 0xdc, 0x78, 0x2c, 0xc7, 0x90, 0xc4, + 0x06, 0x0e, 0x7e, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x29, 0xcd, 0xe8, 0xd4, 0xc8, 0x01, + 0x00, 0x00, } func (m *AddrBookRecord) Marshal() (dAtA []byte, err error) { @@ -254,7 +255,7 @@ func (m *AddrBookRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) { i = encodeVarintPstore(dAtA, i, uint64(size)) } i-- - dAtA[i] = 0x22 + dAtA[i] = 0x1a } if len(m.Addrs) > 0 { for iNdEx := len(m.Addrs) - 1; iNdEx >= 0; iNdEx-- { @@ -657,7 +658,7 @@ func (m *AddrBookRecord) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 4: + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field CertifiedRecord", wireType) } diff --git a/pb/pstore.proto b/pb/pstore.proto index f0a5b53..0d1abd1 100644 --- a/pb/pstore.proto +++ b/pb/pstore.proto @@ -15,7 +15,7 @@ message AddrBookRecord { repeated AddrEntry addrs = 2; // The most recently received signed PeerRecord. - CertifiedRecord certifiedRecord = 4; + CertifiedRecord certified_record = 3; // AddrEntry represents a single multiaddress. message AddrEntry { From 18b766a647216196469ba3cd85bd2586139ccd6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 6 Mar 2020 11:23:10 +0000 Subject: [PATCH 39/39] temp: store all addresses in the addrbook. Temporarily store all addresses in the addrbook without classifying their provenance/authoritativeness; even in the presence of a signed peer record. --- go.mod | 4 ++-- go.sum | 21 ++++++++++----------- pstoreds/addr_book.go | 35 +++++++++++++++++++---------------- pstoremem/addr_book.go | 26 +++++++++++++------------- test/addr_book_suite.go | 13 +++++++++---- 5 files changed, 53 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index ded4ee4..23afd16 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,9 @@ require ( github.com/ipfs/go-ds-leveldb v0.4.0 github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-buffer-pool v0.0.2 - github.com/libp2p/go-libp2p-core v0.3.1-0.20200204201746-48fdd0fe436b + github.com/libp2p/go-libp2p-core v0.4.0 github.com/multiformats/go-base32 v0.0.3 - github.com/multiformats/go-multiaddr v0.2.0 + github.com/multiformats/go-multiaddr v0.2.1 github.com/multiformats/go-multiaddr-fmt v0.1.0 github.com/multiformats/go-multiaddr-net v0.1.1 github.com/multiformats/go-multihash v0.0.13 diff --git a/go.sum b/go.sum index 5027f3d..430b5b0 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= -github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= +github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-datastore v0.3.0/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU= github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= @@ -85,15 +85,15 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= -github.com/libp2p/go-libp2p-core v0.3.1-0.20200204201746-48fdd0fe436b h1:dStuLsQQG74FsC+frL1GDJ2J4KzFQPwySoMJ3Sihscc= -github.com/libp2p/go-libp2p-core v0.3.1-0.20200204201746-48fdd0fe436b/go.mod h1:xblDXGika41jyMQImcF5SzEmM3mRJlXtm7Xa/9QOjFg= +github.com/libp2p/go-libp2p-core v0.4.0 h1:LjZJP/Yy4q8kc724izkYQ9v6YkAmkKCOaE5jLv/NZRo= +github.com/libp2p/go-libp2p-core v0.4.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= @@ -118,8 +118,8 @@ github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= -github.com/multiformats/go-multiaddr v0.2.0 h1:lR52sFwcTCuQb6bTfnXF6zA2XfyYvyd+5a9qECv/J90= -github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= +github.com/multiformats/go-multiaddr v0.2.1 h1:SgG/cw5vqyB5QQe5FPe2TqggU9WtrA9X4nZw7LlVqOI= +github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.1.1 h1:jFFKUuXTXv+3ARyHZi3XUqQO+YWMKgBdhEvuGRfnL6s= @@ -128,11 +128,9 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= -github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -178,7 +176,7 @@ github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -224,6 +222,7 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/pstoreds/addr_book.go b/pstoreds/addr_book.go index bbad22d..a79875c 100644 --- a/pstoreds/addr_book.go +++ b/pstoreds/addr_book.go @@ -438,12 +438,16 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio pr.Lock() defer pr.Unlock() - // if we have a signed PeerRecord, ignore attempts to add unsigned addrs - if !signed && pr.CertifiedRecord != nil { - return nil - } + // // if we have a signed PeerRecord, ignore attempts to add unsigned addrs + // if !signed && pr.CertifiedRecord != nil { + // return nil + // } newExp := time.Now().Add(ttl).Unix() + // TODO this is very inefficient O(m*n); we could build a map to use as an + // index, and test against it. That would turn it into O(m+n). This code + // will be refactored entirely anyway, and it's not being used by users + // (that we know of); so OK to keep it for now. updateExisting := func(entryList []*pb.AddrBookRecord_AddrEntry, incoming ma.Multiaddr) *pb.AddrBookRecord_AddrEntry { for _, have := range entryList { if incoming.Equal(have.Addr) { @@ -461,7 +465,6 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio default: panic("BUG: unimplemented ttl mode") } - return have } } @@ -472,11 +475,11 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio for _, incoming := range addrs { existingEntry := updateExisting(pr.Addrs, incoming) - if existingEntry != nil { - if signed { - entries = append(entries, existingEntry) - } - } else { + if existingEntry == nil { + // if signed { + // entries = append(entries, existingEntry) + // } + // } else { // new addr, add & broadcast entry := &pb.AddrBookRecord_AddrEntry{ Addr: &pb.ProtoAddr{Multiaddr: incoming}, @@ -491,12 +494,12 @@ func (ab *dsAddrBook) setAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duratio } } - if signed { - // when adding signed addrs, we want to keep _only_ the incoming addrs - pr.Addrs = entries - } else { - pr.Addrs = append(pr.Addrs, entries...) - } + // if signed { + // // when adding signed addrs, we want to keep _only_ the incoming addrs + // pr.Addrs = entries + // } else { + pr.Addrs = append(pr.Addrs, entries...) + // } pr.dirty = true pr.clean() diff --git a/pstoremem/addr_book.go b/pstoremem/addr_book.go index ad17395..b77bd69 100644 --- a/pstoremem/addr_book.go +++ b/pstoremem/addr_book.go @@ -155,10 +155,10 @@ func (mab *memoryAddrBook) AddAddr(p peer.ID, addr ma.Multiaddr, ttl time.Durati // This function never reduces the TTL or expiration of an address. func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) { // if we have a valid peer record, ignore unsigned addrs - peerRec := mab.GetPeerRecord(p) - if peerRec != nil { - return - } + // peerRec := mab.GetPeerRecord(p) + // if peerRec != nil { + // return + // } mab.addAddrs(p, addrs, ttl, false) } @@ -241,15 +241,15 @@ func (mab *memoryAddrBook) addAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Du } } - // when adding signed addrs, make sure only the addrs from the input list remain. - if signed { - for k := range amap { - _, ok := addrSet[k] - if !ok { - delete(amap, k) - } - } - } + // // when adding signed addrs, make sure only the addrs from the input list remain. + // if signed { + // for k := range amap { + // _, ok := addrSet[k] + // if !ok { + // delete(amap, k) + // } + // } + // } // if we've expired all the signed addresses for a peer, remove their signed routing state record if len(addrs) == 0 { diff --git a/test/addr_book_suite.go b/test/addr_book_suite.go index 1499a09..f05a418 100644 --- a/test/addr_book_suite.go +++ b/test/addr_book_suite.go @@ -396,7 +396,8 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { } // the non-certified addrs should be gone & we should get only certified addrs back from Addrs - AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) + // AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) + AssertAddressesEqual(t, allAddrs, m.Addrs(id)) // PeersWithAddrs should return a single peer if len(m.PeersWithAddrs()) != 1 { @@ -413,8 +414,10 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { } // once certified addrs exist, trying to add non-certified addrs should have no effect + // m.AddAddrs(id, uncertifiedAddrs, time.Hour) + // AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) m.AddAddrs(id, uncertifiedAddrs, time.Hour) - AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) + AssertAddressesEqual(t, allAddrs, m.Addrs(id)) // we should be able to retrieve the signed peer record rec2 := cab.GetPeerRecord(id) @@ -432,11 +435,13 @@ func testCertifiedAddresses(m pstore.AddrBook) func(*testing.T) { test.AssertNilError(t, err) _, err = cab.ConsumePeerRecord(signedRec, time.Hour) test.AssertNilError(t, err) - AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) + // AssertAddressesEqual(t, certifiedAddrs, m.Addrs(id)) + AssertAddressesEqual(t, allAddrs, m.Addrs(id)) // update TTL on signed addrs to -1 to remove them. // the signed routing record should be deleted - m.SetAddrs(id, certifiedAddrs, -1) + // m.SetAddrs(id, certifiedAddrs, -1) + m.SetAddrs(id, allAddrs, -1) if len(m.Addrs(id)) != 0 { t.Error("expected zero certified addrs after setting TTL to -1") }