Fix Gossip Message ID (#7624)

* fix snappy errors

* gaz

* Update beacon-chain/p2p/pubsub.go

Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com>

beacon-chain/p2p/BUILD.bazel

@@ -63,6 +63,7 @@ go_library(
         "@com_github_ethereum_go_ethereum//p2p/enode:go_default_library",
         "@com_github_ethereum_go_ethereum//p2p/enr:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_golang_snappy//:go_default_library",
         "@com_github_ipfs_go_ipfs_addr//:go_default_library",
         "@com_github_kevinms_leakybucket_go//:go_default_library",
         "@com_github_libp2p_go_libp2p//:go_default_library",
@@ -140,6 +141,7 @@ go_test(
         "@com_github_ethereum_go_ethereum//p2p/enode:go_default_library",
         "@com_github_ethereum_go_ethereum//p2p/enr:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_golang_snappy//:go_default_library",
         "@com_github_kevinms_leakybucket_go//:go_default_library",
         "@com_github_libp2p_go_libp2p//:go_default_library",
         "@com_github_libp2p_go_libp2p_blankhost//:go_default_library",

beacon-chain/p2p/pubsub.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"time"
 
+	"github.com/golang/snappy"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 // JoinTopic will join PubSub topic, if not already joined.
@@ -75,10 +77,24 @@ func (s *Service) SubscribeToTopic(topic string, opts ...pubsub.SubOpt) (*pubsub
 // Content addressable ID function.
 //
 // ETH2 spec defines the message ID as:
-//    message-id: SHA256(message.data)
+//    The `message-id` of a gossipsub message MUST be the following 20 byte value computed from the message data:
+//    If `message.data` has a valid snappy decompression, set `message-id` to the first 20 bytes of the `SHA256` hash of
+//    the concatenation of `MESSAGE_DOMAIN_VALID_SNAPPY` with the snappy decompressed message data,
+//    i.e. `SHA256(MESSAGE_DOMAIN_VALID_SNAPPY + snappy_decompress(message.data))[:20]`.
+//
+//    Otherwise, set `message-id` to the first 20 bytes of the `SHA256` hash of
+//    the concatenation of `MESSAGE_DOMAIN_INVALID_SNAPPY` with the raw message data,
+//    i.e. `SHA256(MESSAGE_DOMAIN_INVALID_SNAPPY + message.data)[:20]`.
 func msgIDFunction(pmsg *pubsub_pb.Message) string {
-	h := hashutil.Hash(pmsg.Data)
-	return string(h[:])
+	decodedData, err := snappy.Decode(nil /*dst*/, pmsg.Data)
+	if err != nil {
+		combinedData := append(params.BeaconNetworkConfig().MessageDomainInvalidSnappy[:], pmsg.Data...)
+		h := hashutil.Hash(combinedData)
+		return string(h[:20])
+	}
+	combinedData := append(params.BeaconNetworkConfig().MessageDomainValidSnappy[:], decodedData...)
+	h := hashutil.Hash(combinedData)
+	return string(h[:20])
 }
 
 func setPubSubParameters() {

beacon-chain/p2p/pubsub_test.go

@@ -7,11 +7,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/golang/snappy"
 	pubsubpb "github.com/libp2p/go-libp2p-pubsub/pb"
 	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p/encoder"
 	testp2p "github.com/prysmaticlabs/prysm/beacon-chain/p2p/testing"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
 	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
@@ -56,9 +58,16 @@ func TestService_PublishToTopicConcurrentMapWrite(t *testing.T) {
 }
 
 func TestMessageIDFunction_HashesCorrectly(t *testing.T) {
-	msg := [32]byte{'J', 'U', 'N', 'K'}
-	pMsg := &pubsubpb.Message{Data: msg[:]}
-	hashedData := hashutil.Hash(pMsg.Data)
-	msgID := string(hashedData[:])
+	invalidSnappy := [32]byte{'J', 'U', 'N', 'K'}
+	pMsg := &pubsubpb.Message{Data: invalidSnappy[:]}
+	hashedData := hashutil.Hash(append(params.BeaconNetworkConfig().MessageDomainInvalidSnappy[:], pMsg.Data...))
+	msgID := string(hashedData[:20])
 	assert.Equal(t, msgID, msgIDFunction(pMsg), "Got incorrect msg id")
+
+	validObj := [32]byte{'v', 'a', 'l', 'i', 'd'}
+	enc := snappy.Encode(nil, validObj[:])
+	nMsg := &pubsubpb.Message{Data: enc}
+	hashedData = hashutil.Hash(append(params.BeaconNetworkConfig().MessageDomainValidSnappy[:], validObj[:]...))
+	msgID = string(hashedData[:20])
+	assert.Equal(t, msgID, msgIDFunction(nMsg), "Got incorrect msg id")
 }

shared/params/mainnet_config.go

@@ -27,6 +27,8 @@ var mainnetNetworkConfig = &NetworkConfig{
 	TtfbTimeout:                       5 * time.Second,
 	RespTimeout:                       10 * time.Second,
 	MaximumGossipClockDisparity:       500 * time.Millisecond,
+	MessageDomainInvalidSnappy:        [4]byte{00, 00, 00, 00},
+	MessageDomainValidSnappy:          [4]byte{01, 00, 00, 00},
 	ETH2Key:                           "eth2",
 	AttSubnetKey:                      "attnets",
 	ContractDeploymentBlock:           0,

shared/params/network_config.go

@@ -23,6 +23,8 @@ type NetworkConfig struct {
 	TtfbTimeout                       time.Duration `yaml:"TTFB_TIMEOUT"`                          // TtfbTimeout is the maximum time to wait for first byte of request response (time-to-first-byte).
 	RespTimeout                       time.Duration `yaml:"RESP_TIMEOUT"`                          // RespTimeout is the maximum time for complete response transfer.
 	MaximumGossipClockDisparity       time.Duration `yaml:"MAXIMUM_GOSSIP_CLOCK_DISPARITY"`        // MaximumGossipClockDisparity is the maximum milliseconds of clock disparity assumed between honest nodes.
+	MessageDomainInvalidSnappy        [4]byte       `yaml:"MESSAGE_DOMAIN_INVALID_SNAPPY"`         // MessageDomainInvalidSnappy is the 4-byte domain for gossip message-id isolation of invalid snappy messages.
+	MessageDomainValidSnappy          [4]byte       `yaml:"MESSAGE_DOMAIN_VALID_SNAPPY"`           // MessageDomainValidSnappy is the 4-byte domain for gossip message-id isolation of valid snappy messages.
 
 	// DiscoveryV5 Config
 	ETH2Key      string // ETH2Key is the ENR key of the eth2 object in an enr.