Enable unaware threshold signature endorsement

This commit contains the following two seemingly unrelated changes:

1. Deduplicate endorsements in createTx
-------------------------------------
This commit makes createTx de-duplicate endorsers according to their identity.
Meaning, when multiple but identical endorsements (same identity) are passed to
the createTx function, it returns a transaction with unique endorsements, and without repeated endorsers.
Since at validation, endorsers are anyway de-duplicated, this commit preserves current system behavior.

2. Support identity based endorsement policy in discovery
------------------------------------------------------
Currently, whenever a chaincode's endorsement policy contains an identity principal, discovery rejects the policy,
saying that it contains a principal it does not support. This commit simply makes discovery classify the principal.

Making clients and gateways unaware of a threshold signature scheme
-------------------------------------------------------------------
This commit enables advanced use cases, in which case the endorsements collected for transaction
submission contain identical identities that represent a threshold or aggregation signature scheme.

In such a case, peers can be deployed with custom endorsement plugins and generate a threshold/aggregation signature,
but the flow of service discovery, the client, and peer gateway remains the same:
1. The client and the gateway interact as usual, and the gateway collects endorsements from several peers
2. The peers, using the endorsement plugin, engage in a protocol at which end they all return the same endorsement
3. The gateway submits the transaction (this time, with a single endorsement) to the orderer

In such a case, the endorsement policy needs to be an OR over two mutually exclusive cases:
1. An AND over a set of organizational principals (such as Org1.Peer and Org2.Peer)
2. An OR over one or more identities (which can only be signed in collaboration among the peers)

In such a case, discovery will always return only combinations of the former kind, but the endorsement policy itself
that will take place upon transaction validation may be of the latter kind.

Change-Id: I9193187ae6b08791f8762a9d325442d156c4f828
Signed-off-by: Yacov Manevich <yacovm@il.ibm.com>

common/policies/inquire/compare.go

@@ -21,6 +21,7 @@ type ComparablePrincipal struct {
 	ou        *msp.OrganizationUnit
 	role      *msp.MSPRole
 	mspID     string
+	idBytes   []byte
 }
 
 // Equal returns whether this ComparablePrincipal is equal to the given ComparablePrincipal.
@@ -44,6 +45,8 @@ func NewComparablePrincipal(principal *msp.MSPPrincipal) *ComparablePrincipal {
 		return cp.ToRole()
 	case msp.MSPPrincipal_ORGANIZATION_UNIT:
 		return cp.ToOURole()
+	case msp.MSPPrincipal_IDENTITY:
+		return cp.ToIdentity()
 	}
 	mapping := msp.MSPPrincipal_Classification_name[int32(principal.PrincipalClassification)]
 	logger.Warning("Received an unsupported principal type:", principal.PrincipalClassification, "mapped to", mapping)
@@ -101,6 +104,10 @@ func (cp *ComparablePrincipal) IsA(other *ComparablePrincipal) bool {
 		return this.role.Role == other.role.Role
 	}
 
+	if this.principal.PrincipalClassification == msp.MSPPrincipal_IDENTITY {
+		return bytes.Equal(this.idBytes, other.idBytes) && this.mspID == other.mspID
+	}
+
 	// Else, we can't say anything, because we have no knowledge
 	// about the OUs that make up the MSP roles - so return false
 	return false
@@ -119,6 +126,19 @@ func (cp *ComparablePrincipal) ToOURole() *ComparablePrincipal {
 	return cp
 }
 
+// ToIdentity converts this ComparablePrincipal to Identity principal, and returns nil on failure
+func (cp *ComparablePrincipal) ToIdentity() *ComparablePrincipal {
+	sID := &msp.SerializedIdentity{}
+	err := proto.Unmarshal(cp.principal.Principal, sID)
+	if err != nil {
+		logger.Warning("Failed unmarshalling principal:", err)
+		return nil
+	}
+	cp.mspID = sID.Mspid
+	cp.idBytes = sID.IdBytes
+	return cp
+}
+
 // ToRole converts this ComparablePrincipal to MSP Role, and returns nil if the conversion failed
 func (cp *ComparablePrincipal) ToRole() *ComparablePrincipal {
 	mspRole := &msp.MSPRole{}

common/policies/inquire/compare_test.go

@@ -47,8 +47,17 @@ func TestNewComparablePrincipal(t *testing.T) {
 		require.Nil(t, NewComparablePrincipal(nil))
 	})
 
-	t.Run("Invalid principal type", func(t *testing.T) {
-		require.Nil(t, NewComparablePrincipal(identity(mspID)))
+	t.Run("Identity", func(t *testing.T) {
+		expectedPrincipal := &ComparablePrincipal{
+			mspID:   mspID,
+			idBytes: []byte("identity"),
+			principal: &msp.MSPPrincipal{
+				PrincipalClassification: msp.MSPPrincipal_IDENTITY,
+				Principal:               protoutil.MarshalOrPanic(&msp.SerializedIdentity{IdBytes: []byte("identity"), Mspid: mspID}),
+			},
+		}
+
+		require.Equal(t, expectedPrincipal, NewComparablePrincipal(identity(mspID, []byte("identity"))))
 	})
 
 	t.Run("Invalid principal input", func(t *testing.T) {
@@ -88,6 +97,9 @@ func TestNewComparablePrincipal(t *testing.T) {
 }
 
 func TestIsA(t *testing.T) {
+	id1 := NewComparablePrincipal(identity("Org1MSP", []byte("identity")))
+	id2 := NewComparablePrincipal(identity("Org2MSP", []byte("identity")))
+	id3 := NewComparablePrincipal(identity("Org1MSP", []byte("identity2")))
 	member1 := NewComparablePrincipal(member("Org1MSP"))
 	member2 := NewComparablePrincipal(member("Org2MSP"))
 	peer1 := NewComparablePrincipal(peer("Org1MSP"))
@@ -145,6 +157,15 @@ func TestIsA(t *testing.T) {
 	t.Run("OUs and Peers aren't the same", func(t *testing.T) {
 		require.False(t, ou1.IsA(peer1))
 	})
+	t.Run("Same identity", func(t *testing.T) {
+		require.True(t, id1.IsA(id1))
+	})
+	t.Run("Different identity bytes", func(t *testing.T) {
+		require.False(t, id1.IsA(id3))
+	})
+	t.Run("Different MSP ID", func(t *testing.T) {
+		require.False(t, id1.IsA(id2))
+	})
 }
 
 func TestIsFound(t *testing.T) {
@@ -160,8 +181,11 @@ func TestIsFound(t *testing.T) {
 
 func TestNewComparablePrincipalSet(t *testing.T) {
 	t.Run("Invalid principal", func(t *testing.T) {
-		principals := []*msp.MSPPrincipal{member("Org1MSP"), identity("Org1MSP")}
-		require.Nil(t, NewComparablePrincipalSet(policies.PrincipalSet(principals)))
+		idCP := NewComparablePrincipal(identity("Org1MSP", []byte("identity")))
+		principals := []*msp.MSPPrincipal{member("Org1MSP"), identity("Org1MSP", []byte("identity"))}
+		cps := NewComparablePrincipalSet(principals)
+		expected := ComparablePrincipalSet([]*ComparablePrincipal{member1, idCP})
+		require.Equal(t, expected, cps)
 	})
 
 	t.Run("Valid Principals", func(t *testing.T) {
@@ -195,9 +219,9 @@ func ou(orgName string) *msp.MSPPrincipal {
 	}
 }
 
-func identity(orgName string) *msp.MSPPrincipal {
+func identity(orgName string, idBytes []byte) *msp.MSPPrincipal {
 	return &msp.MSPPrincipal{
 		PrincipalClassification: msp.MSPPrincipal_IDENTITY,
-		Principal:               protoutil.MarshalOrPanic(&msp.SerializedIdentity{Mspid: orgName, IdBytes: []byte("identity")}),
+		Principal:               protoutil.MarshalOrPanic(&msp.SerializedIdentity{Mspid: orgName, IdBytes: idBytes}),
 	}
 }

discovery/endorsement/endorsement_test.go

@@ -256,11 +256,7 @@ func TestPeersForEndorsement(t *testing.T) {
 	t.Run("NoChaincodeMetadataFromLedger", func(t *testing.T) {
 		// Scenario VII: Policy is found, there are enough peers to satisfy the policy,
 		// but the chaincode metadata cannot be fetched from the ledger.
-		pb := principalBuilder{}
-		policy := pb.newSet().addPrincipal(peerRole("p0")).addPrincipal(peerRole("p6")).
-			newSet().addPrincipal(peerRole("p12")).buildPolicy()
 		g.On("PeersOfChannel").Return(chanPeers.toMembers()).Once()
-		pf.On("PoliciesByChaincode", cc).Return(policy).Once()
 		mf := &metadataFetcher{}
 		mf.On("Metadata").Return(nil).Once()
 		analyzer := NewEndorsementAnalyzer(g, pf, &principalEvaluatorMock{}, mf)
@@ -908,6 +904,40 @@ func TestPeersForEndorsement(t *testing.T) {
 			peerIdentityString("p2"): {},
 		}, extractPeers(desc))
 	})
+
+	t.Run("Identity based endorsement policy doesn't interfere with discovery", func(t *testing.T) {
+		// Scenario XIX: Policy is based on either identities or on organizations
+		// 2 principal combinations:
+		// p0 and p6 (organizations), or p7 (identity)
+
+		pb := principalBuilder{}
+		policy := pb.newSet().addPrincipal(peerRole("p0")).addPrincipal(peerRole("p6")).
+			newSet().addPrincipal(identity("p7")).buildPolicy()
+		g.On("PeersOfChannel").Return(chanPeers.toMembers()).Once()
+		mf := &metadataFetcher{}
+		mf.On("Metadata").Return(&chaincode.Metadata{
+			Name:    cc,
+			Version: "1.0",
+		}).Once()
+		analyzer := NewEndorsementAnalyzer(g, pf, &principalEvaluatorMock{}, mf)
+
+		pf.On("PoliciesByChaincode", cc).Return(policy).Once()
+		desc, err := analyzer.PeersForEndorsement(channel, &peer.ChaincodeInterest{
+			Chaincodes: []*peer.ChaincodeCall{
+				{
+					Name: cc,
+				},
+			},
+		})
+		require.NoError(t, err)
+		require.NotNil(t, desc)
+		require.Len(t, desc.Layouts, 1)
+		require.Len(t, desc.Layouts[0].QuantitiesByGroup, 2)
+		require.Equal(t, map[string]struct{}{
+			peerIdentityString("p0"): {},
+			peerIdentityString("p6"): {},
+		}, extractPeers(desc))
+	})
 }
 
 func TestPeersAuthorizedByCriteria(t *testing.T) {
@@ -1202,6 +1232,16 @@ func peerRole(pkiID string) *msp.MSPPrincipal {
 	}
 }
 
+func identity(pkiID string) *msp.MSPPrincipal {
+	return &msp.MSPPrincipal{
+		PrincipalClassification: msp.MSPPrincipal_IDENTITY,
+		Principal: protoutil.MarshalOrPanic(&msp.SerializedIdentity{
+			Mspid:   pkiID2MSPID[pkiID],
+			IdBytes: []byte(pkiID),
+		}),
+	}
+}
+
 func (pi *peerInfo) withChaincode(name, version string) *peerInfo {
 	if pi.Properties == nil {
 		pi.Properties = &gossip.Properties{}
@@ -1278,15 +1318,26 @@ func (ip inquireablePolicy) SatisfiedBy() []policies.PrincipalSet {
 
 type principalEvaluatorMock struct{}
 
-func (pe *principalEvaluatorMock) SatisfiesPrincipal(channel string, identity []byte, principal *msp.MSPPrincipal) error {
-	peerRole := &msp.MSPRole{}
-	if err := proto.Unmarshal(principal.Principal, peerRole); err != nil {
-		return err
-	}
+func (pe *principalEvaluatorMock) SatisfiesPrincipal(_ string, identity []byte, principal *msp.MSPPrincipal) error {
 	sId := &msp.SerializedIdentity{}
 	if err := proto.Unmarshal(identity, sId); err != nil {
 		return err
 	}
+	if principal.PrincipalClassification == msp.MSPPrincipal_IDENTITY {
+		identityPrincipal := &msp.SerializedIdentity{}
+		if err := proto.Unmarshal(principal.Principal, identityPrincipal); err != nil {
+			return err
+		}
+		if proto.Equal(sId, identityPrincipal) {
+			return nil
+		}
+		return fmt.Errorf("identities do not match")
+	}
+	// Else, it's either an OU type or a role type, so we only classify by MSP ID
+	peerRole := &msp.MSPRole{}
+	if err := proto.Unmarshal(principal.Principal, peerRole); err != nil {
+		return err
+	}
 	if peerRole.MspIdentifier == sId.Mspid {
 		return nil
 	}

protoutil/txutils.go

@@ -208,10 +208,23 @@ func createTx(
 		}
 	}
 
-	// fill endorsements
-	endorsements := make([]*peer.Endorsement, len(resps))
-	for n, r := range resps {
-		endorsements[n] = r.Endorsement
+	// fill endorsements according to their uniqueness
+	endorsersUsed := make(map[string]struct{})
+	var endorsements []*peer.Endorsement
+	for _, r := range resps {
+		if r.Endorsement == nil {
+			continue
+		}
+		key := string(r.Endorsement.Endorser)
+		if _, used := endorsersUsed[key]; used {
+			continue
+		}
+		endorsements = append(endorsements, r.Endorsement)
+		endorsersUsed[key] = struct{}{}
+	}
+
+	if len(endorsements) == 0 {
+		return nil, errors.Errorf("no endorsements")
 	}
 
 	// create ChaincodeEndorsedAction

protoutil/txutils_test.go

@@ -139,6 +139,32 @@ func TestCreateTx(t *testing.T) {
 	require.True(t, proto.Equal(signedTx, unsignedTx), "got: %#v, want: %#v", signedTx, unsignedTx)
 }
 
+func TestDeduplicateEndorsements(t *testing.T) {
+	proposal := &pb.Proposal{
+		Header: protoutil.MarshalOrPanic(&cb.Header{
+			ChannelHeader: protoutil.MarshalOrPanic(&cb.ChannelHeader{
+				Extension: protoutil.MarshalOrPanic(&pb.ChaincodeHeaderExtension{}),
+			}),
+		}),
+	}
+	responses := []*pb.ProposalResponse{
+		{Payload: []byte("payload"), Endorsement: &pb.Endorsement{Endorser: []byte{5, 4, 3}}, Response: &pb.Response{Status: int32(200)}},
+		{Payload: []byte("payload"), Endorsement: &pb.Endorsement{Endorser: []byte{5, 4, 3}}, Response: &pb.Response{Status: int32(200)}},
+	}
+
+	transaction, err := protoutil.CreateTx(proposal, responses...)
+	require.NoError(t, err)
+	require.True(t, proto.Equal(transaction, transaction), "got: %#v, want: %#v", transaction, transaction)
+
+	pl := protoutil.UnmarshalPayloadOrPanic(transaction.Payload)
+	tx, err := protoutil.UnmarshalTransaction(pl.Data)
+	require.NoError(t, err)
+	ccap, err := protoutil.UnmarshalChaincodeActionPayload(tx.Actions[0].Payload)
+	require.NoError(t, err)
+	require.Len(t, ccap.Action.Endorsements, 1)
+	require.Equal(t, []byte{5, 4, 3}, ccap.Action.Endorsements[0].Endorser)
+}
+
 func TestCreateSignedTx(t *testing.T) {
 	var err error
 	prop := &pb.Proposal{}