From ca8e425a7c95602be1e00996649e721244381c6e Mon Sep 17 00:00:00 2001 From: Andrew Richardson Date: Tue, 21 Jun 2022 10:27:43 -0400 Subject: [PATCH 1/3] Verify all operations succeed during E2E test Signed-off-by: Andrew Richardson --- internal/dataexchange/ffdx/dxevent.go | 4 ++++ internal/dataexchange/ffdx/ffdx_test.go | 21 +++++++++++---------- mocks/dataexchangemocks/dx_event.go | 14 ++++++++++++++ pkg/dataexchange/plugin.go | 1 + test/e2e/e2e_test.go | 8 ++++++++ test/e2e/ethereum_contract_test.go | 4 ++++ test/e2e/fabric_contract_test.go | 4 ++++ test/e2e/identity_test.go | 5 +++++ test/e2e/onchain_offchain_test.go | 4 ++++ test/e2e/restclient_test.go | 12 ++++++++++++ test/e2e/tokens_test.go | 5 +++++ 11 files changed, 72 insertions(+), 10 deletions(-) diff --git a/internal/dataexchange/ffdx/dxevent.go b/internal/dataexchange/ffdx/dxevent.go index 304ad4aa3e..5e120efb1c 100644 --- a/internal/dataexchange/ffdx/dxevent.go +++ b/internal/dataexchange/ffdx/dxevent.go @@ -51,6 +51,10 @@ type dxEvent struct { transferResult *dataexchange.TransferResult } +func (e *dxEvent) EventID() string { + return e.id +} + func (e *dxEvent) NamespacedID() string { return e.id } diff --git a/internal/dataexchange/ffdx/ffdx_test.go b/internal/dataexchange/ffdx/ffdx_test.go index 23e7889070..de41c97901 100644 --- a/internal/dataexchange/ffdx/ffdx_test.go +++ b/internal/dataexchange/ffdx/ffdx_test.go @@ -446,7 +446,8 @@ func TestEvents(t *testing.T) { h.SetHandler(mcb) mcb.On("DXEvent", mock.MatchedBy(func(ev dataexchange.DXEvent) bool { - return ev.NamespacedID() == "1" && + return ev.EventID() == "1" && + ev.NamespacedID() == "tx12345" && ev.Type() == dataexchange.DXEventTypeTransferResult && ev.TransferResult().TrackingID == "tx12345" && ev.TransferResult().Status == core.OpStatusFailed && @@ -457,7 +458,7 @@ func TestEvents(t *testing.T) { assert.Equal(t, `{"action":"ack","id":"1"}`, string(msg)) mcb.On("DXEvent", mock.MatchedBy(func(ev dataexchange.DXEvent) bool { - return ev.NamespacedID() == "2" && + return ev.EventID() == "2" && ev.Type() == dataexchange.DXEventTypeTransferResult && ev.TransferResult().TrackingID == "tx12345" && ev.TransferResult().Status == core.OpStatusSucceeded @@ -467,7 +468,7 @@ func TestEvents(t *testing.T) { assert.Equal(t, `{"action":"ack","id":"2"}`, string(msg)) mcb.On("DXEvent", mock.MatchedBy(func(ev dataexchange.DXEvent) bool { - return ev.NamespacedID() == "3" && + return ev.EventID() == "3" && ev.Type() == dataexchange.DXEventTypeTransferResult && ev.TransferResult().TrackingID == "tx12345" && ev.TransferResult().Status == core.OpStatusSucceeded && @@ -479,7 +480,7 @@ func TestEvents(t *testing.T) { assert.Equal(t, `{"action":"ack","id":"3"}`, string(msg)) mcb.On("DXEvent", mock.MatchedBy(func(ev dataexchange.DXEvent) bool { - return ev.NamespacedID() == "4" && + return ev.EventID() == "4" && ev.Type() == dataexchange.DXEventTypeMessageReceived && ev.MessageReceived().PeerID == "peer1" && string(ev.MessageReceived().Data) == "message1" @@ -489,7 +490,7 @@ func TestEvents(t *testing.T) { assert.Equal(t, `{"action":"ack","id":"4","manifest":"{\"manifest\":true}"}`, string(msg)) mcb.On("DXEvent", mock.MatchedBy(func(ev dataexchange.DXEvent) bool { - return ev.NamespacedID() == "5" && + return ev.EventID() == "5" && ev.Type() == dataexchange.DXEventTypeTransferResult && ev.TransferResult().TrackingID == "tx12345" && ev.TransferResult().Status == core.OpStatusFailed && @@ -500,7 +501,7 @@ func TestEvents(t *testing.T) { assert.Equal(t, `{"action":"ack","id":"5"}`, string(msg)) mcb.On("DXEvent", mock.MatchedBy(func(ev dataexchange.DXEvent) bool { - return ev.NamespacedID() == "6" && + return ev.EventID() == "6" && ev.Type() == dataexchange.DXEventTypeTransferResult && ev.TransferResult().TrackingID == "tx12345" && ev.TransferResult().Status == core.OpStatusSucceeded && @@ -521,7 +522,7 @@ func TestEvents(t *testing.T) { hash := fftypes.NewRandB32() mcb.On("DXEvent", mock.MatchedBy(func(ev dataexchange.DXEvent) bool { - return ev.NamespacedID() == "9" && + return ev.EventID() == "9" && ev.Type() == dataexchange.DXEventTypePrivateBlobReceived && ev.PrivateBlobReceived().Hash.Equals(hash) })).Run(acker()).Return(nil) @@ -530,7 +531,7 @@ func TestEvents(t *testing.T) { assert.Equal(t, `{"action":"ack","id":"9"}`, string(msg)) mcb.On("DXEvent", mock.MatchedBy(func(ev dataexchange.DXEvent) bool { - return ev.NamespacedID() == "10" && + return ev.EventID() == "10" && ev.Type() == dataexchange.DXEventTypeTransferResult && ev.TransferResult().TrackingID == "tx12345" && ev.TransferResult().Status == core.OpStatusSucceeded && @@ -560,7 +561,7 @@ func TestEventsWithManifest(t *testing.T) { h.SetHandler(mcb) mcb.On("DXEvent", mock.MatchedBy(func(ev dataexchange.DXEvent) bool { - return ev.NamespacedID() == "1" && + return ev.EventID() == "1" && ev.Type() == dataexchange.DXEventTypeTransferResult && ev.TransferResult().Status == core.OpStatusPending })).Run(acker()).Return(nil) @@ -569,7 +570,7 @@ func TestEventsWithManifest(t *testing.T) { assert.Equal(t, `{"action":"ack","id":"1"}`, string(msg)) mcb.On("DXEvent", mock.MatchedBy(func(ev dataexchange.DXEvent) bool { - return ev.NamespacedID() == "2" && + return ev.EventID() == "2" && ev.Type() == dataexchange.DXEventTypeTransferResult && ev.TransferResult().Status == core.OpStatusPending })).Run(acker()).Return(nil) diff --git a/mocks/dataexchangemocks/dx_event.go b/mocks/dataexchangemocks/dx_event.go index f2cc489e9e..d251dd5344 100644 --- a/mocks/dataexchangemocks/dx_event.go +++ b/mocks/dataexchangemocks/dx_event.go @@ -22,6 +22,20 @@ func (_m *DXEvent) AckWithManifest(manifest string) { _m.Called(manifest) } +// EventID provides a mock function with given fields: +func (_m *DXEvent) EventID() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + // MessageReceived provides a mock function with given fields: func (_m *DXEvent) MessageReceived() *dataexchange.MessageReceived { ret := _m.Called() diff --git a/pkg/dataexchange/plugin.go b/pkg/dataexchange/plugin.go index 7a57467526..523a2d7b60 100644 --- a/pkg/dataexchange/plugin.go +++ b/pkg/dataexchange/plugin.go @@ -108,6 +108,7 @@ type DXEventType int // DXEvent is a single interface that can be passed to all events type DXEvent interface { + EventID() string NamespacedID() string Ack() AckWithManifest(manifest string) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 33c5b97767..4d1d446667 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -437,3 +437,11 @@ func checkObject(t *testing.T, expected interface{}, actual interface{}) bool { } return match } + +func verifyAllOperationsSucceeded(t *testing.T, clients []*resty.Client, startTime time.Time) { + for _, client := range clients { + for _, op := range GetOperations(t, client, startTime) { + assert.Equal(t, core.OpStatusSucceeded, op.Status, "Client: %s, Operation %s (%s)", client.BaseURL, op.ID, op.Type) + } + } +} diff --git a/test/e2e/ethereum_contract_test.go b/test/e2e/ethereum_contract_test.go index d794bcb6f6..cd549c5142 100644 --- a/test/e2e/ethereum_contract_test.go +++ b/test/e2e/ethereum_contract_test.go @@ -136,6 +136,10 @@ func (suite *EthereumContractTestSuite) BeforeTest(suiteName, testName string) { suite.testState = beforeE2ETest(suite.T()) } +func (suite *EthereumContractTestSuite) AfterTest(suiteName, testName string) { + verifyAllOperationsSucceeded(suite.T(), []*resty.Client{suite.testState.client1, suite.testState.client2}, suite.testState.startTime) +} + func (suite *EthereumContractTestSuite) TestDirectInvokeMethod() { defer suite.testState.done() diff --git a/test/e2e/fabric_contract_test.go b/test/e2e/fabric_contract_test.go index bbed1341bb..3c99c42c36 100644 --- a/test/e2e/fabric_contract_test.go +++ b/test/e2e/fabric_contract_test.go @@ -138,6 +138,10 @@ func (suite *FabricContractTestSuite) BeforeTest(suiteName, testName string) { suite.testState = beforeE2ETest(suite.T()) } +func (suite *FabricContractTestSuite) AfterTest(suiteName, testName string) { + verifyAllOperationsSucceeded(suite.T(), []*resty.Client{suite.testState.client1, suite.testState.client2}, suite.testState.startTime) +} + func (suite *FabricContractTestSuite) TestE2EContractEvents() { defer suite.testState.done() diff --git a/test/e2e/identity_test.go b/test/e2e/identity_test.go index 81e0c8f764..44fda88cc7 100644 --- a/test/e2e/identity_test.go +++ b/test/e2e/identity_test.go @@ -22,6 +22,7 @@ import ( "strings" "time" + "github.com/go-resty/resty/v2" "github.com/hyperledger/firefly-common/pkg/fftypes" "github.com/hyperledger/firefly/pkg/core" "github.com/stretchr/testify/assert" @@ -38,6 +39,10 @@ func (suite *IdentityTestSuite) BeforeTest(suiteName, testName string) { suite.testState = beforeE2ETest(suite.T()) } +func (suite *IdentityTestSuite) AfterTest(suiteName, testName string) { + verifyAllOperationsSucceeded(suite.T(), []*resty.Client{suite.testState.client1, suite.testState.client2}, suite.testState.startTime) +} + func (suite *IdentityTestSuite) TestCustomChildIdentityBroadcasts() { defer suite.testState.done() diff --git a/test/e2e/onchain_offchain_test.go b/test/e2e/onchain_offchain_test.go index 015fc4868a..2bde1cebd3 100644 --- a/test/e2e/onchain_offchain_test.go +++ b/test/e2e/onchain_offchain_test.go @@ -44,6 +44,10 @@ func (suite *OnChainOffChainTestSuite) BeforeTest(suiteName, testName string) { suite.testState = beforeE2ETest(suite.T()) } +func (suite *OnChainOffChainTestSuite) AfterTest(suiteName, testName string) { + verifyAllOperationsSucceeded(suite.T(), []*resty.Client{suite.testState.client1, suite.testState.client2}, suite.testState.startTime) +} + func (suite *OnChainOffChainTestSuite) TestE2EBroadcast() { defer suite.testState.done() diff --git a/test/e2e/restclient_test.go b/test/e2e/restclient_test.go index db80aa05fb..953a6eb0bd 100644 --- a/test/e2e/restclient_test.go +++ b/test/e2e/restclient_test.go @@ -64,6 +64,7 @@ var ( urlContractListeners = "/namespaces/default/contracts/listeners" urlContractAPI = "/namespaces/default/apis" urlBlockchainEvents = "/namespaces/default/blockchainevents" + urlOperations = "/namespaces/default/operations" urlGetOrganizations = "/namespaces/default/network/organizations" urlGetOrgKeys = "/namespaces/default/identities/%s/verifiers" ) @@ -817,3 +818,14 @@ func GetBlockchainEvent(t *testing.T, client *resty.Client, eventID string) (int require.Equal(t, 200, resp.StatusCode(), "GET %s [%d]: %s", path, resp.StatusCode(), resp.String()) return res, err } + +func GetOperations(t *testing.T, client *resty.Client, startTime time.Time) (operations []*core.Operation) { + path := urlOperations + resp, err := client.R(). + SetQueryParam("created", fmt.Sprintf(">%d", startTime.UnixNano())). + SetResult(&operations). + Get(path) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode(), "GET %s [%d]: %s", path, resp.StatusCode(), resp.String()) + return operations +} diff --git a/test/e2e/tokens_test.go b/test/e2e/tokens_test.go index 66cd14f589..49113836cf 100644 --- a/test/e2e/tokens_test.go +++ b/test/e2e/tokens_test.go @@ -21,6 +21,7 @@ import ( "math/rand" "time" + "github.com/go-resty/resty/v2" "github.com/hyperledger/firefly-common/pkg/fftypes" "github.com/hyperledger/firefly/pkg/core" "github.com/stretchr/testify/assert" @@ -43,6 +44,10 @@ func (suite *TokensTestSuite) BeforeTest(suiteName, testName string) { suite.testState = beforeE2ETest(suite.T()) } +func (suite *TokensTestSuite) AfterTest(suiteName, testName string) { + verifyAllOperationsSucceeded(suite.T(), []*resty.Client{suite.testState.client1, suite.testState.client2}, suite.testState.startTime) +} + func (suite *TokensTestSuite) TestE2EFungibleTokensAsync() { defer suite.testState.done() From 63e4da997724a1202f2507ddcbdf7f06cfaa966d Mon Sep 17 00:00:00 2001 From: Andrew Richardson Date: Tue, 21 Jun 2022 10:27:58 -0400 Subject: [PATCH 2/3] Resolve DX operations via requestID (not event ID) Fixes #871 Signed-off-by: Andrew Richardson --- internal/dataexchange/ffdx/dxevent.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/dataexchange/ffdx/dxevent.go b/internal/dataexchange/ffdx/dxevent.go index 5e120efb1c..8020e3c733 100644 --- a/internal/dataexchange/ffdx/dxevent.go +++ b/internal/dataexchange/ffdx/dxevent.go @@ -45,6 +45,7 @@ type wsEvent struct { type dxEvent struct { ffdx *FFDX id string + requestID string dxType dataexchange.DXEventType messageReceived *dataexchange.MessageReceived privateBlobReceived *dataexchange.PrivateBlobReceived @@ -56,7 +57,7 @@ func (e *dxEvent) EventID() string { } func (e *dxEvent) NamespacedID() string { - return e.id + return e.requestID } func (e *dxEvent) Type() dataexchange.DXEventType { @@ -92,7 +93,7 @@ func (e *dxEvent) TransferResult() *dataexchange.TransferResult { func (h *FFDX) dispatchEvent(msg *wsEvent) { var err error - e := &dxEvent{ffdx: h, id: msg.EventID} + e := &dxEvent{ffdx: h, id: msg.EventID, requestID: msg.RequestID} switch msg.Type { case messageFailed: e.dxType = dataexchange.DXEventTypeTransferResult From d0a0507f432c707ca06338ed7cdb93c47defcbfb Mon Sep 17 00:00:00 2001 From: Andrew Richardson Date: Wed, 22 Jun 2022 10:21:27 -0400 Subject: [PATCH 3/3] Add retries when checking for operation success Operations are updated by a separate processing loop in core, and may take a moment to get marked as successful after completion. Signed-off-by: Andrew Richardson --- test/e2e/e2e_test.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 4d1d446667..52de56510d 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -439,9 +439,24 @@ func checkObject(t *testing.T, expected interface{}, actual interface{}) bool { } func verifyAllOperationsSucceeded(t *testing.T, clients []*resty.Client, startTime time.Time) { - for _, client := range clients { - for _, op := range GetOperations(t, client, startTime) { - assert.Equal(t, core.OpStatusSucceeded, op.Status, "Client: %s, Operation %s (%s)", client.BaseURL, op.ID, op.Type) + tries := 3 + delay := 2 * time.Second + + var pending string + for i := 0; i < tries; i++ { + pending = "" + for _, client := range clients { + for _, op := range GetOperations(t, client, startTime) { + if op.Status != core.OpStatusSucceeded { + pending += fmt.Sprintf("Operation '%s' (%s) on '%s' is not successful\n", op.ID, op.Type, client.BaseURL) + } + } } + if pending == "" { + return + } + time.Sleep(delay) } + + assert.Fail(t, pending) }