From 06afcef74ae22a62b0c05bc7d1524a00a0e5bfbf Mon Sep 17 00:00:00 2001 From: Andrew Richardson Date: Thu, 9 Jun 2022 12:25:53 -0400 Subject: [PATCH 1/2] Add namespace to pins Signed-off-by: Andrew Richardson --- .../000092_add_pin_namespace.down.sql | 3 + .../postgres/000092_add_pin_namespace.up.sql | 5 + .../sqlite/000092_add_pin_namespace.down.sql | 1 + .../sqlite/000092_add_pin_namespace.up.sql | 2 + docs/swagger/swagger.yaml | 157 +++++++++++++++++- internal/apiserver/route_get_status_pins.go | 2 +- .../apiserver/route_get_status_pins_test.go | 2 +- internal/apiserver/routes.go | 2 +- internal/coremsgs/en_struct_descriptions.go | 1 + internal/database/sqlcommon/pin_sql.go | 28 +--- internal/database/sqlcommon/pin_sql_test.go | 35 +--- internal/events/batch_pin_complete.go | 1 + internal/orchestrator/data_query.go | 3 +- internal/orchestrator/data_query_test.go | 2 +- internal/orchestrator/orchestrator.go | 2 +- internal/orchestrator/persistence_events.go | 3 +- .../orchestrator/persistence_events_test.go | 2 +- mocks/databasemocks/callbacks.go | 6 +- mocks/databasemocks/plugin.go | 14 -- mocks/orchestratormocks/orchestrator.go | 18 +- pkg/core/pin.go | 1 + pkg/database/plugin.go | 11 +- 22 files changed, 207 insertions(+), 94 deletions(-) create mode 100644 db/migrations/postgres/000092_add_pin_namespace.down.sql create mode 100644 db/migrations/postgres/000092_add_pin_namespace.up.sql create mode 100644 db/migrations/sqlite/000092_add_pin_namespace.down.sql create mode 100644 db/migrations/sqlite/000092_add_pin_namespace.up.sql diff --git a/db/migrations/postgres/000092_add_pin_namespace.down.sql b/db/migrations/postgres/000092_add_pin_namespace.down.sql new file mode 100644 index 0000000000..44c4c79c83 --- /dev/null +++ b/db/migrations/postgres/000092_add_pin_namespace.down.sql @@ -0,0 +1,3 @@ +BEGIN; +ALTER TABLE pins DROP COLUMN namespace; +COMMIT; diff --git a/db/migrations/postgres/000092_add_pin_namespace.up.sql b/db/migrations/postgres/000092_add_pin_namespace.up.sql new file mode 100644 index 0000000000..36d55b6340 --- /dev/null +++ b/db/migrations/postgres/000092_add_pin_namespace.up.sql @@ -0,0 +1,5 @@ +BEGIN; +ALTER TABLE pins ADD COLUMN namespace VARCHAR(64); +UPDATE pins SET namespace = 'ff_system'; +ALTER TABLE pins ALTER COLUMN namespace SET NOT NULL; +COMMIT; diff --git a/db/migrations/sqlite/000092_add_pin_namespace.down.sql b/db/migrations/sqlite/000092_add_pin_namespace.down.sql new file mode 100644 index 0000000000..3871b630f0 --- /dev/null +++ b/db/migrations/sqlite/000092_add_pin_namespace.down.sql @@ -0,0 +1 @@ +ALTER TABLE pins DROP COLUMN namespace; diff --git a/db/migrations/sqlite/000092_add_pin_namespace.up.sql b/db/migrations/sqlite/000092_add_pin_namespace.up.sql new file mode 100644 index 0000000000..df14865e9f --- /dev/null +++ b/db/migrations/sqlite/000092_add_pin_namespace.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE pins ADD COLUMN namespace VARCHAR(64); +UPDATE pins SET namespace = "ff_system"; diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml index 7de8df97b3..1ff69d72eb 100644 --- a/docs/swagger/swagger.yaml +++ b/docs/swagger/swagger.yaml @@ -20105,6 +20105,158 @@ paths: description: "" tags: - Non-Default Namespace + /namespaces/{ns}/status/pins: + get: + description: Queries the pins table that is the status of the event aggregator + operationId: getStatusPinsNamespace + parameters: + - description: The namespace which scopes this request + in: path + name: ns + required: true + schema: + example: default + type: string + - description: Server-side request timeout (milliseconds, or set a custom suffix + like 10s) + in: header + name: Request-Timeout + schema: + default: 2m0s + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: batch + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: created + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: dispatched + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: hash + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: index + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: masked + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: sequence + schema: + type: string + - description: Sort field. For multi-field sort use comma separated values (or + multiple query values) with '-' prefix for descending + in: query + name: sort + schema: + type: string + - description: Ascending sort order (overrides all fields in a multi-field sort) + in: query + name: ascending + schema: + type: string + - description: Descending sort order (overrides all fields in a multi-field + sort) + in: query + name: descending + schema: + type: string + - description: 'The number of records to skip (max: 1,000). Unsuitable for bulk + operations' + in: query + name: skip + schema: + type: string + - description: 'The maximum number of records to return (max: 1,000)' + in: query + name: limit + schema: + example: "25" + type: string + - description: Return a total count as well as items (adds extra database processing) + in: query + name: count + schema: + type: string + responses: + "200": + content: + application/json: + schema: + items: + properties: + batch: + description: The UUID of the batch of messages this pin is part + of + format: uuid + type: string + batchHash: + description: The manifest hash batch of messages this pin is + part of + format: byte + type: string + created: + description: The time the FireFly node created the pin + format: date-time + type: string + dispatched: + description: Once true, this pin has been processed and will + not be processed again + type: boolean + hash: + description: The hash represents a topic within a message in + the batch. If a message has multiple topics, then multiple + pins are created. If the message is private, the hash is masked + for privacy + format: byte + type: string + index: + description: The index of this pin within the batch. One pin + is created for each topic, of each message in the batch + format: int64 + type: integer + masked: + description: True if the pin is for a private message, and hence + is masked with the group ID and salted with a nonce so observers + of the blockchain cannot use pin hash to match this transaction + to other transactions or participants + type: boolean + namespace: + description: The namespace of the pin + type: string + sequence: + description: The order of the pin in the local FireFly database, + which matches the order in which pins were delivered to FireFly + by the blockchain connector event stream + format: int64 + type: integer + signer: + description: The blockchain signing key that submitted this + transaction, as passed through to FireFly by the smart contract + that emitted the blockchain event + type: string + type: object + type: array + description: Success + default: + description: "" + tags: + - Non-Default Namespace /namespaces/{ns}/subscriptions: get: description: Gets a list of subscriptions @@ -27564,6 +27716,9 @@ paths: of the blockchain cannot use pin hash to match this transaction to other transactions or participants type: boolean + namespace: + description: The namespace of the pin + type: string sequence: description: The order of the pin in the local FireFly database, which matches the order in which pins were delivered to FireFly @@ -27581,7 +27736,7 @@ paths: default: description: "" tags: - - Global + - Default Namespace /status/websockets: get: description: Gets the status of the current WebSocket connections to this node diff --git a/internal/apiserver/route_get_status_pins.go b/internal/apiserver/route_get_status_pins.go index 5f162e2cf6..d44242b979 100644 --- a/internal/apiserver/route_get_status_pins.go +++ b/internal/apiserver/route_get_status_pins.go @@ -38,7 +38,7 @@ var getStatusPins = &ffapi.Route{ Extensions: &coreExtensions{ FilterFactory: database.PinQueryFactory, CoreJSONHandler: func(r *ffapi.APIRequest, cr *coreRequest) (output interface{}, err error) { - return filterResult(cr.or.GetPins(cr.ctx, cr.filter)) + return filterResult(cr.or.GetPins(cr.ctx, extractNamespace(r.PP), cr.filter)) }, }, } diff --git a/internal/apiserver/route_get_status_pins_test.go b/internal/apiserver/route_get_status_pins_test.go index 4ea3d4e19e..9a555ec709 100644 --- a/internal/apiserver/route_get_status_pins_test.go +++ b/internal/apiserver/route_get_status_pins_test.go @@ -31,7 +31,7 @@ func TestGetStatusPins(t *testing.T) { req.Header.Set("Content-Type", "application/json; charset=utf-8") res := httptest.NewRecorder() - o.On("GetPins", mock.Anything, mock.Anything). + o.On("GetPins", mock.Anything, "default", mock.Anything). Return([]*core.Pin{}, nil, nil) r.ServeHTTP(res, req) diff --git a/internal/apiserver/routes.go b/internal/apiserver/routes.go index cc24aa8d55..49e2f5d198 100644 --- a/internal/apiserver/routes.go +++ b/internal/apiserver/routes.go @@ -45,7 +45,6 @@ var routes = append( getNamespace, getNamespaces, getStatusBatchManager, - getStatusPins, getStatusWebSockets, }), namespacedRoutes([]*ffapi.Route{ @@ -95,6 +94,7 @@ var routes = append( getOpByID, getOps, getStatus, + getStatusPins, getSubscriptionByID, getSubscriptions, getTokenAccountPools, diff --git a/internal/coremsgs/en_struct_descriptions.go b/internal/coremsgs/en_struct_descriptions.go index 5164f5fda3..5e6fce00d8 100644 --- a/internal/coremsgs/en_struct_descriptions.go +++ b/internal/coremsgs/en_struct_descriptions.go @@ -443,6 +443,7 @@ var ( // Pin field descriptions PinSequence = ffm("Pin.sequence", "The order of the pin in the local FireFly database, which matches the order in which pins were delivered to FireFly by the blockchain connector event stream") + PinNamespace = ffm("Pin.namespace", "The namespace of the pin") PinMasked = ffm("Pin.masked", "True if the pin is for a private message, and hence is masked with the group ID and salted with a nonce so observers of the blockchain cannot use pin hash to match this transaction to other transactions or participants") PinHash = ffm("Pin.hash", "The hash represents a topic within a message in the batch. If a message has multiple topics, then multiple pins are created. If the message is private, the hash is masked for privacy") PinBatch = ffm("Pin.batch", "The UUID of the batch of messages this pin is part of") diff --git a/internal/database/sqlcommon/pin_sql.go b/internal/database/sqlcommon/pin_sql.go index f8d64ddb82..317450e575 100644 --- a/internal/database/sqlcommon/pin_sql.go +++ b/internal/database/sqlcommon/pin_sql.go @@ -30,6 +30,7 @@ import ( var ( pinColumns = []string{ + "namespace", "masked", "hash", "batch_id", @@ -93,7 +94,7 @@ func (s *SQLCommon) attemptPinInsert(ctx context.Context, tx *txWrapper, pin *co s.setPinInsertValues(sq.Insert(pinsTable).Columns(pinColumns...), pin), func() { log.L(ctx).Debugf("Triggering creation event for pin %d", pin.Sequence) - s.callbacks.OrderedCollectionEvent(database.CollectionPins, core.ChangeEventTypeCreated, pin.Sequence) + s.callbacks.OrderedCollectionNSEvent(database.CollectionPins, core.ChangeEventTypeCreated, pin.Namespace, pin.Sequence) }, ) return err @@ -101,6 +102,7 @@ func (s *SQLCommon) attemptPinInsert(ctx context.Context, tx *txWrapper, pin *co func (s *SQLCommon) setPinInsertValues(query sq.InsertBuilder, pin *core.Pin) sq.InsertBuilder { return query.Values( + pin.Namespace, pin.Masked, pin.Hash, pin.Batch, @@ -128,7 +130,7 @@ func (s *SQLCommon) InsertPins(ctx context.Context, pins []*core.Pin) error { err := s.insertTxRows(ctx, pinsTable, tx, query, func() { for i, pin := range pins { pin.Sequence = sequences[i] - s.callbacks.OrderedCollectionEvent(database.CollectionPins, core.ChangeEventTypeCreated, pin.Sequence) + s.callbacks.OrderedCollectionNSEvent(database.CollectionPins, core.ChangeEventTypeCreated, pin.Namespace, pin.Sequence) } }, sequences, true /* we want the caller to be able to retry with individual upserts */) if err != nil { @@ -149,6 +151,7 @@ func (s *SQLCommon) InsertPins(ctx context.Context, pins []*core.Pin) error { func (s *SQLCommon) pinResult(ctx context.Context, row *sql.Rows) (*core.Pin, error) { pin := core.Pin{} err := row.Scan( + &pin.Namespace, &pin.Masked, &pin.Hash, &pin.Batch, @@ -218,24 +221,3 @@ func (s *SQLCommon) UpdatePins(ctx context.Context, filter database.Filter, upda return s.commitTx(ctx, tx, autoCommit) } - -func (s *SQLCommon) DeletePin(ctx context.Context, sequence int64) (err error) { - - ctx, tx, autoCommit, err := s.beginOrUseTx(ctx) - if err != nil { - return err - } - defer s.rollbackTx(ctx, tx, autoCommit) - - err = s.deleteTx(ctx, pinsTable, tx, sq.Delete(pinsTable).Where(sq.Eq{ - sequenceColumn: sequence, - }), - func() { - s.callbacks.OrderedCollectionEvent(database.CollectionPins, core.ChangeEventTypeDeleted, sequence) - }) - if err != nil { - return err - } - - return s.commitTx(ctx, tx, autoCommit) -} diff --git a/internal/database/sqlcommon/pin_sql_test.go b/internal/database/sqlcommon/pin_sql_test.go index 59c135307d..b549f2f777 100644 --- a/internal/database/sqlcommon/pin_sql_test.go +++ b/internal/database/sqlcommon/pin_sql_test.go @@ -39,6 +39,7 @@ func TestPinsE2EWithDB(t *testing.T) { // Create a new pin entry pin := &core.Pin{ + Namespace: "ns", Masked: true, Hash: fftypes.NewRandB32(), Batch: fftypes.NewUUID(), @@ -49,8 +50,7 @@ func TestPinsE2EWithDB(t *testing.T) { Dispatched: false, } - s.callbacks.On("OrderedCollectionEvent", database.CollectionPins, core.ChangeEventTypeCreated, mock.Anything).Return() - s.callbacks.On("OrderedCollectionEvent", database.CollectionPins, core.ChangeEventTypeDeleted, mock.Anything).Return() + s.callbacks.On("OrderedCollectionNSEvent", database.CollectionPins, core.ChangeEventTypeCreated, "ns", mock.Anything).Return() err := s.UpsertPin(ctx, pin) assert.NoError(t, err) @@ -83,13 +83,6 @@ func TestPinsE2EWithDB(t *testing.T) { assert.Equal(t, existingSequence, pin.Sequence) assert.True(t, pin.Dispatched) - // Test delete - err = s.DeletePin(ctx, pin.Sequence) - assert.NoError(t, err) - p, _, err := s.GetPins(ctx, filter) - assert.NoError(t, err) - assert.Equal(t, 0, len(p)) - s.callbacks.AssertExpectations(t) } @@ -157,10 +150,10 @@ func TestInsertPinsMultiRowOK(t *testing.T) { s.features.MultiRowInsert = true s.fakePSQLInsert = true - pin1 := &core.Pin{Hash: fftypes.NewRandB32()} - pin2 := &core.Pin{Hash: fftypes.NewRandB32()} - s.callbacks.On("OrderedCollectionEvent", database.CollectionPins, core.ChangeEventTypeCreated, int64(1001)) - s.callbacks.On("OrderedCollectionEvent", database.CollectionPins, core.ChangeEventTypeCreated, int64(1002)) + pin1 := &core.Pin{Namespace: "ns1", Hash: fftypes.NewRandB32()} + pin2 := &core.Pin{Namespace: "ns1", Hash: fftypes.NewRandB32()} + s.callbacks.On("OrderedCollectionNSEvent", database.CollectionPins, core.ChangeEventTypeCreated, "ns1", int64(1001)) + s.callbacks.On("OrderedCollectionNSEvent", database.CollectionPins, core.ChangeEventTypeCreated, "ns1", int64(1002)) mock.ExpectBegin() mock.ExpectQuery("INSERT.*").WillReturnRows(sqlmock.NewRows([]string{sequenceColumn}). @@ -258,19 +251,3 @@ func TestUpdatePinsBadUpdate(t *testing.T) { err := s.UpdatePins(ctx, database.PinQueryFactory.NewFilter(ctx).Eq("bad", 1), database.PinQueryFactory.NewUpdate(ctx).Set("dispatched", true)) assert.Regexp(t, "FF00142", err) } - -func TestPinDeleteBeginFail(t *testing.T) { - s, mock := newMockProvider().init() - mock.ExpectBegin().WillReturnError(fmt.Errorf("pop")) - err := s.DeletePin(context.Background(), 12345) - assert.Regexp(t, "FF10114", err) -} - -func TestPinDeleteFail(t *testing.T) { - s, mock := newMockProvider().init() - mock.ExpectBegin() - mock.ExpectExec("DELETE .*").WillReturnError(fmt.Errorf("pop")) - mock.ExpectRollback() - err := s.DeletePin(context.Background(), 12345) - assert.Regexp(t, "FF10118", err) -} diff --git a/internal/events/batch_pin_complete.go b/internal/events/batch_pin_complete.go index 799722ec7c..e9f3f2b6b8 100644 --- a/internal/events/batch_pin_complete.go +++ b/internal/events/batch_pin_complete.go @@ -99,6 +99,7 @@ func (em *eventManager) persistContexts(ctx context.Context, batchPin *blockchai pins := make([]*core.Pin, len(batchPin.Contexts)) for idx, hash := range batchPin.Contexts { pins[idx] = &core.Pin{ + Namespace: batchPin.Namespace, Masked: private, Hash: hash, Batch: batchPin.BatchID, diff --git a/internal/orchestrator/data_query.go b/internal/orchestrator/data_query.go index 5dbee521a9..5b33bafe2f 100644 --- a/internal/orchestrator/data_query.go +++ b/internal/orchestrator/data_query.go @@ -377,7 +377,8 @@ func (or *orchestrator) GetTransactionBlockchainEvents(ctx context.Context, ns, return or.databases["database_0"].GetBlockchainEvents(ctx, filter) } -func (or *orchestrator) GetPins(ctx context.Context, filter database.AndFilter) ([]*core.Pin, *database.FilterResult, error) { +func (or *orchestrator) GetPins(ctx context.Context, ns string, filter database.AndFilter) ([]*core.Pin, *database.FilterResult, error) { + filter = or.scopeNS(ns, filter) return or.databases["database_0"].GetPins(ctx, filter) } diff --git a/internal/orchestrator/data_query_test.go b/internal/orchestrator/data_query_test.go index 71fef2b6f6..b91ce9b2f4 100644 --- a/internal/orchestrator/data_query_test.go +++ b/internal/orchestrator/data_query_test.go @@ -837,6 +837,6 @@ func TestGetPins(t *testing.T) { or.mdi.On("GetPins", mock.Anything, mock.Anything).Return([]*core.Pin{}, nil, nil) fb := database.PinQueryFactory.NewFilter(context.Background()) f := fb.And(fb.Eq("hash", u)) - _, _, err := or.GetPins(context.Background(), f) + _, _, err := or.GetPins(context.Background(), "ns1", f) assert.NoError(t, err) } diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go index 12c527ce83..a6e558e0f5 100644 --- a/internal/orchestrator/orchestrator.go +++ b/internal/orchestrator/orchestrator.go @@ -133,7 +133,7 @@ type Orchestrator interface { GetEventsWithReferences(ctx context.Context, ns string, filter database.AndFilter) ([]*core.EnrichedEvent, *database.FilterResult, error) GetBlockchainEventByID(ctx context.Context, ns, id string) (*core.BlockchainEvent, error) GetBlockchainEvents(ctx context.Context, ns string, filter database.AndFilter) ([]*core.BlockchainEvent, *database.FilterResult, error) - GetPins(ctx context.Context, filter database.AndFilter) ([]*core.Pin, *database.FilterResult, error) + GetPins(ctx context.Context, ns string, filter database.AndFilter) ([]*core.Pin, *database.FilterResult, error) // Charts GetChartHistogram(ctx context.Context, ns string, startTime int64, endTime int64, buckets int64, tableName database.CollectionName) ([]*core.ChartHistogram, error) diff --git a/internal/orchestrator/persistence_events.go b/internal/orchestrator/persistence_events.go index 7a38e9db08..ab084913eb 100644 --- a/internal/orchestrator/persistence_events.go +++ b/internal/orchestrator/persistence_events.go @@ -43,13 +43,14 @@ func (or *orchestrator) OrderedUUIDCollectionNSEvent(resType database.OrderedUUI }) } -func (or *orchestrator) OrderedCollectionEvent(resType database.OrderedCollection, eventType core.ChangeEventType, sequence int64) { +func (or *orchestrator) OrderedCollectionNSEvent(resType database.OrderedCollectionNS, eventType core.ChangeEventType, ns string, sequence int64) { if eventType == core.ChangeEventTypeCreated && resType == database.CollectionPins { or.events.NewPins() <- sequence } or.adminEvents.Dispatch(&core.ChangeEvent{ Collection: string(resType), Type: eventType, + Namespace: ns, Sequence: &sequence, }) } diff --git a/internal/orchestrator/persistence_events_test.go b/internal/orchestrator/persistence_events_test.go index dd7fdc51bb..dd161f8606 100644 --- a/internal/orchestrator/persistence_events_test.go +++ b/internal/orchestrator/persistence_events_test.go @@ -51,7 +51,7 @@ func TestPinCreated(t *testing.T) { } mae.On("Dispatch", mock.Anything).Return() mem.On("NewPins").Return((chan<- int64)(make(chan int64, 1))) - o.OrderedCollectionEvent(database.CollectionPins, core.ChangeEventTypeCreated, 12345) + o.OrderedCollectionNSEvent(database.CollectionPins, core.ChangeEventTypeCreated, "ns1", 12345) mem.AssertExpectations(t) mae.AssertExpectations(t) } diff --git a/mocks/databasemocks/callbacks.go b/mocks/databasemocks/callbacks.go index 78fd443f05..0f4428d815 100644 --- a/mocks/databasemocks/callbacks.go +++ b/mocks/databasemocks/callbacks.go @@ -21,9 +21,9 @@ func (_m *Callbacks) HashCollectionNSEvent(resType database.HashCollectionNS, ev _m.Called(resType, eventType, ns, hash) } -// OrderedCollectionEvent provides a mock function with given fields: resType, eventType, sequence -func (_m *Callbacks) OrderedCollectionEvent(resType database.OrderedCollection, eventType core.ChangeEventType, sequence int64) { - _m.Called(resType, eventType, sequence) +// OrderedCollectionNSEvent provides a mock function with given fields: resType, eventType, ns, sequence +func (_m *Callbacks) OrderedCollectionNSEvent(resType database.OrderedCollectionNS, eventType core.ChangeEventType, ns string, sequence int64) { + _m.Called(resType, eventType, ns, sequence) } // OrderedUUIDCollectionNSEvent provides a mock function with given fields: resType, eventType, ns, id, sequence diff --git a/mocks/databasemocks/plugin.go b/mocks/databasemocks/plugin.go index 2d42e849a9..fa5290fecc 100644 --- a/mocks/databasemocks/plugin.go +++ b/mocks/databasemocks/plugin.go @@ -135,20 +135,6 @@ func (_m *Plugin) DeleteOffset(ctx context.Context, t fftypes.FFEnum, name strin return r0 } -// DeletePin provides a mock function with given fields: ctx, sequence -func (_m *Plugin) DeletePin(ctx context.Context, sequence int64) error { - ret := _m.Called(ctx, sequence) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { - r0 = rf(ctx, sequence) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // DeleteSubscriptionByID provides a mock function with given fields: ctx, id func (_m *Plugin) DeleteSubscriptionByID(ctx context.Context, id *fftypes.UUID) error { ret := _m.Called(ctx, id) diff --git a/mocks/orchestratormocks/orchestrator.go b/mocks/orchestratormocks/orchestrator.go index c30c600f8a..90ce8fb9c6 100644 --- a/mocks/orchestratormocks/orchestrator.go +++ b/mocks/orchestratormocks/orchestrator.go @@ -964,13 +964,13 @@ func (_m *Orchestrator) GetOperationsNamespaced(ctx context.Context, ns string, return r0, r1, r2 } -// GetPins provides a mock function with given fields: ctx, filter -func (_m *Orchestrator) GetPins(ctx context.Context, filter database.AndFilter) ([]*core.Pin, *database.FilterResult, error) { - ret := _m.Called(ctx, filter) +// GetPins provides a mock function with given fields: ctx, ns, filter +func (_m *Orchestrator) GetPins(ctx context.Context, ns string, filter database.AndFilter) ([]*core.Pin, *database.FilterResult, error) { + ret := _m.Called(ctx, ns, filter) var r0 []*core.Pin - if rf, ok := ret.Get(0).(func(context.Context, database.AndFilter) []*core.Pin); ok { - r0 = rf(ctx, filter) + if rf, ok := ret.Get(0).(func(context.Context, string, database.AndFilter) []*core.Pin); ok { + r0 = rf(ctx, ns, filter) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*core.Pin) @@ -978,8 +978,8 @@ func (_m *Orchestrator) GetPins(ctx context.Context, filter database.AndFilter) } var r1 *database.FilterResult - if rf, ok := ret.Get(1).(func(context.Context, database.AndFilter) *database.FilterResult); ok { - r1 = rf(ctx, filter) + if rf, ok := ret.Get(1).(func(context.Context, string, database.AndFilter) *database.FilterResult); ok { + r1 = rf(ctx, ns, filter) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(*database.FilterResult) @@ -987,8 +987,8 @@ func (_m *Orchestrator) GetPins(ctx context.Context, filter database.AndFilter) } var r2 error - if rf, ok := ret.Get(2).(func(context.Context, database.AndFilter) error); ok { - r2 = rf(ctx, filter) + if rf, ok := ret.Get(2).(func(context.Context, string, database.AndFilter) error); ok { + r2 = rf(ctx, ns, filter) } else { r2 = ret.Error(2) } diff --git a/pkg/core/pin.go b/pkg/core/pin.go index a2a6fd0caf..62c8d23c05 100644 --- a/pkg/core/pin.go +++ b/pkg/core/pin.go @@ -40,6 +40,7 @@ import "github.com/hyperledger/firefly-common/pkg/fftypes" // type Pin struct { Sequence int64 `ffstruct:"Pin" json:"sequence"` + Namespace string `ffstruct:"Pin" json:"namespace"` Masked bool `ffstruct:"Pin" json:"masked,omitempty"` Hash *fftypes.Bytes32 `ffstruct:"Pin" json:"hash,omitempty"` Batch *fftypes.UUID `ffstruct:"Pin" json:"batch,omitempty"` diff --git a/pkg/database/plugin.go b/pkg/database/plugin.go index 57bee634de..c589ea8240 100644 --- a/pkg/database/plugin.go +++ b/pkg/database/plugin.go @@ -210,9 +210,6 @@ type iPinCollection interface { // UpdatePins - Updates pins UpdatePins(ctx context.Context, filter Filter, update Update) (err error) - - // DeletePin - Delete a pin - DeletePin(ctx context.Context, sequence int64) (err error) } type iOperationCollection interface { @@ -602,11 +599,11 @@ const ( CollectionEvents OrderedUUIDCollectionNS = "events" ) -// OrderedCollection is a collection that is ordered, and that sequence is the only key -type OrderedCollection CollectionName +// OrderedCollectionNS is a collection that is ordered, and that sequence is the only key +type OrderedCollectionNS CollectionName const ( - CollectionPins OrderedCollection = "pins" + CollectionPins OrderedCollectionNS = "pins" ) // UUIDCollectionNS is the most common type of collection - each entry has a UUID that @@ -691,7 +688,7 @@ type PostCompletionHook func() type Callbacks interface { // OrderedUUIDCollectionNSEvent emits the sequence on insert, but it will be -1 on update OrderedUUIDCollectionNSEvent(resType OrderedUUIDCollectionNS, eventType core.ChangeEventType, ns string, id *fftypes.UUID, sequence int64) - OrderedCollectionEvent(resType OrderedCollection, eventType core.ChangeEventType, sequence int64) + OrderedCollectionNSEvent(resType OrderedCollectionNS, eventType core.ChangeEventType, ns string, sequence int64) UUIDCollectionNSEvent(resType UUIDCollectionNS, eventType core.ChangeEventType, ns string, id *fftypes.UUID) UUIDCollectionEvent(resType UUIDCollection, eventType core.ChangeEventType, id *fftypes.UUID) HashCollectionNSEvent(resType HashCollectionNS, eventType core.ChangeEventType, ns string, hash *fftypes.Bytes32) From fd74b33961ef4a1bd101a0a73d585545722cd306 Mon Sep 17 00:00:00 2001 From: Andrew Richardson Date: Thu, 9 Jun 2022 12:40:07 -0400 Subject: [PATCH 2/2] Rename /status/pins to just /pins Signed-off-by: Andrew Richardson --- docs/swagger/swagger.yaml | 594 +++++++++--------- ...e_get_status_pins.go => route_get_pins.go} | 8 +- ...us_pins_test.go => route_get_pins_test.go} | 4 +- internal/apiserver/routes.go | 2 +- internal/coremsgs/en_api_translations.go | 2 +- 5 files changed, 305 insertions(+), 305 deletions(-) rename internal/apiserver/{route_get_status_pins.go => route_get_pins.go} (89%) rename internal/apiserver/{route_get_status_pins_test.go => route_get_pins_test.go} (91%) diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml index 1ff69d72eb..68f2345414 100644 --- a/docs/swagger/swagger.yaml +++ b/docs/swagger/swagger.yaml @@ -19921,6 +19921,158 @@ paths: description: "" tags: - Non-Default Namespace + /namespaces/{ns}/pins: + get: + description: Queries the list of pins received from the blockchain + operationId: getPinsNamespace + parameters: + - description: The namespace which scopes this request + in: path + name: ns + required: true + schema: + example: default + type: string + - description: Server-side request timeout (milliseconds, or set a custom suffix + like 10s) + in: header + name: Request-Timeout + schema: + default: 2m0s + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: batch + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: created + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: dispatched + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: hash + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: index + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: masked + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: sequence + schema: + type: string + - description: Sort field. For multi-field sort use comma separated values (or + multiple query values) with '-' prefix for descending + in: query + name: sort + schema: + type: string + - description: Ascending sort order (overrides all fields in a multi-field sort) + in: query + name: ascending + schema: + type: string + - description: Descending sort order (overrides all fields in a multi-field + sort) + in: query + name: descending + schema: + type: string + - description: 'The number of records to skip (max: 1,000). Unsuitable for bulk + operations' + in: query + name: skip + schema: + type: string + - description: 'The maximum number of records to return (max: 1,000)' + in: query + name: limit + schema: + example: "25" + type: string + - description: Return a total count as well as items (adds extra database processing) + in: query + name: count + schema: + type: string + responses: + "200": + content: + application/json: + schema: + items: + properties: + batch: + description: The UUID of the batch of messages this pin is part + of + format: uuid + type: string + batchHash: + description: The manifest hash batch of messages this pin is + part of + format: byte + type: string + created: + description: The time the FireFly node created the pin + format: date-time + type: string + dispatched: + description: Once true, this pin has been processed and will + not be processed again + type: boolean + hash: + description: The hash represents a topic within a message in + the batch. If a message has multiple topics, then multiple + pins are created. If the message is private, the hash is masked + for privacy + format: byte + type: string + index: + description: The index of this pin within the batch. One pin + is created for each topic, of each message in the batch + format: int64 + type: integer + masked: + description: True if the pin is for a private message, and hence + is masked with the group ID and salted with a nonce so observers + of the blockchain cannot use pin hash to match this transaction + to other transactions or participants + type: boolean + namespace: + description: The namespace of the pin + type: string + sequence: + description: The order of the pin in the local FireFly database, + which matches the order in which pins were delivered to FireFly + by the blockchain connector event stream + format: int64 + type: integer + signer: + description: The blockchain signing key that submitted this + transaction, as passed through to FireFly by the smart contract + that emitted the blockchain event + type: string + type: object + type: array + description: Success + default: + description: "" + tags: + - Non-Default Namespace /namespaces/{ns}/status: get: description: Gets the status of this node @@ -20105,158 +20257,6 @@ paths: description: "" tags: - Non-Default Namespace - /namespaces/{ns}/status/pins: - get: - description: Queries the pins table that is the status of the event aggregator - operationId: getStatusPinsNamespace - parameters: - - description: The namespace which scopes this request - in: path - name: ns - required: true - schema: - example: default - type: string - - description: Server-side request timeout (milliseconds, or set a custom suffix - like 10s) - in: header - name: Request-Timeout - schema: - default: 2m0s - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: batch - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: created - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: dispatched - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: hash - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: index - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: masked - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: sequence - schema: - type: string - - description: Sort field. For multi-field sort use comma separated values (or - multiple query values) with '-' prefix for descending - in: query - name: sort - schema: - type: string - - description: Ascending sort order (overrides all fields in a multi-field sort) - in: query - name: ascending - schema: - type: string - - description: Descending sort order (overrides all fields in a multi-field - sort) - in: query - name: descending - schema: - type: string - - description: 'The number of records to skip (max: 1,000). Unsuitable for bulk - operations' - in: query - name: skip - schema: - type: string - - description: 'The maximum number of records to return (max: 1,000)' - in: query - name: limit - schema: - example: "25" - type: string - - description: Return a total count as well as items (adds extra database processing) - in: query - name: count - schema: - type: string - responses: - "200": - content: - application/json: - schema: - items: - properties: - batch: - description: The UUID of the batch of messages this pin is part - of - format: uuid - type: string - batchHash: - description: The manifest hash batch of messages this pin is - part of - format: byte - type: string - created: - description: The time the FireFly node created the pin - format: date-time - type: string - dispatched: - description: Once true, this pin has been processed and will - not be processed again - type: boolean - hash: - description: The hash represents a topic within a message in - the batch. If a message has multiple topics, then multiple - pins are created. If the message is private, the hash is masked - for privacy - format: byte - type: string - index: - description: The index of this pin within the batch. One pin - is created for each topic, of each message in the batch - format: int64 - type: integer - masked: - description: True if the pin is for a private message, and hence - is masked with the group ID and salted with a nonce so observers - of the blockchain cannot use pin hash to match this transaction - to other transactions or participants - type: boolean - namespace: - description: The namespace of the pin - type: string - sequence: - description: The order of the pin in the local FireFly database, - which matches the order in which pins were delivered to FireFly - by the blockchain connector event stream - format: int64 - type: integer - signer: - description: The blockchain signing key that submitted this - transaction, as passed through to FireFly by the smart contract - that emitted the blockchain event - type: string - type: object - type: array - description: Success - default: - description: "" - tags: - - Non-Default Namespace /namespaces/{ns}/subscriptions: get: description: Gets a list of subscriptions @@ -27323,6 +27323,151 @@ paths: description: "" tags: - Default Namespace + /pins: + get: + description: Queries the list of pins received from the blockchain + operationId: getPins + parameters: + - description: Server-side request timeout (milliseconds, or set a custom suffix + like 10s) + in: header + name: Request-Timeout + schema: + default: 2m0s + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: batch + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: created + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: dispatched + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: hash + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: index + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: masked + schema: + type: string + - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' + in: query + name: sequence + schema: + type: string + - description: Sort field. For multi-field sort use comma separated values (or + multiple query values) with '-' prefix for descending + in: query + name: sort + schema: + type: string + - description: Ascending sort order (overrides all fields in a multi-field sort) + in: query + name: ascending + schema: + type: string + - description: Descending sort order (overrides all fields in a multi-field + sort) + in: query + name: descending + schema: + type: string + - description: 'The number of records to skip (max: 1,000). Unsuitable for bulk + operations' + in: query + name: skip + schema: + type: string + - description: 'The maximum number of records to return (max: 1,000)' + in: query + name: limit + schema: + example: "25" + type: string + - description: Return a total count as well as items (adds extra database processing) + in: query + name: count + schema: + type: string + responses: + "200": + content: + application/json: + schema: + items: + properties: + batch: + description: The UUID of the batch of messages this pin is part + of + format: uuid + type: string + batchHash: + description: The manifest hash batch of messages this pin is + part of + format: byte + type: string + created: + description: The time the FireFly node created the pin + format: date-time + type: string + dispatched: + description: Once true, this pin has been processed and will + not be processed again + type: boolean + hash: + description: The hash represents a topic within a message in + the batch. If a message has multiple topics, then multiple + pins are created. If the message is private, the hash is masked + for privacy + format: byte + type: string + index: + description: The index of this pin within the batch. One pin + is created for each topic, of each message in the batch + format: int64 + type: integer + masked: + description: True if the pin is for a private message, and hence + is masked with the group ID and salted with a nonce so observers + of the blockchain cannot use pin hash to match this transaction + to other transactions or participants + type: boolean + namespace: + description: The namespace of the pin + type: string + sequence: + description: The order of the pin in the local FireFly database, + which matches the order in which pins were delivered to FireFly + by the blockchain connector event stream + format: int64 + type: integer + signer: + description: The blockchain signing key that submitted this + transaction, as passed through to FireFly by the smart contract + that emitted the blockchain event + type: string + type: object + type: array + description: Success + default: + description: "" + tags: + - Default Namespace /status: get: description: Gets the status of this node @@ -27592,151 +27737,6 @@ paths: description: "" tags: - Global - /status/pins: - get: - description: Queries the pins table that is the status of the event aggregator - operationId: getStatusPins - parameters: - - description: Server-side request timeout (milliseconds, or set a custom suffix - like 10s) - in: header - name: Request-Timeout - schema: - default: 2m0s - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: batch - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: created - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: dispatched - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: hash - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: index - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: masked - schema: - type: string - - description: 'Data filter field. Prefixes supported: > >= < <= @ ^ ! !@ !^' - in: query - name: sequence - schema: - type: string - - description: Sort field. For multi-field sort use comma separated values (or - multiple query values) with '-' prefix for descending - in: query - name: sort - schema: - type: string - - description: Ascending sort order (overrides all fields in a multi-field sort) - in: query - name: ascending - schema: - type: string - - description: Descending sort order (overrides all fields in a multi-field - sort) - in: query - name: descending - schema: - type: string - - description: 'The number of records to skip (max: 1,000). Unsuitable for bulk - operations' - in: query - name: skip - schema: - type: string - - description: 'The maximum number of records to return (max: 1,000)' - in: query - name: limit - schema: - example: "25" - type: string - - description: Return a total count as well as items (adds extra database processing) - in: query - name: count - schema: - type: string - responses: - "200": - content: - application/json: - schema: - items: - properties: - batch: - description: The UUID of the batch of messages this pin is part - of - format: uuid - type: string - batchHash: - description: The manifest hash batch of messages this pin is - part of - format: byte - type: string - created: - description: The time the FireFly node created the pin - format: date-time - type: string - dispatched: - description: Once true, this pin has been processed and will - not be processed again - type: boolean - hash: - description: The hash represents a topic within a message in - the batch. If a message has multiple topics, then multiple - pins are created. If the message is private, the hash is masked - for privacy - format: byte - type: string - index: - description: The index of this pin within the batch. One pin - is created for each topic, of each message in the batch - format: int64 - type: integer - masked: - description: True if the pin is for a private message, and hence - is masked with the group ID and salted with a nonce so observers - of the blockchain cannot use pin hash to match this transaction - to other transactions or participants - type: boolean - namespace: - description: The namespace of the pin - type: string - sequence: - description: The order of the pin in the local FireFly database, - which matches the order in which pins were delivered to FireFly - by the blockchain connector event stream - format: int64 - type: integer - signer: - description: The blockchain signing key that submitted this - transaction, as passed through to FireFly by the smart contract - that emitted the blockchain event - type: string - type: object - type: array - description: Success - default: - description: "" - tags: - - Default Namespace /status/websockets: get: description: Gets the status of the current WebSocket connections to this node diff --git a/internal/apiserver/route_get_status_pins.go b/internal/apiserver/route_get_pins.go similarity index 89% rename from internal/apiserver/route_get_status_pins.go rename to internal/apiserver/route_get_pins.go index d44242b979..17e8a3f5be 100644 --- a/internal/apiserver/route_get_status_pins.go +++ b/internal/apiserver/route_get_pins.go @@ -25,13 +25,13 @@ import ( "github.com/hyperledger/firefly/pkg/database" ) -var getStatusPins = &ffapi.Route{ - Name: "getStatusPins", - Path: "status/pins", +var getPins = &ffapi.Route{ + Name: "getPins", + Path: "pins", Method: http.MethodGet, PathParams: nil, QueryParams: nil, - Description: coremsgs.APIEndpointsGetStatusPins, + Description: coremsgs.APIEndpointsGetPins, JSONInputValue: nil, JSONOutputValue: func() interface{} { return []core.Pin{} }, JSONOutputCodes: []int{http.StatusOK}, diff --git a/internal/apiserver/route_get_status_pins_test.go b/internal/apiserver/route_get_pins_test.go similarity index 91% rename from internal/apiserver/route_get_status_pins_test.go rename to internal/apiserver/route_get_pins_test.go index 9a555ec709..b43e18f1e8 100644 --- a/internal/apiserver/route_get_status_pins_test.go +++ b/internal/apiserver/route_get_pins_test.go @@ -25,9 +25,9 @@ import ( "github.com/stretchr/testify/mock" ) -func TestGetStatusPins(t *testing.T) { +func TestGetPins(t *testing.T) { o, r := newTestAPIServer() - req := httptest.NewRequest("GET", "/api/v1/status/pins", nil) + req := httptest.NewRequest("GET", "/api/v1/pins", nil) req.Header.Set("Content-Type", "application/json; charset=utf-8") res := httptest.NewRecorder() diff --git a/internal/apiserver/routes.go b/internal/apiserver/routes.go index 49e2f5d198..67394f2d36 100644 --- a/internal/apiserver/routes.go +++ b/internal/apiserver/routes.go @@ -94,7 +94,7 @@ var routes = append( getOpByID, getOps, getStatus, - getStatusPins, + getPins, getSubscriptionByID, getSubscriptions, getTokenAccountPools, diff --git a/internal/coremsgs/en_api_translations.go b/internal/coremsgs/en_api_translations.go index 74f0229aa0..d5f4f9eaf1 100644 --- a/internal/coremsgs/en_api_translations.go +++ b/internal/coremsgs/en_api_translations.go @@ -123,7 +123,7 @@ var ( APIEndpointsGetOpByID = ffm("api.endpoints.getOpByID", "Gets an operation by ID") APIEndpointsGetOps = ffm("api.endpoints.getOps", "Gets a a list of operations") APIEndpointsGetStatusBatchManager = ffm("api.endpoints.getStatusBatchManager", "Gets the status of the batch manager") - APIEndpointsGetStatusPins = ffm("api.endpoints.getStatusPins", "Queries the pins table that is the status of the event aggregator") + APIEndpointsGetPins = ffm("api.endpoints.getPins", "Queries the list of pins received from the blockchain") APIEndpointsGetStatusWebSockets = ffm("api.endpoints.getStatusWebSockets", "Gets the status of the current WebSocket connections to this node") APIEndpointsGetStatus = ffm("api.endpoints.getStatus", "Gets the status of this node") APIEndpointsGetSubscriptionByID = ffm("api.endpoints.getSubscriptionByID", "Gets a subscription by its ID")