Decode BSON documents correctly in API responses (#32)

Previously the BSON documents in the result sets were decoded as
bson.D when stored in EiffelEvent.Data. This meant that the original
objects turned up as slices of objects,

    [{"Key": "name", "Value": "foo"]

instead of objects,

    {"name": "foo"}

which obviously meant that they were no longer valid Eiffel events.
We address this by deriving the EiffelEvent data type from
map[string]interface{} which is the same underlying type as bson.M,
the unordered map type.

This made the internal/schema package more or less empty so the
remaining EiffelEvent definition was moved to internal/database/drivers
where the database driver interface is defined.

When we implement other API endpoints we'll need to access members
of the Eiffel events in the responses and there are a few different
ways of doing that, but let's deal with that later.

The projections of the MongoDB queries were also adjusted to omit the
unwanted "_id" field.

Adding eiffelevents-sdk-go and running "go mod tidy" resulted in an
extra "require" section in go.mod. Not entirely sure why.

go.mod

@@ -13,22 +13,36 @@ require (
 )
 
 require (
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Showmax/go-fqdn v1.0.0 // indirect
+	github.com/clarketm/json v1.17.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/klauspost/compress v1.9.5 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/tidwall/gjson v1.9.4 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.0.2 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect
-	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 // indirect
-	golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2 // indirect
+	golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect
+	golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79 // indirect
 	golang.org/x/text v0.3.5 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+)
+
+require (
+	github.com/eiffel-community/eiffelevents-sdk-go v0.0.0-20220128085857-41fb1ce1ccc2
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )

internal/database/drivers/drivers.go

@@ -22,18 +22,19 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"github.com/eiffel-community/eiffel-goer/internal/requests"
-	"github.com/eiffel-community/eiffel-goer/internal/schema"
 )
 
+type EiffelEvent map[string]interface{}
+
 type DatabaseDriver interface {
 	Get(context.Context, *url.URL, *log.Entry) (Database, error)
 	SupportsScheme(string) bool
 }
 
 type Database interface {
-	GetEvents(context.Context, requests.MultipleEventsRequest) ([]schema.EiffelEvent, error)
-	SearchEvent(context.Context, string) (schema.EiffelEvent, error)
-	UpstreamDownstreamSearch(context.Context, string) ([]schema.EiffelEvent, error)
-	GetEventByID(context.Context, string) (schema.EiffelEvent, error)
+	GetEvents(context.Context, requests.MultipleEventsRequest) ([]EiffelEvent, error)
+	SearchEvent(context.Context, string) (EiffelEvent, error)
+	UpstreamDownstreamSearch(context.Context, string) ([]EiffelEvent, error)
+	GetEventByID(context.Context, string) (EiffelEvent, error)
 	Close(context.Context) error
 }

internal/database/drivers/mongodb/mongodb.go

@@ -36,7 +36,6 @@ import (
 	"github.com/eiffel-community/eiffel-goer/internal/database/drivers"
 	"github.com/eiffel-community/eiffel-goer/internal/query"
 	"github.com/eiffel-community/eiffel-goer/internal/requests"
-	"github.com/eiffel-community/eiffel-goer/internal/schema"
 )
 
 // Database is a MongoDB database connection.
@@ -155,7 +154,7 @@ func (m *Database) collections(ctx context.Context, filter bson.D) ([]string, er
 }
 
 // GetEvents gets all events information.
-func (m *Database) GetEvents(ctx context.Context, request requests.MultipleEventsRequest) ([]schema.EiffelEvent, error) {
+func (m *Database) GetEvents(ctx context.Context, request requests.MultipleEventsRequest) ([]drivers.EiffelEvent, error) {
 	filter, err := buildFilter(request.Conditions)
 	if err != nil {
 		m.logger.Errorf("Database: %v", err)
@@ -168,10 +167,12 @@ func (m *Database) GetEvents(ctx context.Context, request requests.MultipleEvent
 	}
 
 	m.logger.Debugf("fetching events from %d collections", len(collections))
-	var allEvents []schema.EiffelEvent
+	var allEvents []drivers.EiffelEvent
 	for _, collection := range collections {
-		var events []schema.EiffelEvent
-		cursor, err := m.database.Collection(collection).Find(ctx, filter)
+		var events []drivers.EiffelEvent
+		cursor, err := m.database.Collection(collection).Find(ctx, filter,
+			// Remove the _id field from the resulting document.
+			options.Find().SetProjection(bson.M{"_id": 0}))
 		if err != nil {
 			continue
 		}
@@ -185,33 +186,35 @@ func (m *Database) GetEvents(ctx context.Context, request requests.MultipleEvent
 }
 
 // SearchEvent searches for an event based on event ID.
-func (m *Database) SearchEvent(ctx context.Context, id string) (schema.EiffelEvent, error) {
-	return schema.EiffelEvent{}, errors.New("not yet implemented")
+func (m *Database) SearchEvent(ctx context.Context, id string) (drivers.EiffelEvent, error) {
+	return drivers.EiffelEvent{}, errors.New("not yet implemented")
 }
 
 // UpstreamDownstreamSearch searches for events upstream and/or downstream of event by ID.
-func (m *Database) UpstreamDownstreamSearch(ctx context.Context, id string) ([]schema.EiffelEvent, error) {
+func (m *Database) UpstreamDownstreamSearch(ctx context.Context, id string) ([]drivers.EiffelEvent, error) {
 	return nil, errors.New("not yet implemented")
 }
 
 // GetEventByID gets an event by ID in all collections.
-func (m *Database) GetEventByID(ctx context.Context, id string) (schema.EiffelEvent, error) {
+func (m *Database) GetEventByID(ctx context.Context, id string) (drivers.EiffelEvent, error) {
 	collections, err := m.collections(ctx, bson.D{})
 	if err != nil {
-		return schema.EiffelEvent{}, err
+		return nil, err
 	}
 	filter := bson.D{{Key: "meta.id", Value: id}}
 	for _, collection := range collections {
-		var event schema.EiffelEvent
-		singleResult := m.database.Collection(collection).FindOne(ctx, filter)
+		var event bson.M
+		singleResult := m.database.Collection(collection).FindOne(ctx, filter,
+			// Remove the _id field from the resulting document.
+			options.FindOne().SetProjection(bson.M{"_id": 0}))
 		err := singleResult.Decode(&event)
 		if err != nil {
 			continue
 		} else {
-			return event, nil
+			return drivers.EiffelEvent(event), nil
 		}
 	}
-	return schema.EiffelEvent{}, fmt.Errorf("%q not found in any collection", id)
+	return nil, fmt.Errorf("%q not found in any collection", id)
 }
 
 // Close the database connection.

pkg/v1alpha1/api/api_test.go

@@ -28,7 +28,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/eiffel-community/eiffel-goer/internal/schema"
+	"github.com/eiffel-community/eiffel-goer/internal/database/drivers"
 	"github.com/eiffel-community/eiffel-goer/pkg/application"
 	"github.com/eiffel-community/eiffel-goer/test/mock_config"
 	"github.com/eiffel-community/eiffel-goer/test/mock_drivers"
@@ -56,9 +56,9 @@ func TestRoutes(t *testing.T) {
 	mockCfg.EXPECT().DBConnectionString().Return("").AnyTimes()
 	mockCfg.EXPECT().APIPort().Return(":8080").AnyTimes()
 	// Have to use 'gomock.Any()' for the context as mux adds values to the request context.
-	mockDB.EXPECT().GetEventByID(gomock.Any(), eventID).Return(schema.EiffelEvent{}, nil)
-	mockDB.EXPECT().GetEvents(gomock.Any(), gomock.Any()).Return([]schema.EiffelEvent{}, nil)
-	mockDB.EXPECT().UpstreamDownstreamSearch(gomock.Any(), "id").Return([]schema.EiffelEvent{}, nil)
+	mockDB.EXPECT().GetEventByID(gomock.Any(), eventID).Return(drivers.EiffelEvent{}, nil)
+	mockDB.EXPECT().GetEvents(gomock.Any(), gomock.Any()).Return([]drivers.EiffelEvent{}, nil)
+	mockDB.EXPECT().UpstreamDownstreamSearch(gomock.Any(), "id").Return([]drivers.EiffelEvent{}, nil)
 
 	for _, testCase := range tests {
 		t.Run(testCase.name, func(t *testing.T) {

pkg/v1alpha1/handlers/events/events.go

@@ -29,7 +29,6 @@ import (
 	"github.com/eiffel-community/eiffel-goer/internal/query"
 	"github.com/eiffel-community/eiffel-goer/internal/requests"
 	"github.com/eiffel-community/eiffel-goer/internal/responses"
-	eiffelSchema "github.com/eiffel-community/eiffel-goer/internal/schema"
 )
 
 type EventHandler struct {
@@ -75,10 +74,10 @@ func getTags(tagName string, item interface{}) map[string]struct{} {
 }
 
 type multiResponse struct {
-	PageNo           int32                      `json:"pageNo"`
-	PageSize         int32                      `json:"pageSize"`
-	TotalNumberItems int                        `json:"totalNumberItems"`
-	Items            []eiffelSchema.EiffelEvent `json:"items"`
+	PageNo           int32                 `json:"pageNo"`
+	PageSize         int32                 `json:"pageSize"`
+	TotalNumberItems int                   `json:"totalNumberItems"`
+	Items            []drivers.EiffelEvent `json:"items"`
 }
 
 // buildConditions takes a raw URL query, parses out all conditions and removes ignoreKeys.

pkg/v1alpha1/handlers/events/events_test.go

@@ -22,12 +22,14 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/eiffel-community/eiffelevents-sdk-go"
 	"github.com/golang/mock/gomock"
 	"github.com/gorilla/mux"
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
-	"github.com/eiffel-community/eiffel-goer/internal/schema"
+	"github.com/eiffel-community/eiffel-goer/internal/database/drivers"
 	"github.com/eiffel-community/eiffel-goer/test/mock_config"
 	"github.com/eiffel-community/eiffel-goer/test/mock_drivers"
 )
@@ -47,22 +49,17 @@ var activityJSON = []byte(`
 }
 `)
 
-func loadEvent() (schema.EiffelEvent, error) {
-	event := schema.EiffelEvent{}
-	err := json.Unmarshal(activityJSON, &event)
-	if err != nil {
-		return event, err
-	}
-	return event, nil
-}
-
 // Test that the events/{id} endpoint work as expected.
 func TestEvents(t *testing.T) {
-	event, err := loadEvent()
-	if err != nil {
-		t.Error(err)
-	}
-	eventID := event.Meta.ID
+	// Load the test event twice; once as a drivers.EiffelEvent and once as
+	// a proper struct via eiffelevents. The latter is only used to easily
+	// extract the event ID.
+	eventMap := make(drivers.EiffelEvent)
+	require.NoError(t, json.Unmarshal(activityJSON, &eventMap))
+	event, err := eiffelevents.UnmarshalAny(activityJSON)
+	require.NoError(t, err)
+	eventID := event.(eiffelevents.MetaTeller).ID()
+
 	badRequest := httptest.NewRequest(http.MethodGet, "/events/"+eventID, nil)
 	q := badRequest.URL.Query()
 	q.Add("nah", "hello")
@@ -85,7 +82,7 @@ func TestEvents(t *testing.T) {
 			ctrl := gomock.NewController(t)
 			mockCfg := mock_config.NewMockConfig(ctrl)
 			mockDB := mock_drivers.NewMockDatabase(ctrl)
-			mockDB.EXPECT().GetEventByID(gomock.Any(), eventID).Return(event, testCase.mockError)
+			mockDB.EXPECT().GetEventByID(gomock.Any(), eventID).Return(eventMap, testCase.mockError)
 			app := Get(mockCfg, mockDB, &log.Entry{})
 			handler := mux.NewRouter()
 			handler.HandleFunc("/events/{id:[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}}", app.Read)
@@ -94,9 +91,9 @@ func TestEvents(t *testing.T) {
 			handler.ServeHTTP(responseRecorder, testCase.request)
 
 			assert.Equalf(t, testCase.statusCode, responseRecorder.Code, "Input URL: %s", testCase.request.URL)
-			eventFromResponse := schema.EiffelEvent{}
-			assert.NoError(t, json.Unmarshal(responseRecorder.Body.Bytes(), &eventFromResponse))
-			assert.Equal(t, testCase.eventID, eventFromResponse.Meta.ID)
+			if responseRecorder.Code == http.StatusOK {
+				assert.JSONEq(t, string(activityJSON), responseRecorder.Body.String())
+			}
 		})
 	}
 }