diff --git a/Dockerfile b/Dockerfile index 7c1c561..f77eea5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,8 @@ RUN BUILDINFO_PACKAGE="${ORG_PATH}/service-status-go/buildinfo." \ && BUILDER="builder=$(go version)" \ && LDFLAGS="-s -w -X '"${BUILDINFO_PACKAGE}$VERSION"' -X '"${BUILDINFO_PACKAGE}$DATETIME"' -X '"${BUILDINFO_PACKAGE}$REPOSITORY"' -X '"${BUILDINFO_PACKAGE}$REVISION"' -X '"${BUILDINFO_PACKAGE}$BUILDER"'" \ && git config --global url."https://${GITHUB_USERNAME}:${GITHUB_TOKEN}@github.com".insteadOf "https://github.com" \ + && mkdir -p /artifacts/schemas/ \ + && cp -r /${SRC_FOLDER}/schemas /artifacts/schemas \ && CGO_ENABLED=0 go build -mod=readonly -a -o /artifacts/${PROJECT} -ldflags="${LDFLAGS}" COPY ./suggestion-config.json /artifacts/suggestion-config.json diff --git a/annotations/cypher.go b/annotations/cypher.go index 53a39f5..742bc25 100644 --- a/annotations/cypher.go +++ b/annotations/cypher.go @@ -5,16 +5,14 @@ import ( "errors" "fmt" "net/url" - "path" - "regexp" "strings" - cmneo4j "github.com/Financial-Times/cm-neo4j-driver" -) + "github.com/Financial-Times/cm-annotations-ontology/model" -var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$") + "github.com/Financial-Times/cm-annotations-ontology/neo4j" -var UnsupportedPredicateErr = errors.New("Unsupported predicate") + cmneo4j "github.com/Financial-Times/cm-neo4j-driver" +) // Service interface. Compatible with the baserwftapp service EXCEPT for // 1) the Write function, which has signature Write(thing interface{}) error... @@ -22,7 +20,7 @@ var UnsupportedPredicateErr = errors.New("Unsupported predicate") // The problem is that we have a list of things, and the uuid is for a related OTHER thing // TODO - move to implement a shared defined Service interface? type Service interface { - Write(contentUUID string, annotationLifecycle string, platformVersion string, thing interface{}) (bookmark string, err error) + Write(contentUUID string, annotationLifecycle string, platformVersion string, anns interface{}) (bookmark string, err error) Read(contentUUID string, bookmark string, annotationLifecycle string) (thing interface{}, found bool, err error) Delete(contentUUID string, annotationLifecycle string) (found bool, bookmark string, err error) Check() (err error) @@ -37,10 +35,6 @@ type service struct { publicAPIURL string } -const ( - nextVideoAnnotationsLifecycle = "annotations-next-video" -) - // NewCypherAnnotationsService instantiate driver func NewCypherAnnotationsService(driver *cmneo4j.Driver, publicAPIURL string) (Service, error) { _, err := url.ParseRequestURI(publicAPIURL) @@ -53,54 +47,34 @@ func NewCypherAnnotationsService(driver *cmneo4j.Driver, publicAPIURL string) (S // DecodeJSON decodes to a list of annotations, for ease of use this is a struct itself func (s service) DecodeJSON(dec *json.Decoder) (interface{}, error) { - a := Annotations{} + a := model.Annotations{} err := dec.Decode(&a) return a, err } -func (s service) Read(contentUUID string, bookmark string, annotationLifecycle string) (thing interface{}, found bool, err error) { - results := []Annotation{} - statement := ` - MATCH (c:Thing{uuid:$contentUUID})-[rel{lifecycle:$annotationLifecycle}]->(cc:Thing) - RETURN - cc.uuid as id, - cc.preflabel as prefLabel, - labels(cc) as types, - type(rel) as predicate, - rel.relevanceScore as relevanceScore, - rel.confidenceScore as confidenceScore, - rel.annotatedBy as annotatedBy, - rel.annotatedDate as annotatedDate - ORDER BY id` - - query := []*cmneo4j.Query{{ - Cypher: statement, - Params: map[string]interface{}{ - "contentUUID": contentUUID, - "annotationLifecycle": annotationLifecycle, - }, - Result: &results, - }} +func (s service) Read(contentUUID string, bookmark string, annotationLifecycle string) (ann interface{}, found bool, err error) { + query, results := neo4j.GetReadQuery(contentUUID, annotationLifecycle) _, err = s.driver.ReadMultiple(query, []string{bookmark}) if errors.Is(err, cmneo4j.ErrNoResultsFound) { - return Annotations{}, false, nil + return model.Annotations{}, false, nil } if err != nil { - return Annotations{}, false, fmt.Errorf("error executing delete queries: %w", err) + return model.Annotations{}, false, fmt.Errorf("error executing delete queries: %w", err) } - for idx := range results { - mapToResponseFormat(&results[idx], s.publicAPIURL) + mappedResults := *results + for idx := range mappedResults { + mapToResponseFormat(&mappedResults[idx], s.publicAPIURL) } - return Annotations(results), true, nil + return results, true, nil } // Delete removes all the annotations for this content. Ignore the nodes on either end - // may leave nodes that are only 'things' inserted by this writer: clean up // as a result of this will need to happen externally if required func (s service) Delete(contentUUID string, annotationLifecycle string) (bool, string, error) { - query := buildDeleteQuery(contentUUID, annotationLifecycle, true) + query := neo4j.BuildDeleteQuery(contentUUID, annotationLifecycle, true) bookmark, err := s.driver.WriteMultiple([]*cmneo4j.Query{query}, nil) if err != nil { @@ -117,23 +91,25 @@ func (s service) Delete(contentUUID string, annotationLifecycle string) (bool, s // Write a set of annotations associated with a piece of content. Any annotations // already there will be removed -func (s service) Write(contentUUID string, annotationLifecycle string, platformVersion string, thing interface{}) (string, error) { - annotationsToWrite, ok := thing.(Annotations) - if ok == false { - return "", errors.New("thing is not of type Annotations") - } +func (s service) Write(contentUUID string, annotationLifecycle string, platformVersion string, anns interface{}) (string, error) { if contentUUID == "" { return "", errors.New("content uuid is required") } - if err := validateAnnotations(&annotationsToWrite); err != nil { - return "", err + queries := append([]*cmneo4j.Query{}, neo4j.BuildDeleteQuery(contentUUID, annotationLifecycle, false)) + + annotations, ok := anns.([]interface{}) + if !ok { + return "", errors.New("error in casting annotations") } - queries := append([]*cmneo4j.Query{}, buildDeleteQuery(contentUUID, annotationLifecycle, false)) + for _, annotationToWrite := range annotations { + annotation, ok := annotationToWrite.(map[string]interface{}) + if !ok { + return "", errors.New("error in casting annotation") + } - for _, annotationToWrite := range annotationsToWrite { - query, err := createAnnotationQuery(contentUUID, annotationToWrite, platformVersion, annotationLifecycle) + query, err := neo4j.CreateAnnotationQuery(contentUUID, annotation, platformVersion, annotationLifecycle) if err != nil { return "", fmt.Errorf("create annotation query failed: %w", err) } @@ -153,21 +129,7 @@ func (s service) Check() error { } func (s service) Count(annotationLifecycle string, bookmark string, platformVersion string) (int, error) { - var results []struct { - Count int `json:"c"` - } - - query := []*cmneo4j.Query{{ - Cypher: `MATCH ()-[r{platformVersion:$platformVersion}]->() - WHERE r.lifecycle = $lifecycle - OR r.lifecycle IS NULL - RETURN count(r) as c`, - Params: map[string]interface{}{ - "platformVersion": platformVersion, - "lifecycle": annotationLifecycle, - }, - Result: &results, - }} + query, results := neo4j.Count(annotationLifecycle, platformVersion) _, err := s.driver.ReadMultiple(query, []string{bookmark}) if errors.Is(err, cmneo4j.ErrNoResultsFound) { @@ -187,123 +149,10 @@ func (s service) Initialise() error { return err } -func createAnnotationRelationship(relation string) (statement string) { - stmt := ` - MERGE (content:Thing{uuid:$contentID}) - MERGE (concept:Thing{uuid:$conceptID}) - MERGE (content)-[pred:%s {lifecycle:$annotationLifecycle}]->(concept) - SET pred=$annProps - ` - statement = fmt.Sprintf(stmt, relation) - return statement -} - -func getRelationshipFromPredicate(predicate string) (string, error) { - r, ok := relations[extractPredicateFromURI(predicate)] - if !ok { - return "", UnsupportedPredicateErr - } - return r, nil -} - -func createAnnotationQuery(contentUUID string, ann Annotation, platformVersion string, annotationLifecycle string) (*cmneo4j.Query, error) { - thingID, err := extractUUIDFromURI(ann.ID) - if err != nil { - return nil, err - } - - params := map[string]interface{}{} - params["platformVersion"] = platformVersion - params["lifecycle"] = annotationLifecycle - - if ann.AnnotatedBy != "" { - params["annotatedBy"], err = extractUUIDFromURI(ann.AnnotatedBy) - if err != nil { - return nil, err - } - } - if ann.AnnotatedDate != "" { - params["annotatedDateEpoch"] = ann.AnnotatedDateEpoch - params["annotatedDate"] = ann.AnnotatedDate - } - if ann.RelevanceScore != 0.0 { - params["relevanceScore"] = ann.RelevanceScore - } - if ann.ConfidenceScore != 0.0 { - params["confidenceScore"] = ann.ConfidenceScore - } - - relation, err := getRelationshipFromPredicate(ann.Predicate) - if err != nil { - return nil, err - } - - query := &cmneo4j.Query{ - Cypher: createAnnotationRelationship(relation), - Params: map[string]interface{}{ - "contentID": contentUUID, - "conceptID": thingID, - "annotationLifecycle": annotationLifecycle, - "annProps": params, - }, - } - - return query, nil -} - -func buildDeleteQuery(contentUUID string, annotationLifecycle string, includeStats bool) *cmneo4j.Query { - statement := `OPTIONAL MATCH (:Thing{uuid:$contentID})-[r{lifecycle:$annotationLifecycle}]->(t:Thing) - DELETE r` - query := &cmneo4j.Query{ - Cypher: statement, - Params: map[string]interface{}{ - "contentID": contentUUID, - "annotationLifecycle": annotationLifecycle, - }, - IncludeSummary: includeStats, - } - return query -} - -func validateAnnotations(annotations *Annotations) error { - //TODO - for consistency, we should probably just not create the annotation? - for _, annotation := range *annotations { - if annotation.ID == "" { - return ValidationError{fmt.Sprintf("Concept uuid missing for annotation %+v", annotation)} - } - } - return nil -} - -// ValidationError is thrown when the annotations are not valid because mandatory information is missing -type ValidationError struct { - Msg string -} - -func (v ValidationError) Error() string { - return v.Msg -} - -func mapToResponseFormat(ann *Annotation, publicAPIURL string) { +func mapToResponseFormat(ann *model.Annotation, publicAPIURL string) { ann.ID = thingURL(ann.ID, publicAPIURL) - if ann.AnnotatedBy != "" { - ann.AnnotatedBy = thingURL(ann.AnnotatedBy, publicAPIURL) - } } func thingURL(uuid, baseURL string) string { return strings.TrimRight(baseURL, "/") + "/things/" + uuid } - -func extractUUIDFromURI(uri string) (string, error) { - result := uuidExtractRegex.FindStringSubmatch(uri) - if len(result) == 2 { - return result[1], nil - } - return "", fmt.Errorf("couldn't extract uuid from uri %s", uri) -} - -func extractPredicateFromURI(uri string) string { - _, result := path.Split(uri) - return result -} diff --git a/annotations/cypher_integration_test.go b/annotations/cypher_integration_test.go index d8d01e5..21daafa 100644 --- a/annotations/cypher_integration_test.go +++ b/annotations/cypher_integration_test.go @@ -4,28 +4,37 @@ package annotations import ( + "encoding/json" "errors" "fmt" "os" + "path" "testing" + "github.com/Financial-Times/cm-annotations-ontology/model" + cmneo4j "github.com/Financial-Times/cm-neo4j-driver" logger "github.com/Financial-Times/go-logger/v2" "github.com/stretchr/testify/assert" ) const ( - brandUUID = "8e21cbd4-e94b-497a-a43b-5b2309badeb3" - PACPlatformVersion = "pac" - nextVideoPlatformVersion = "next-video" - contentLifecycle = "content" - PACAnnotationLifecycle = "annotations-pac" - apiHost = "http://api.ft.com" + brandUUID = "8e21cbd4-e94b-497a-a43b-5b2309badeb3" + PACPlatformVersion = "pac" + nextVideoPlatformVersion = "next-video" + nextVideoAnnotationsLifecycle = "next-video" + contentLifecycle = "content" + PACAnnotationLifecycle = "annotations-pac" + apiHost = "http://api.ft.com" + v2AnnotationLifecycle = "annotations-v2" + v2PlatformVersion = "v2" + contentUUID = "32b089d2-2aae-403d-be6e-877404f586cf" + oldConceptUUID = "ad28ddc7-4743-4ed3-9fad-5012b61fb919" + conceptUUID = "a7732a22-3884-4bfe-9761-fef161e41d69" + secondConceptUUID = "c834adfa-10c9-4748-8a21-c08537172706" ) func TestConstraintsApplied(t *testing.T) { - t.Skip("Skip, because the driver doesn't support EnsureConstraints/Indexes for Neo4j less than 4.x") - assert := assert.New(t) driver := getNeo4jDriver(t) annotationsService, err := NewCypherAnnotationsService(driver, apiHost) @@ -60,7 +69,7 @@ func TestWriteFailsWhenNoConceptIDSupplied(t *testing.T) { annotationsService, err := NewCypherAnnotationsService(driver, apiHost) assert.NoError(err, "creating cypher annotations service failed") - conceptWithoutID := Annotations{Annotation{ + conceptWithoutID := model.Annotations{model.Annotation{ PrefLabel: "prefLabel", Types: []string{ "http://www.ft.com/ontology/organisation/Organisation", @@ -73,35 +82,8 @@ func TestWriteFailsWhenNoConceptIDSupplied(t *testing.T) { AnnotatedDate: "2016-01-01T19:43:47.314Z", }} - _, err = annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, conceptWithoutID) + _, err = annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, convertAnnotations(t, conceptWithoutID)) assert.Error(err, "Should have failed to write annotation") - _, ok := err.(ValidationError) - assert.True(ok, "Should have returned a validation error") -} - -func TestWriteFailsForInvalidPredicate(t *testing.T) { - assert := assert.New(t) - driver := getNeo4jDriver(t) - annotationsService, err := NewCypherAnnotationsService(driver, apiHost) - assert.NoError(err, "creating cypher annotations service failed") - - conceptWithInvalidPredicate := Annotation{ - ID: fmt.Sprintf("http://api.ft.com/things/%s", oldConceptUUID), - PrefLabel: "prefLabel", - Types: []string{ - "http://www.ft.com/ontology/person/Person", - "http://www.ft.com/ontology/core/Thing", - "http://www.ft.com/ontology/concept/Concept", - }, - Predicate: "hasAFakePredicate", - RelevanceScore: 0.9, - ConfidenceScore: 0.8, - AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - AnnotatedDate: "2016-01-01T19:43:47.314Z", - } - - _, err = annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, Annotations{conceptWithInvalidPredicate}) - assert.EqualError(err, "create annotation query failed: Unsupported predicate") } func TestDeleteRemovesAnnotationsButNotConceptsOrContent(t *testing.T) { @@ -111,7 +93,7 @@ func TestDeleteRemovesAnnotationsButNotConceptsOrContent(t *testing.T) { assert.NoError(err, "creating cypher annotations service failed") annotationsToDelete := exampleConcepts(conceptUUID) - bookmark, err := annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, annotationsToDelete) + bookmark, err := annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, convertAnnotations(t, annotationsToDelete)) assert.NoError(err, "Failed to write annotation") readAnnotationsForContentUUIDAndCheckKeyFieldsMatch(t, annotationsService, contentUUID, v2AnnotationLifecycle, bookmark, annotationsToDelete) @@ -121,7 +103,7 @@ func TestDeleteRemovesAnnotationsButNotConceptsOrContent(t *testing.T) { anns, found, err := annotationsService.Read(contentUUID, bookmark, v2AnnotationLifecycle) - assert.Equal(Annotations{}, anns, "Found annotation for content %s when it should have been deleted", contentUUID) + assert.Equal(model.Annotations{}, anns, "Found annotation for content %s when it should have been deleted", contentUUID) assert.False(found, "Found annotation for content %s when it should have been deleted", contentUUID) assert.NoError(err, "Error trying to find annotation for content %s", contentUUID) @@ -141,7 +123,7 @@ func TestWriteAllValuesPresent(t *testing.T) { assert.NoError(err, "creating cypher annotations service failed") annotationsToWrite := exampleConcepts(conceptUUID) - bookmark, err := annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, annotationsToWrite) + bookmark, err := annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, convertAnnotations(t, annotationsToWrite)) assert.NoError(err, "Failed to write annotation") readAnnotationsForContentUUIDAndCheckKeyFieldsMatch(t, annotationsService, contentUUID, v2AnnotationLifecycle, bookmark, annotationsToWrite) @@ -172,7 +154,7 @@ func TestWriteDoesNotRemoveExistingIsClassifiedByBrandRelationshipsWithoutLifecy annotationsToWrite := exampleConcepts(conceptUUID) - _, err = annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, annotationsToWrite) + _, err = annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, convertAnnotations(t, annotationsToWrite)) assert.NoError(err, "Failed to write annotation") checkRelationship(t, assert, contentUUID, "v2") @@ -222,7 +204,7 @@ func TestWriteDoesNotRemoveExistingIsClassifiedByBrandRelationshipsWithContentLi annotationsToWrite := exampleConcepts(conceptUUID) - _, err = annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, annotationsToWrite) + _, err = annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, convertAnnotations(t, annotationsToWrite)) assert.NoError(err, "Failed to write annotation") checkRelationship(t, assert, contentUUID, "v2") @@ -280,7 +262,7 @@ func TestWriteDoesRemoveExistingIsClassifiedForPACTermsAndTheirRelationships(t * assert.NoError(driver.Write(contentQuery)) - _, err = annotationsService.Write(contentUUID, PACAnnotationLifecycle, PACPlatformVersion, exampleConcepts(conceptUUID)) + _, err = annotationsService.Write(contentUUID, PACAnnotationLifecycle, PACPlatformVersion, convertAnnotations(t, exampleConcepts(conceptUUID))) assert.NoError(err, "Failed to write annotation") found, bookmark, err := annotationsService.Delete(contentUUID, PACAnnotationLifecycle) assert.True(found, "Didn't manage to delete annotations for content uuid %s", contentUUID) @@ -339,8 +321,8 @@ func TestWriteAndReadMultipleAnnotations(t *testing.T) { annotationsService, err := NewCypherAnnotationsService(driver, apiHost) assert.NoError(err, "creating cypher annotations service failed") - multiConceptAnnotations := Annotations{ - Annotation{ + multiConceptAnnotations := model.Annotations{ + model.Annotation{ ID: getURI(conceptUUID), PrefLabel: "prefLabel", Types: []string{ @@ -354,7 +336,7 @@ func TestWriteAndReadMultipleAnnotations(t *testing.T) { AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", AnnotatedDate: "2016-01-01T19:43:47.314Z", }, - Annotation{ + model.Annotation{ ID: getURI(secondConceptUUID), PrefLabel: "prefLabel", Types: []string{ @@ -370,7 +352,7 @@ func TestWriteAndReadMultipleAnnotations(t *testing.T) { }, } - bookmark, err := annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, multiConceptAnnotations) + bookmark, err := annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, convertAnnotations(t, multiConceptAnnotations)) assert.NoError(err, "Failed to write annotation") readAnnotationsForContentUUIDAndCheckKeyFieldsMatch(t, annotationsService, contentUUID, v2AnnotationLifecycle, bookmark, multiConceptAnnotations) @@ -399,7 +381,7 @@ func TestNextVideoAnnotationsUpdatesAnnotations(t *testing.T) { err = driver.Write(contentQuery) assert.NoError(err, "Error creating test data in database.") - _, err = annotationsService.Write(contentUUID, nextVideoAnnotationsLifecycle, nextVideoPlatformVersion, exampleConcepts(secondConceptUUID)) + _, err = annotationsService.Write(contentUUID, nextVideoAnnotationsLifecycle, nextVideoPlatformVersion, convertAnnotations(t, exampleConcepts(secondConceptUUID))) assert.NoError(err, "Failed to write annotation.") result := []struct { @@ -434,13 +416,13 @@ func TestUpdateWillRemovePreviousAnnotations(t *testing.T) { assert.NoError(err, "creating cypher annotations service failed") oldAnnotationsToWrite := exampleConcepts(oldConceptUUID) - bookmark, err := annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, oldAnnotationsToWrite) + bookmark, err := annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, convertAnnotations(t, oldAnnotationsToWrite)) assert.NoError(err, "Failed to write annotations") readAnnotationsForContentUUIDAndCheckKeyFieldsMatch(t, annotationsService, contentUUID, v2AnnotationLifecycle, bookmark, oldAnnotationsToWrite) updatedAnnotationsToWrite := exampleConcepts(conceptUUID) - bookmark, err = annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, updatedAnnotationsToWrite) + bookmark, err = annotationsService.Write(contentUUID, v2AnnotationLifecycle, v2PlatformVersion, convertAnnotations(t, updatedAnnotationsToWrite)) assert.NoError(err, "Failed to write updated annotations") readAnnotationsForContentUUIDAndCheckKeyFieldsMatch(t, annotationsService, contentUUID, v2AnnotationLifecycle, bookmark, updatedAnnotationsToWrite) @@ -463,21 +445,22 @@ func getNeo4jDriver(t *testing.T) *cmneo4j.Driver { return driver } -func readAnnotationsForContentUUIDAndCheckKeyFieldsMatch(t *testing.T, svc Service, contentUUID, annotationLifecycle, bookmark string, expectedAnnotations []Annotation) { +// nolint:all +func readAnnotationsForContentUUIDAndCheckKeyFieldsMatch(t *testing.T, svc Service, contentUUID, annotationLifecycle, bookmark string, expectedAnnotations []model.Annotation) { assert := assert.New(t) storedThings, found, err := svc.Read(contentUUID, bookmark, annotationLifecycle) - storedAnnotations := storedThings.(Annotations) + storedAnnotations := storedThings.(*[]model.Annotation) assert.NoError(err, "Error finding annotations for contentUUID %s", contentUUID) assert.True(found, "Didn't find annotations for contentUUID %s", contentUUID) - assert.Equal(len(expectedAnnotations), len(storedAnnotations), "Didn't get the same number of annotations") - for idx, expectedAnnotation := range expectedAnnotations { - storedAnnotation := storedAnnotations[idx] + assert.Equal(len(expectedAnnotations), len(*storedAnnotations), "Didn't get the same number of annotations") + + for idx, storedAnnotation := range *storedAnnotations { + expectedAnnotation := expectedAnnotations[idx] // In annotations write, we don't store anything other than ID for the concept (so type will only be 'Thing' and pref label will not // be present UNLESS the concept has been written by some other system) assert.Equal(expectedAnnotation.ID, storedAnnotation.ID, "ID is not the same") - expectedPredicate, err := getRelationshipFromPredicate(expectedAnnotation.Predicate) - assert.NoError(err, "error getting relationship from predicate %s", expectedAnnotation.Predicate) + expectedPredicate := getRelationshipFromPredicate(expectedAnnotation.Predicate) assert.Equal(expectedPredicate, storedAnnotation.Predicate, "Predicates are not the same") assert.Equal(expectedAnnotation.RelevanceScore, storedAnnotation.RelevanceScore, "Relevance score is not the same") assert.Equal(expectedAnnotation.ConfidenceScore, storedAnnotation.ConfidenceScore, "Confidence score is not the same") @@ -604,6 +587,55 @@ func deleteNode(driver *cmneo4j.Driver, uuid string) error { return driver.Write(query) } -func exampleConcepts(uuid string) Annotations { - return Annotations{exampleConcept(uuid)} +func exampleConcepts(uuid string) model.Annotations { + return model.Annotations{ + model.Annotation{ + ID: getURI(uuid), + PrefLabel: "prefLabel", + Types: []string{ + "http://www.ft.com/ontology/organisation/Organisation", + "http://www.ft.com/ontology/core/Thing", + "http://www.ft.com/ontology/concept/Concept", + }, + Predicate: "mentions", + RelevanceScore: 0.9, + ConfidenceScore: 0.8, + AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", + AnnotatedDate: "2016-01-01T19:43:47.314Z", + }, + } +} + +func getURI(uuid string) string { + return fmt.Sprintf("http://api.ft.com/things/%s", uuid) +} + +func convertAnnotations(t *testing.T, anns model.Annotations) []interface{} { + var annSlice []interface{} + for _, ann := range anns { + var annMap map[string]interface{} + data, err := json.Marshal(ann) + if err != nil { + t.Fatal(err) + } + err = json.Unmarshal(data, &annMap) + if err != nil { + t.Fatal(err) + } + annSlice = append(annSlice, annMap) + } + return annSlice +} + +func getRelationshipFromPredicate(predicate string) string { + r, ok := model.Relations[extractPredicateFromURI(predicate)] + if !ok { + return "" + } + return r +} + +func extractPredicateFromURI(uri string) string { + _, result := path.Split(uri) + return result } diff --git a/annotations/cypher_test.go b/annotations/cypher_test.go deleted file mode 100644 index 2c0e312..0000000 --- a/annotations/cypher_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package annotations - -import ( - "errors" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -const ( - contentUUID = "32b089d2-2aae-403d-be6e-877404f586cf" - v2PlatformVersion = "v2" - pacPlatformVersion = "pac" - v2AnnotationLifecycle = "annotations-v2" - pacAnnotationLifecycle = "annotations-pac" -) - -func TestCreateAnnotationQuery(t *testing.T) { - assert := assert.New(t) - annotationToWrite := exampleConcept(oldConceptUUID) - - query, err := createAnnotationQuery(contentUUID, annotationToWrite, v2PlatformVersion, v2AnnotationLifecycle) - assert.NoError(err, "Cypher query for creating annotations couldn't be created.") - params := query.Params["annProps"].(map[string]interface{}) - assert.Equal(v2PlatformVersion, params["platformVersion"], fmt.Sprintf("\nExpected: %s\nActual: %s", v2PlatformVersion, params["platformVersion"])) -} - -func TestCreateAnnotationQueryWithPredicate(t *testing.T) { - testCases := []struct { - name string - relationship string - annotationToWrite Annotation - lifecycle string - platformVersion string - }{ - { - name: "isClassifiedBy", - relationship: "IS_CLASSIFIED_BY", - annotationToWrite: conceptWithPredicate, - platformVersion: v2PlatformVersion, - lifecycle: v2AnnotationLifecycle, - }, - { - name: "about", - relationship: "ABOUT", - annotationToWrite: conceptWithAboutPredicate, - platformVersion: v2PlatformVersion, - lifecycle: v2AnnotationLifecycle, - }, - { - name: "hasAuthor", - relationship: "HAS_AUTHOR", - annotationToWrite: conceptWithHasAuthorPredicate, - platformVersion: v2PlatformVersion, - lifecycle: v2AnnotationLifecycle, - }, - { - name: "hasContributor", - relationship: "HAS_CONTRIBUTOR", - annotationToWrite: conceptWithHasContributorPredicate, - platformVersion: pacPlatformVersion, - lifecycle: pacAnnotationLifecycle, - }, - { - name: "hasDisplayTag", - relationship: "HAS_DISPLAY_TAG", - annotationToWrite: conceptWithHasDisplayTagPredicate, - platformVersion: pacPlatformVersion, - lifecycle: pacAnnotationLifecycle, - }, - { - name: "implicitlyClassifiedBy", - relationship: "IMPLICITLY_CLASSIFIED_BY", - annotationToWrite: conceptWithImplicitlyClassifiedByPredicate, - platformVersion: pacPlatformVersion, - lifecycle: pacAnnotationLifecycle, - }, - { - name: "hasBrand", - relationship: "HAS_BRAND", - annotationToWrite: conceptWithHasBrandPredicate, - platformVersion: pacPlatformVersion, - lifecycle: pacAnnotationLifecycle, - }, - } - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - assert := assert.New(t) - query, err := createAnnotationQuery(contentUUID, test.annotationToWrite, test.platformVersion, test.lifecycle) - - assert.NoError(err, "Cypher query for creating annotations couldn't be created.") - assert.Contains(query.Cypher, test.relationship, "Relationship name is not inserted!") - assert.NotContains(query.Cypher, "MENTIONS", fmt.Sprintf("%s should be inserted instead of MENTIONS", test.relationship)) - }) - } -} - -func TestCreateAnnotationQueryWithoutPredicate(t *testing.T) { - assert := assert.New(t) - annotation := exampleConcept(oldConceptUUID) - annotation.Predicate = "" - - _, err := createAnnotationQuery(contentUUID, annotation, v2PlatformVersion, v2AnnotationLifecycle) - assert.True(errors.Is(err, UnsupportedPredicateErr), "Creating annotation without predicate is not allowed.") -} - -func TestGetRelationshipFromPredicate(t *testing.T) { - var tests = []struct { - predicate string - relationship string - }{ - {"mentions", "MENTIONS"}, - {"isClassifiedBy", "IS_CLASSIFIED_BY"}, - {"implicitlyClassifiedBy", "IMPLICITLY_CLASSIFIED_BY"}, - {"about", "ABOUT"}, - {"isPrimarilyClassifiedBy", "IS_PRIMARILY_CLASSIFIED_BY"}, - {"majorMentions", "MAJOR_MENTIONS"}, - {"hasAuthor", "HAS_AUTHOR"}, - {"hasContributor", "HAS_CONTRIBUTOR"}, - {"hasDisplayTag", "HAS_DISPLAY_TAG"}, - {"hasBrand", "HAS_BRAND"}, - } - - for _, test := range tests { - actualRelationship, err := getRelationshipFromPredicate(test.predicate) - assert.NoError(t, err) - - if test.relationship != actualRelationship { - t.Errorf("\nExpected: %s\nActual: %s", test.relationship, actualRelationship) - } - } -} diff --git a/annotations/example_data_test.go b/annotations/example_data_test.go deleted file mode 100644 index 9520da8..0000000 --- a/annotations/example_data_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package annotations - -import "fmt" - -const ( - conceptUUID = "a7732a22-3884-4bfe-9761-fef161e41d69" - oldConceptUUID = "ad28ddc7-4743-4ed3-9fad-5012b61fb919" - secondConceptUUID = "c834adfa-10c9-4748-8a21-c08537172706" -) - -func getURI(uuid string) string { - return fmt.Sprintf("http://api.ft.com/things/%s", uuid) -} - -var ( - conceptWithPredicate = Annotation{ - ID: getURI(oldConceptUUID), - PrefLabel: "prefLabel", - Types: []string{ - "http://www.ft.com/ontology/organisation/Organisation", - "http://www.ft.com/ontology/core/Thing", - "http://www.ft.com/ontology/concept/Concept", - }, - Predicate: "isClassifiedBy", - RelevanceScore: 0.9, - ConfidenceScore: 0.8, - AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - AnnotatedDate: "2016-01-01T19:43:47.314Z", - } - conceptWithAboutPredicate = Annotation{ - ID: getURI(oldConceptUUID), - PrefLabel: "prefLabel", - Types: []string{ - "http://www.ft.com/ontology/organisation/Organisation", - "http://www.ft.com/ontology/core/Thing", - "http://www.ft.com/ontology/concept/Concept", - }, - Predicate: "about", - RelevanceScore: 0.9, - ConfidenceScore: 0.8, - AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - AnnotatedDate: "2016-01-01T19:43:47.314Z", - } - conceptWithHasAuthorPredicate = Annotation{ - ID: getURI(oldConceptUUID), - PrefLabel: "prefLabel", - Types: []string{ - "http://www.ft.com/ontology/person/Person", - "http://www.ft.com/ontology/core/Thing", - "http://www.ft.com/ontology/concept/Concept", - }, - Predicate: "hasAuthor", - RelevanceScore: 0.9, - ConfidenceScore: 0.8, - AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - AnnotatedDate: "2016-01-01T19:43:47.314Z", - } - conceptWithHasContributorPredicate = Annotation{ - ID: getURI(oldConceptUUID), - PrefLabel: "prefLabel", - Types: []string{ - "http://www.ft.com/ontology/person/Person", - "http://www.ft.com/ontology/core/Thing", - "http://www.ft.com/ontology/concept/Concept", - }, - Predicate: "hasContributor", - RelevanceScore: 0.9, - ConfidenceScore: 0.8, - AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - AnnotatedDate: "2016-01-01T19:43:47.314Z", - } - conceptWithHasDisplayTagPredicate = Annotation{ - ID: getURI(oldConceptUUID), - PrefLabel: "prefLabel", - Types: []string{ - "http://www.ft.com/ontology/person/Person", - "http://www.ft.com/ontology/core/Thing", - "http://www.ft.com/ontology/concept/Concept", - }, - Predicate: "hasDisplayTag", - RelevanceScore: 0.9, - ConfidenceScore: 0.8, - AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - AnnotatedDate: "2016-01-01T19:43:47.314Z", - } - - conceptWithImplicitlyClassifiedByPredicate = Annotation{ - ID: getURI(oldConceptUUID), - PrefLabel: "prefLabel", - Types: []string{ - "http://www.ft.com/ontology/person/Person", - "http://www.ft.com/ontology/core/Thing", - "http://www.ft.com/ontology/concept/Concept", - }, - Predicate: "implicitlyClassifiedBy", - RelevanceScore: 0.9, - ConfidenceScore: 0.8, - AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - AnnotatedDate: "2016-01-01T19:43:47.314Z", - } - - conceptWithHasBrandPredicate = Annotation{ - ID: getURI(oldConceptUUID), - PrefLabel: "prefLabel", - Types: []string{ - "http://www.ft.com/ontology/product/Brand", - "http://www.ft.com/ontology/core/Thing", - "http://www.ft.com/ontology/concept/Concept", - }, - Predicate: "hasBrand", - RelevanceScore: 0.9, - ConfidenceScore: 0.8, - AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - AnnotatedDate: "2016-01-01T19:43:47.314Z", - } -) - -func exampleConcept(uuid string) Annotation { - return Annotation{ - ID: getURI(uuid), - PrefLabel: "prefLabel", - Types: []string{ - "http://www.ft.com/ontology/organisation/Organisation", - "http://www.ft.com/ontology/core/Thing", - "http://www.ft.com/ontology/concept/Concept", - }, - Predicate: "mentions", - RelevanceScore: 0.9, - ConfidenceScore: 0.8, - AnnotatedBy: "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - AnnotatedDate: "2016-01-01T19:43:47.314Z", - } -} diff --git a/annotations/model.go b/annotations/model.go deleted file mode 100644 index b7c8e31..0000000 --- a/annotations/model.go +++ /dev/null @@ -1,30 +0,0 @@ -package annotations - -// Annotations represents a collection of Annotation instances -type Annotations []Annotation - -// Annotation is the main struct containing the annotations attributes -type Annotation struct { - ID string `json:"id,omitempty"` - PrefLabel string `json:"prefLabel,omitempty"` - Types []string `json:"types,omitempty"` - Predicate string `json:"predicate,omitempty"` - RelevanceScore float64 `json:"relevanceScore,omitempty"` - ConfidenceScore float64 `json:"confidenceScore,omitempty"` - AnnotatedBy string `json:"annotatedBy,omitempty"` - AnnotatedDate string `json:"annotatedDate,omitempty"` - AnnotatedDateEpoch int64 `json:"annotatedDateEpoch,omitempty"` -} - -var relations = map[string]string{ - "mentions": "MENTIONS", - "isClassifiedBy": "IS_CLASSIFIED_BY", - "implicitlyClassifiedBy": "IMPLICITLY_CLASSIFIED_BY", - "about": "ABOUT", - "isPrimarilyClassifiedBy": "IS_PRIMARILY_CLASSIFIED_BY", - "majorMentions": "MAJOR_MENTIONS", - "hasAuthor": "HAS_AUTHOR", - "hasContributor": "HAS_CONTRIBUTOR", - "hasDisplayTag": "HAS_DISPLAY_TAG", - "hasBrand": "HAS_BRAND", -} diff --git a/annotations/model_test.go b/annotations/model_test.go deleted file mode 100644 index ea4608d..0000000 --- a/annotations/model_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package annotations - -import ( - "encoding/json" - "io/ioutil" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestUnMarshallingAnnotation(t *testing.T) { - assert := assert.New(t) - annotations := Annotations{} - jason, err := ioutil.ReadFile("examplePutBody.json") - assert.NoError(err, "Unexpected error") - - err = json.Unmarshal([]byte(jason), &annotations) - assert.NoError(err, "Unexpected error") -} diff --git a/exampleAnnotationsMessage.json b/exampleAnnotationsMessage.json index c6a38d5..21d6d7b 100644 --- a/exampleAnnotationsMessage.json +++ b/exampleAnnotationsMessage.json @@ -2,14 +2,19 @@ "uuid": "3a636e78-5a47-11e7-9bc8-8055f264aa8b", "annotations": [ { - "id": "http://api.ft.com/things/2cca9e2a-2248-3e48-abc1-93d718b91bbe", - "prefLabel": "China Politics & Policy", - "predicate": "majorMentions", + "id": "http://api.ft.com/things/2384fa7a-d514-3d6a-a0ea-3a711f66d0d8", + "prefLabel": "Apple", "types": [ - "http://www.ft.com/ontology/Topic" + "http://www.ft.com/ontology/organisation/Organisation", + "http://www.ft.com/ontology/core/Thing", + "http://www.ft.com/ontology/concept/Concept" ], + "predicate": "mentions", "relevanceScore": 1, - "confidenceScore": 1 + "confidenceScore": 0.9932743203464962, + "annotatedBy": "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", + "annotatedDate": "2016-01-20T19:43:47.314Z", + "annotatedDateEpoch": 1453180597 } ] } diff --git a/annotations/examplePutBody.json b/examplePutBody.json similarity index 79% rename from annotations/examplePutBody.json rename to examplePutBody.json index d3126b1..d53412b 100644 --- a/annotations/examplePutBody.json +++ b/examplePutBody.json @@ -7,11 +7,12 @@ "http://www.ft.com/ontology/core/Thing", "http://www.ft.com/ontology/concept/Concept" ], - "predicate": "isClassifiedBy", + "predicate": "mentions", "relevanceScore": 1, "confidenceScore": 0.9932743203464962, "annotatedBy": "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - "annotatedDate": "2016-01-20T19:43:47.314Z" + "annotatedDate": "2016-01-20T19:43:47.314Z", + "annotatedDateEpoch": 1453180597 }, { "id": "http://api.ft.com/things/ccaa202e-3d27-3b75-b2f2-261cf5038a1f", @@ -21,9 +22,11 @@ "http://www.ft.com/ontology/core/Thing", "http://www.ft.com/ontology/concept/Concept" ], + "predicate": "mentions", "relevanceScore": 0.375, "confidenceScore": 0.9996836123273414, "annotatedBy": "http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a", - "annotatedDate": "2016-01-20T19:43:47.314Z" + "annotatedDate": "2016-01-20T19:43:47.314Z", + "annotatedDateEpoch": 1453180597 } -] +] \ No newline at end of file diff --git a/forwarder/forwarder.go b/forwarder/forwarder.go index 9910441..df0d8ef 100644 --- a/forwarder/forwarder.go +++ b/forwarder/forwarder.go @@ -7,7 +7,6 @@ import ( "github.com/Financial-Times/kafka-client-go/v3" - "github.com/Financial-Times/annotations-rw-neo4j/v4/annotations" "github.com/google/uuid" ) @@ -26,7 +25,7 @@ type outputMessage struct { // QueueForwarder is the interface implemented by types that can send annotation messages to a queue. type QueueForwarder interface { - SendMessage(transactionID string, originSystem string, bookmark string, platformVersion string, uuid string, annotations annotations.Annotations) error + SendMessage(transactionID string, originSystem string, bookmark string, platformVersion string, uuid string, annotations interface{}) error } type kafkaProducer interface { @@ -40,7 +39,7 @@ type Forwarder struct { } // SendMessage marshals an annotations payload using the outputMessage format and sends it to a Kafka. -func (f Forwarder) SendMessage(transactionID string, originSystem string, bookmark string, platformVersion string, uuid string, annotations annotations.Annotations) error { +func (f Forwarder) SendMessage(transactionID string, originSystem string, bookmark string, platformVersion string, uuid string, annotations interface{}) error { headers := CreateHeaders(transactionID, originSystem, bookmark) body, err := f.prepareBody(platformVersion, uuid, annotations, headers["Message-Timestamp"]) if err != nil { @@ -50,7 +49,7 @@ func (f Forwarder) SendMessage(transactionID string, originSystem string, bookma return f.Producer.SendMessage(kafka.NewFTMessage(headers, body)) } -func (f Forwarder) prepareBody(platformVersion string, uuid string, anns annotations.Annotations, lastModified string) (string, error) { +func (f Forwarder) prepareBody(platformVersion string, uuid string, anns interface{}, lastModified string) (string, error) { wrappedMsg := outputMessage{ Payload: map[string]interface{}{ strings.ToLower(f.MessageType): anns, diff --git a/forwarder/forwarder_test.go b/forwarder/forwarder_test.go index 7e02be5..48e4f0c 100644 --- a/forwarder/forwarder_test.go +++ b/forwarder/forwarder_test.go @@ -3,20 +3,20 @@ package forwarder_test import ( "encoding/json" "fmt" - "io/ioutil" + "os" "regexp" "testing" "time" - "github.com/Financial-Times/annotations-rw-neo4j/v4/annotations" - "github.com/Financial-Times/annotations-rw-neo4j/v4/forwarder" - + "github.com/Financial-Times/cm-annotations-ontology/model" "github.com/Financial-Times/kafka-client-go/v3" + + "github.com/Financial-Times/annotations-rw-neo4j/v4/forwarder" ) type InputMessage struct { - Annotations annotations.Annotations `json:"annotations"` - UUID string `json:"uuid"` + Annotations model.Annotations `json:"annotations"` + UUID string `json:"uuid"` } const transactionID = "example-transaction-id" @@ -24,10 +24,10 @@ const originSystem = "http://cmdb.ft.com/systems/pac" const bookmark = "FB:kcwQnrEEnFpfSJ2PtiykK/JNh8oBozhIkA==" func TestSendMessage(t *testing.T) { - const expectedAnnotationsOutputBody = `{"payload":{"annotations":[{"id":"http://api.ft.com/things/2cca9e2a-2248-3e48-abc1-93d718b91bbe","prefLabel":"China Politics \u0026 Policy","types":["http://www.ft.com/ontology/Topic"],"predicate":"majorMentions","relevanceScore":1,"confidenceScore":1}],"lastModified":"%s","uuid":"3a636e78-5a47-11e7-9bc8-8055f264aa8b"},"contentUri":"http://pac.annotations-rw-neo4j.svc.ft.com/annotations/3a636e78-5a47-11e7-9bc8-8055f264aa8b","lastModified":"%[1]s"}` - const expectedSuggestionsOutputBody = `{"payload":{"lastModified":"%s","suggestions":[{"id":"http://api.ft.com/things/2cca9e2a-2248-3e48-abc1-93d718b91bbe","prefLabel":"China Politics \u0026 Policy","types":["http://www.ft.com/ontology/Topic"],"predicate":"majorMentions","relevanceScore":1,"confidenceScore":1}],"uuid":"3a636e78-5a47-11e7-9bc8-8055f264aa8b"},"contentUri":"http://v2.suggestions-rw-neo4j.svc.ft.com/annotations/3a636e78-5a47-11e7-9bc8-8055f264aa8b","lastModified":"%[1]s"}` + const expectedAnnotationsOutputBody = `{"payload":{"annotations":[{"id":"http://api.ft.com/things/2384fa7a-d514-3d6a-a0ea-3a711f66d0d8","prefLabel":"Apple","types":["http://www.ft.com/ontology/organisation/Organisation","http://www.ft.com/ontology/core/Thing","http://www.ft.com/ontology/concept/Concept"],"predicate":"mentions","relevanceScore":1,"confidenceScore":0.9932743203464962,"annotatedBy":"http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a","annotatedDate":"2016-01-20T19:43:47.314Z","annotatedDateEpoch":1453180597}],"lastModified":"%s","uuid":"3a636e78-5a47-11e7-9bc8-8055f264aa8b"},"contentUri":"http://pac.annotations-rw-neo4j.svc.ft.com/annotations/3a636e78-5a47-11e7-9bc8-8055f264aa8b","lastModified":"%[1]s"}` + const expectedSuggestionsOutputBody = `{"payload":{"lastModified":"%s","suggestions":[{"id":"http://api.ft.com/things/2384fa7a-d514-3d6a-a0ea-3a711f66d0d8","prefLabel":"Apple","types":["http://www.ft.com/ontology/organisation/Organisation","http://www.ft.com/ontology/core/Thing","http://www.ft.com/ontology/concept/Concept"],"predicate":"mentions","relevanceScore":1,"confidenceScore":0.9932743203464962,"annotatedBy":"http://api.ft.com/things/0edd3c31-1fd0-4ef6-9230-8d545be3880a","annotatedDate":"2016-01-20T19:43:47.314Z","annotatedDateEpoch":1453180597}],"uuid":"3a636e78-5a47-11e7-9bc8-8055f264aa8b"},"contentUri":"http://v2.suggestions-rw-neo4j.svc.ft.com/annotations/3a636e78-5a47-11e7-9bc8-8055f264aa8b","lastModified":"%[1]s"}` - body, err := ioutil.ReadFile("../exampleAnnotationsMessage.json") + body, err := os.ReadFile("../exampleAnnotationsMessage.json") if err != nil { t.Fatal("Unexpected error reading example message") } diff --git a/go.mod b/go.mod index 75815f7..ce64dda 100644 --- a/go.mod +++ b/go.mod @@ -3,46 +3,49 @@ module github.com/Financial-Times/annotations-rw-neo4j/v4 go 1.19 require ( + github.com/Financial-Times/cm-annotations-ontology v1.0.0-init github.com/Financial-Times/cm-neo4j-driver v1.1.0 - github.com/Financial-Times/go-fthealth v0.0.0-20180807113633-3d8eb430d5b5 + github.com/Financial-Times/go-fthealth v0.0.0-20200609161010-4c53fbef65fa github.com/Financial-Times/go-logger/v2 v2.0.1 github.com/Financial-Times/http-handlers-go/v2 v2.1.0 github.com/Financial-Times/kafka-client-go/v3 v3.0.4 - github.com/Financial-Times/service-status-go v0.0.0-20160323111542-3f5199736a3d + github.com/Financial-Times/service-status-go v0.0.0-20210115125138-41b7375f9b94 github.com/Financial-Times/transactionid-utils-go v0.2.0 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/jawher/mow.cli v1.0.4 github.com/pkg/errors v0.8.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.4 ) require ( + github.com/Financial-Times/upp-content-validator-kit/v3 v3.0.2 // indirect github.com/Shopify/sarama v1.33.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.2.0 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect + github.com/eapache/go-resiliency v1.3.0 // indirect + github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-uuid v1.0.2 // indirect - github.com/hashicorp/go-version v1.0.0 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/go-version v1.3.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect - github.com/jcmturner/gofork v1.0.0 // indirect - github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.15.0 // indirect + github.com/klauspost/compress v1.16.6 // indirect github.com/neo4j/neo4j-go-driver/v4 v4.3.3 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/stretchr/objx v0.4.0 // indirect - golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + github.com/stretchr/objx v0.5.0 // indirect + golang.org/x/crypto v0.11.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 7f0dd18..fd2c9d8 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,22 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Financial-Times/cm-annotations-ontology v1.0.0-init h1:3pY/WVMh0n4ZjEQHvg3hLu4zz9b17tyFow83aeW1QYs= +github.com/Financial-Times/cm-annotations-ontology v1.0.0-init/go.mod h1:tqOtG3y4Kkp/yrZSNVfO465DiPW8xKBPN5Uk7bvhHyo= github.com/Financial-Times/cm-neo4j-driver v1.1.0 h1:9TZgyE7Vl8iuH0MRQlrVLkjzLFwIQsCn/td806WU7A8= github.com/Financial-Times/cm-neo4j-driver v1.1.0/go.mod h1:fQ77C8A+6I+ihwe6Ob8Y7sIzU3s/DTrjBZJGVQOeum0= -github.com/Financial-Times/go-fthealth v0.0.0-20180807113633-3d8eb430d5b5 h1:XH5h45aAyG1bAFBYmkgJkT4q13CbkCJ+gj9+rIfzuL8= -github.com/Financial-Times/go-fthealth v0.0.0-20180807113633-3d8eb430d5b5/go.mod h1:gpAzq6W5rCheYlY32JOIxS/VjVcYHbC2PkMzQngHT9c= +github.com/Financial-Times/go-fthealth v0.0.0-20200609161010-4c53fbef65fa h1:jhrRhI2ihDmhAhmCucnabU6gyuQIDBcId7kKjnyp7Ec= +github.com/Financial-Times/go-fthealth v0.0.0-20200609161010-4c53fbef65fa/go.mod h1:gpAzq6W5rCheYlY32JOIxS/VjVcYHbC2PkMzQngHT9c= github.com/Financial-Times/go-logger/v2 v2.0.1 h1:iekEfSsUtlkg+YkXTZo+/fIN2VbZ2/3Hl9yolP3z5X8= github.com/Financial-Times/go-logger/v2 v2.0.1/go.mod h1:Jpky5JYSX7xjGUClfA9hEMDmn40tUbfQQITjVIFGQiM= github.com/Financial-Times/http-handlers-go/v2 v2.1.0 h1:kBj41WrDXUGUiLHEjWI1zQUbRTNEw0JxSaaXQwZkib8= github.com/Financial-Times/http-handlers-go/v2 v2.1.0/go.mod h1:Tgkc7TqJXl/NFxB8eP8CX7YU5X01gbrL55LqNzo4YVY= github.com/Financial-Times/kafka-client-go/v3 v3.0.4 h1:7gfyzCpNclC6bpMaSAHlOvWsXhCMrG+VCTe2oknJoRo= github.com/Financial-Times/kafka-client-go/v3 v3.0.4/go.mod h1:+xSPZqQTS2iN13T2WxsL3IdqZz0BbaLZFJma1QfmZ3U= -github.com/Financial-Times/service-status-go v0.0.0-20160323111542-3f5199736a3d h1:USNBTIof6vWGM49SYrxvC5Y8NqyDL3YuuYmID81ORZQ= -github.com/Financial-Times/service-status-go v0.0.0-20160323111542-3f5199736a3d/go.mod h1:7zULC9rrq6KxFkpB3Y5zNVaEwrf1g2m3dvXJBPDXyvM= +github.com/Financial-Times/service-status-go v0.0.0-20210115125138-41b7375f9b94 h1:C4nQYbczJmMKAf6FtEh0FouoU7YN4fX9APKc8Xdcqso= +github.com/Financial-Times/service-status-go v0.0.0-20210115125138-41b7375f9b94/go.mod h1:7zULC9rrq6KxFkpB3Y5zNVaEwrf1g2m3dvXJBPDXyvM= github.com/Financial-Times/transactionid-utils-go v0.2.0 h1:YcET5Hd1fUGWWpQSVszYUlAc15ca8tmjRetUuQKRqEQ= github.com/Financial-Times/transactionid-utils-go v0.2.0/go.mod h1:tPAcAFs/dR6Q7hBDGNyUyixHRvg/n9NW/JTq8C58oZ0= +github.com/Financial-Times/upp-content-validator-kit/v3 v3.0.2 h1:rGaQ+QyoEbWB/W/O4O8ioK4Ha3MEuIPk052UGkGunhA= +github.com/Financial-Times/upp-content-validator-kit/v3 v3.0.2/go.mod h1:sEq5cNuDZp22VKT3YhsVoBs5DVIDWZzij8SfSP7V58Q= github.com/Shopify/sarama v1.33.0 h1:2K4mB9M4fo46sAM7t6QTsmSO8dLX1OqznLM7vn3OjZ8= github.com/Shopify/sarama v1.33.0/go.mod h1:lYO7LwEBkE0iAeTl94UfPSrDaavFzSFlmn+5isARATQ= github.com/Shopify/toxiproxy/v2 v2.3.0 h1:62YkpiP4bzdhKMH+6uC5E95y608k3zDwdzuBMsnn3uQ= @@ -24,10 +28,12 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1-0.20170711183451-adab96458c51/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= +github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= +github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM= +github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= @@ -64,10 +70,11 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.0.0 h1:21MVWPKDphxa7ineQQTrCU5brh7OuVVAzGOCnnCPtE8= -github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw= +github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jawher/mow.cli v1.0.4 h1:hKjm95J7foZ2ngT8tGb15Aq9rj751R7IUDjG+5e3cGA= github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= @@ -75,16 +82,19 @@ github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFK github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= -github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA= github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= +github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= +github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= +github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -117,13 +127,16 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqn github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 h1:uIkTLo0AGRc8l7h5l9r+GcYi9qfVPt6lD4/bhmzfiKo= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -132,8 +145,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= @@ -144,8 +158,10 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -156,8 +172,10 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -175,8 +193,10 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/healthcheck.go b/healthcheck.go index fc16147..bf03c06 100644 --- a/healthcheck.go +++ b/healthcheck.go @@ -97,7 +97,7 @@ func (h healthCheckHandler) checkKafkaConnectivity() (string, error) { } // Checker does more stuff -//TODO use the shared utility check +// TODO use the shared utility check func (hc healthCheckHandler) Checker() (string, error) { if err := hc.annotationsService.Check(); err != nil { return "Error connecting to neo4j", err diff --git a/helm/annotations-rw-neo4j/app-configs/annotations-rw-neo4j_eks_delivery.yaml b/helm/annotations-rw-neo4j/app-configs/annotations-rw-neo4j_eks_delivery.yaml index 66c060a..a80eaa8 100644 --- a/helm/annotations-rw-neo4j/app-configs/annotations-rw-neo4j_eks_delivery.yaml +++ b/helm/annotations-rw-neo4j/app-configs/annotations-rw-neo4j_eks_delivery.yaml @@ -11,3 +11,4 @@ env: PRODUCER_TOPIC: PostConceptAnnotations KAFKA_LAG_TOLERANCE: 100 LIFECYCLE_CONFIG_PATH: annotation-config.json + JSON_SCHEMA_NAME: annotations-pac.json;annotations-next-video.json diff --git a/helm/annotations-rw-neo4j/app-configs/suggestions-rw-neo4j_eks_delivery.yaml b/helm/annotations-rw-neo4j/app-configs/suggestions-rw-neo4j_eks_delivery.yaml index 4b3bc85..060aaea 100644 --- a/helm/annotations-rw-neo4j/app-configs/suggestions-rw-neo4j_eks_delivery.yaml +++ b/helm/annotations-rw-neo4j/app-configs/suggestions-rw-neo4j_eks_delivery.yaml @@ -10,3 +10,4 @@ env: CONSUMER_TOPICS: ConceptSuggestions KAFKA_LAG_TOLERANCE: 100 LIFECYCLE_CONFIG_PATH: suggestion-config.json + JSON_SCHEMA_NAME: annotations-v2.json diff --git a/helm/annotations-rw-neo4j/templates/deployment.yaml b/helm/annotations-rw-neo4j/templates/deployment.yaml index abe2ecb..822170b 100644 --- a/helm/annotations-rw-neo4j/templates/deployment.yaml +++ b/helm/annotations-rw-neo4j/templates/deployment.yaml @@ -21,62 +21,65 @@ spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ .Values.service.name }} - topologyKey: "kubernetes.io/hostname" + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ .Values.service.name }} + topologyKey: "kubernetes.io/hostname" containers: - - name: {{ .Values.service.name }} - image: "{{ .Values.image.repository }}:{{ .Chart.Version }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: APP_NAME - value: {{ .Values.service.name }} - - name: APP_SYSTEM_CODE - value: {{ .Values.service.systemCode }} - - name: APP_PORT - value: "8080" - - name: NEO_URL - valueFrom: - configMapKeyRef: - name: global-config - key: neo4j.cluster.bolt.url - - name: SHOULD_CONSUME_MESSAGES - value: "{{ .Values.env.SHOULD_CONSUME_MESSAGES }}" - - name: SHOULD_FORWARD_MESSAGES - value: "{{ .Values.env.SHOULD_FORWARD_MESSAGES }}" - - name: CONSUMER_GROUP - value: {{ .Values.env.CONSUMER_GROUP }} - - name: CONSUMER_TOPICS - value: {{ .Values.env.CONSUMER_TOPICS }} - - name: PRODUCER_TOPIC - value: {{ .Values.env.PRODUCER_TOPIC }} - - name: KAFKA_LAG_TOLERANCE - value: "{{ .Values.env.KAFKA_LAG_TOLERANCE }}" - - name: KAFKA_ADDRESS - valueFrom: - configMapKeyRef: - name: global-config - key: msk.kafka.broker.url - - name: LIFECYCLE_CONFIG_PATH - value: {{ .Values.env.LIFECYCLE_CONFIG_PATH }} - - name: API_HOST - value: http://api.ft.com - ports: - - containerPort: 8080 - livenessProbe: - tcpSocket: - port: 8080 - initialDelaySeconds: 10 - readinessProbe: - httpGet: - path: "/__gtg" - port: 8080 - initialDelaySeconds: 5 - periodSeconds: 30 - resources: -{{ toYaml .Values.resources | indent 12 }} - + - name: {{ .Values.service.name }} + image: "{{ .Values.image.repository }}:{{ .Chart.Version }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: APP_NAME + value: {{ .Values.service.name }} + - name: APP_SYSTEM_CODE + value: {{ .Values.service.systemCode }} + - name: APP_PORT + value: "8080" + - name: NEO_URL + valueFrom: + configMapKeyRef: + name: global-config + key: neo4j.cluster.bolt.url + - name: SHOULD_CONSUME_MESSAGES + value: "{{ .Values.env.SHOULD_CONSUME_MESSAGES }}" + - name: SHOULD_FORWARD_MESSAGES + value: "{{ .Values.env.SHOULD_FORWARD_MESSAGES }}" + - name: CONSUMER_GROUP + value: {{ .Values.env.CONSUMER_GROUP }} + - name: CONSUMER_TOPICS + value: {{ .Values.env.CONSUMER_TOPICS }} + - name: PRODUCER_TOPIC + value: {{ .Values.env.PRODUCER_TOPIC }} + - name: KAFKA_LAG_TOLERANCE + value: "{{ .Values.env.KAFKA_LAG_TOLERANCE }}" + - name: KAFKA_ADDRESS + valueFrom: + configMapKeyRef: + name: global-config + key: msk.kafka.broker.url + - name: LIFECYCLE_CONFIG_PATH + value: {{ .Values.env.LIFECYCLE_CONFIG_PATH }} + - name: API_HOST + value: http://api.ft.com + - name: JSON_SCHEMAS_PATH + value: "{{ .Values.env.JSON_SCHEMAS_PATH }}" + - name: JSON_SCHEMA_NAME + value: "{{ .Values.env.JSON_SCHEMA_NAME }}" + ports: + - containerPort: 8080 + livenessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 10 + readinessProbe: + httpGet: + path: "/__gtg" + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 30 + resources: + {{ toYaml .Values.resources | indent 12 }} diff --git a/helm/annotations-rw-neo4j/values.yaml b/helm/annotations-rw-neo4j/values.yaml index e68cdad..8a10296 100644 --- a/helm/annotations-rw-neo4j/values.yaml +++ b/helm/annotations-rw-neo4j/values.yaml @@ -13,3 +13,5 @@ resources: memory: 40Mi limits: memory: 256Mi +env: + JSON_SCHEMAS_PATH: "/schemas" diff --git a/http_handler.go b/http_handler.go index efa6ccc..72a74a8 100644 --- a/http_handler.go +++ b/http_handler.go @@ -22,8 +22,9 @@ const ( bookmarkHeader = "Neo4j-Bookmark" ) -//service def +// service def type httpHandler struct { + validator jsonValidator annotationsService annotations.Service forwarder forwarder.QueueForwarder originMap map[string]string @@ -189,20 +190,20 @@ func (hh *httpHandler) PutAnnotations(w http.ResponseWriter, r *http.Request) { } tid := transactionidutils.GetTransactionIDFromRequest(r) - bookmark, err := hh.annotationsService.Write(uuid, lifecycle, platformVersion, anns) - if errors.Is(err, annotations.UnsupportedPredicateErr) { - hh.log.WithUUID(uuid).WithTransactionID(tid).WithError(err).Error("invalid predicate provided") - writeJSONError(w, "Please provide a valid predicate", http.StatusBadRequest) - return + for _, ann := range anns { + err = hh.validator.Validate(ann) + if err != nil { + hh.log.WithUUID(uuid).WithTransactionID(tid).WithError(err).Error("failed validating annotations") + msg := fmt.Sprintf("Error validating annotations (%v)", err) + writeJSONError(w, msg, http.StatusBadRequest) + return + } } + bookmark, err := hh.annotationsService.Write(uuid, lifecycle, platformVersion, anns) if err != nil { hh.log.WithUUID(uuid).WithTransactionID(tid).WithError(err).Error("failed writing annotations") msg := fmt.Sprintf("Error creating annotations (%v)", err) - if _, ok := err.(annotations.ValidationError); ok { - writeJSONError(w, msg, http.StatusBadRequest) - return - } hh.log.WithMonitoringEvent("SaveNeo4j", tid, hh.messageType).WithUUID(uuid).WithError(err).Error(msg) writeJSONError(w, msg, http.StatusServiceUnavailable) return @@ -236,8 +237,8 @@ func jsonMessage(msgText string) []byte { return []byte(fmt.Sprintf(`{"message":"%s"}`, msgText)) } -func decode(body io.Reader) (annotations.Annotations, error) { - var anns annotations.Annotations +func decode(body io.Reader) ([]interface{}, error) { + var anns []interface{} err := json.NewDecoder(body).Decode(&anns) return anns, err } diff --git a/http_handler_test.go b/http_handler_test.go index cfcbe0c..594b560 100644 --- a/http_handler_test.go +++ b/http_handler_test.go @@ -7,13 +7,13 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "os" "testing" - "github.com/Financial-Times/kafka-client-go/v3" - - "github.com/Financial-Times/annotations-rw-neo4j/v4/annotations" + "github.com/Financial-Times/cm-annotations-ontology/validator" logger "github.com/Financial-Times/go-logger/v2" + "github.com/Financial-Times/kafka-client-go/v3" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -30,7 +30,7 @@ const ( type HttpHandlerTestSuite struct { suite.Suite body []byte - annotations annotations.Annotations + annotations []interface{} annotationsService *mockAnnotationsService forwarder *mockForwarder message kafka.FTMessage @@ -40,12 +40,16 @@ type HttpHandlerTestSuite struct { tid string messageType string log *logger.UPPLogger + validator jsonValidator } func (suite *HttpHandlerTestSuite) SetupTest() { + os.Setenv("JSON_SCHEMAS_PATH", "./schemas") + os.Setenv("JSON_SCHEMA_NAME", "annotations-pac.json;annotations-next-video.json;annotations-v2.json") + suite.log = logger.NewUPPInfoLogger("annotations-rw") var err error - suite.body, err = ioutil.ReadFile("annotations/examplePutBody.json") + suite.body, err = ioutil.ReadFile("examplePutBody.json") assert.NoError(suite.T(), err, "Unexpected error") suite.annotations, err = decode(bytes.NewReader(suite.body)) @@ -57,6 +61,7 @@ func (suite *HttpHandlerTestSuite) SetupTest() { suite.healthCheckHandler = healthCheckHandler{} suite.originMap, suite.lifecycleMap, suite.messageType, err = readConfigMap("annotation-config.json") + suite.validator = validator.NewSchemaValidator(suite.log).GetJSONValidator() assert.NoError(suite.T(), err, "Unexpected error") } @@ -70,7 +75,7 @@ func (suite *HttpHandlerTestSuite) TestPutHandler_Success() { suite.forwarder.On("SendMessage", suite.tid, "http://cmdb.ft.com/systems/pac", bookmark, platformVersion, knownUUID, suite.annotations).Return(nil).Once() request := newRequest("PUT", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", suite.body) request.Header.Add("X-Request-Id", suite.tid) - handler := httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} + handler := httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} rec := httptest.NewRecorder() router(&handler, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusCreated == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusCreated)) @@ -81,7 +86,7 @@ func (suite *HttpHandlerTestSuite) TestPutHandler_Success() { func (suite *HttpHandlerTestSuite) TestPutHandler_ParseError() { request := newRequest("PUT", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", []byte(`{"id": "1234"}`)) request.Header.Add("X-Request-Id", suite.tid) - handler := httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} + handler := httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} rec := httptest.NewRecorder() router(&handler, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusBadRequest == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusBadRequest)) @@ -90,7 +95,7 @@ func (suite *HttpHandlerTestSuite) TestPutHandler_ParseError() { func (suite *HttpHandlerTestSuite) TestPutHandler_ValidationError() { request := newRequest("PUT", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", []byte(`"{"thing": {"prefLabel": "Apple"}`)) request.Header.Add("X-Request-Id", suite.tid) - handler := httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} + handler := httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} rec := httptest.NewRecorder() router(&handler, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusBadRequest == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusBadRequest)) @@ -98,7 +103,7 @@ func (suite *HttpHandlerTestSuite) TestPutHandler_ValidationError() { func (suite *HttpHandlerTestSuite) TestPutHandler_NotJson() { request := newRequest("PUT", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "text/html", suite.body) - handler := httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} + handler := httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} rec := httptest.NewRecorder() router(&handler, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusBadRequest == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusBadRequest)) @@ -108,28 +113,18 @@ func (suite *HttpHandlerTestSuite) TestPutHandler_WriteFailed() { suite.annotationsService.On("Write", knownUUID, annotationLifecycle, platformVersion, suite.annotations).Return("", errors.New("Write failed")) request := newRequest("PUT", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", suite.body) request.Header.Add("X-Request-Id", suite.tid) - handler := httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} + handler := httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} rec := httptest.NewRecorder() router(&handler, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusServiceUnavailable == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusServiceUnavailable)) } -func (suite *HttpHandlerTestSuite) TestPutHandler_InvalidPredicate() { - suite.annotationsService.On("Write", knownUUID, annotationLifecycle, platformVersion, suite.annotations).Return("", annotations.UnsupportedPredicateErr) - request := newRequest("PUT", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", suite.body) - request.Header.Add("X-Request-Id", suite.tid) - handler := httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} - rec := httptest.NewRecorder() - router(&handler, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) - assert.True(suite.T(), http.StatusBadRequest == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusBadRequest)) -} - func (suite *HttpHandlerTestSuite) TestPutHandler_ForwardingFailed() { suite.annotationsService.On("Write", knownUUID, annotationLifecycle, platformVersion, suite.annotations).Return(bookmark, nil) suite.forwarder.On("SendMessage", suite.tid, "http://cmdb.ft.com/systems/pac", bookmark, platformVersion, knownUUID, suite.annotations).Return(errors.New("forwarding failed")) request := newRequest("PUT", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", suite.body) request.Header.Add("X-Request-Id", suite.tid) - handler := httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} + handler := httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} rec := httptest.NewRecorder() router(&handler, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusInternalServerError == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusInternalServerError)) @@ -140,7 +135,7 @@ func (suite *HttpHandlerTestSuite) TestGetHandler_Success() { suite.annotationsService.On("Read", knownUUID, mock.Anything, annotationLifecycle).Return(suite.annotations, true, nil) request := newRequest("GET", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", nil) rec := httptest.NewRecorder() - router(&httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) + router(&httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusOK == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusOK)) expectedResponse, err := json.Marshal(suite.annotations) assert.NoError(suite.T(), err, "") @@ -151,7 +146,7 @@ func (suite *HttpHandlerTestSuite) TestGetHandler_NotFound() { suite.annotationsService.On("Read", knownUUID, mock.Anything, annotationLifecycle).Return(nil, false, nil) request := newRequest("GET", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", nil) rec := httptest.NewRecorder() - router(&httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) + router(&httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusNotFound == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusNotFound)) } @@ -159,14 +154,14 @@ func (suite *HttpHandlerTestSuite) TestGetHandler_ReadError() { suite.annotationsService.On("Read", knownUUID, mock.Anything, annotationLifecycle).Return(nil, false, errors.New("Read error")) request := newRequest("GET", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", nil) rec := httptest.NewRecorder() - router(&httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) + router(&httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusServiceUnavailable == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusServiceUnavailable)) } func (suite *HttpHandlerTestSuite) TestGetHandler_InvalidLifecycle() { request := newRequest("GET", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, "annotations-invalid"), "application/json", nil) rec := httptest.NewRecorder() - router(&httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) + router(&httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusBadRequest == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusBadRequest)) } @@ -174,7 +169,7 @@ func (suite *HttpHandlerTestSuite) TestDeleteHandler_Success() { suite.annotationsService.On("Delete", knownUUID, annotationLifecycle).Return(true, bookmark, nil) request := newRequest("DELETE", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", nil) rec := httptest.NewRecorder() - router(&httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) + router(&httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusNoContent == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusNoContent)) } @@ -182,7 +177,7 @@ func (suite *HttpHandlerTestSuite) TestDeleteHandler_NotFound() { suite.annotationsService.On("Delete", knownUUID, annotationLifecycle).Return(false, bookmark, nil) request := newRequest("DELETE", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", nil) rec := httptest.NewRecorder() - router(&httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) + router(&httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusNotFound == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusNotFound)) } @@ -190,7 +185,7 @@ func (suite *HttpHandlerTestSuite) TestDeleteHandler_DeleteError() { suite.annotationsService.On("Delete", knownUUID, annotationLifecycle).Return(false, bookmark, errors.New("Delete error")) request := newRequest("DELETE", fmt.Sprintf("/content/%s/annotations/%s", knownUUID, annotationLifecycle), "application/json", nil) rec := httptest.NewRecorder() - router(&httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) + router(&httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusServiceUnavailable == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusServiceUnavailable)) } @@ -198,7 +193,7 @@ func (suite *HttpHandlerTestSuite) TestCount_Success() { suite.annotationsService.On("Count", annotationLifecycle, mock.Anything, platformVersion).Return(10, nil) request := newRequest("GET", fmt.Sprintf("/content/annotations/%s/__count", annotationLifecycle), "application/json", nil) rec := httptest.NewRecorder() - router(&httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) + router(&httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log}, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusOK == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusOK)) } @@ -206,7 +201,7 @@ func (suite *HttpHandlerTestSuite) TestCount_CountError() { suite.annotationsService.On("Count", annotationLifecycle, mock.Anything, platformVersion).Return(0, errors.New("Count error")) request := newRequest("GET", fmt.Sprintf("/content/annotations/%s/__count", annotationLifecycle), "application/json", nil) rec := httptest.NewRecorder() - handler := httpHandler{suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} + handler := httpHandler{suite.validator, suite.annotationsService, suite.forwarder, suite.originMap, suite.lifecycleMap, suite.messageType, suite.log} router(&handler, &suite.healthCheckHandler, suite.log).ServeHTTP(rec, request) assert.True(suite.T(), http.StatusServiceUnavailable == rec.Code, fmt.Sprintf("Wrong response code, was %d, should be %d", rec.Code, http.StatusServiceUnavailable)) } diff --git a/main.go b/main.go index db469de..a60bc75 100644 --- a/main.go +++ b/main.go @@ -10,13 +10,15 @@ import ( "os/signal" "syscall" + "github.com/Financial-Times/cm-annotations-ontology/validator" + "github.com/Financial-Times/annotations-rw-neo4j/v4/annotations" "github.com/Financial-Times/annotations-rw-neo4j/v4/forwarder" cmneo4j "github.com/Financial-Times/cm-neo4j-driver" + "github.com/Financial-Times/kafka-client-go/v3" logger "github.com/Financial-Times/go-logger/v2" "github.com/Financial-Times/http-handlers-go/v2/httphandlers" - "github.com/Financial-Times/kafka-client-go/v3" status "github.com/Financial-Times/service-status-go/httphandlers" "github.com/gorilla/mux" @@ -144,7 +146,10 @@ func main() { } } + validator := validator.NewSchemaValidator(log) + hh := httpHandler{ + validator: validator.GetJSONValidator(), annotationsService: annotationsService, forwarder: f, originMap: originMap, @@ -160,6 +165,7 @@ func main() { healtcheckHandler.consumer = consumer qh = queueHandler{ + validator: validator.GetJSONValidator(), annotationsService: annotationsService, consumer: consumer, forwarder: f, diff --git a/mocks.go b/mocks.go index 7c66e31..ca57e43 100644 --- a/mocks.go +++ b/mocks.go @@ -5,7 +5,6 @@ import ( "github.com/Financial-Times/kafka-client-go/v3" - "github.com/Financial-Times/annotations-rw-neo4j/v4/annotations" "github.com/stretchr/testify/mock" ) @@ -13,7 +12,7 @@ type mockForwarder struct { mock.Mock } -func (mf *mockForwarder) SendMessage(transactionID string, originSystem string, bookmark string, platformVersion string, uuid string, annotations annotations.Annotations) error { +func (mf *mockForwarder) SendMessage(transactionID string, originSystem string, bookmark string, platformVersion string, uuid string, annotations interface{}) error { args := mf.Called(transactionID, originSystem, bookmark, platformVersion, uuid, annotations) return args.Error(0) } diff --git a/queue_handler.go b/queue_handler.go index fda330a..f6b890e 100644 --- a/queue_handler.go +++ b/queue_handler.go @@ -19,12 +19,6 @@ const ( cmsMessageType = "cms-content-published" ) -type queueMessage struct { - UUID string - Annotations annotations.Annotations `json:"annotations,omitempty"` - Suggestions annotations.Annotations `json:"suggestions,omitempty"` -} - type kafkaConsumer interface { Start(func(message kafka.FTMessage)) Close() error @@ -32,7 +26,12 @@ type kafkaConsumer interface { ConnectivityCheck() error } +type jsonValidator interface { + Validate(interface{}) error +} + type queueHandler struct { + validator jsonValidator annotationsService annotations.Service consumer kafkaConsumer forwarder forwarder.QueueForwarder @@ -69,33 +68,44 @@ func (qh *queueHandler) Ingest() { return } - annMsg := new(queueMessage) + var annMsg map[string]interface{} err = json.Unmarshal([]byte(message.Body), &annMsg) if err != nil { qh.log.WithTransactionID(tid).Error("Cannot process received message", tid) return } + contentUUID := annMsg["uuid"].(string) var bookmark string if qh.messageType == "Annotations" { - bookmark, err = qh.annotationsService.Write(annMsg.UUID, lifecycle, platformVersion, annMsg.Annotations) + err = qh.validate(annMsg["annotations"]) + if err != nil { + qh.log.WithError(err).Error("Validation error") + return + } + bookmark, err = qh.annotationsService.Write(contentUUID, lifecycle, platformVersion, annMsg["annotations"]) } else { - bookmark, err = qh.annotationsService.Write(annMsg.UUID, lifecycle, platformVersion, annMsg.Suggestions) + err = qh.validate(annMsg["suggestions"]) + if err != nil { + qh.log.WithError(err).Error("Validation error") + return + } + bookmark, err = qh.annotationsService.Write(contentUUID, lifecycle, platformVersion, annMsg["suggestions"]) } if err != nil { - qh.log.WithMonitoringEvent("SaveNeo4j", tid, qh.messageType).WithUUID(annMsg.UUID).WithError(err).Error("Cannot write to Neo4j") + qh.log.WithMonitoringEvent("SaveNeo4j", tid, qh.messageType).WithUUID(contentUUID).WithError(err).Error("Cannot write to Neo4j") return } - qh.log.WithMonitoringEvent("SaveNeo4j", tid, qh.messageType).WithUUID(annMsg.UUID).Infof("%s successfully written in Neo4j", qh.messageType) + qh.log.WithMonitoringEvent("SaveNeo4j", tid, qh.messageType).WithUUID(contentUUID).Infof("%s successfully written in Neo4j", qh.messageType) //forward message to the next queue if qh.forwarder != nil { - qh.log.WithTransactionID(tid).WithUUID(annMsg.UUID).Debug("Forwarding message to the next queue") - err := qh.forwarder.SendMessage(tid, originSystem, bookmark, platformVersion, annMsg.UUID, annMsg.Annotations) + qh.log.WithTransactionID(tid).WithUUID(contentUUID).Debug("Forwarding message to the next queue") + err := qh.forwarder.SendMessage(tid, originSystem, bookmark, platformVersion, contentUUID, annMsg["annotations"]) if err != nil { - qh.log.WithError(err).WithUUID(annMsg.UUID).WithTransactionID(tid).Error("Could not forward a message to kafka") + qh.log.WithError(err).WithUUID(contentUUID).WithTransactionID(tid).Error("Could not forward a message to kafka") return } return @@ -115,3 +125,13 @@ func (qh *queueHandler) getSourceFromHeader(originSystem string) (string, string } return annotationLifecycle, platformVersion, nil } + +func (qh *queueHandler) validate(annotations interface{}) error { + for _, annotation := range annotations.([]interface{}) { + err := qh.validator.Validate(annotation) + if err != nil { + return err + } + } + return nil +} diff --git a/queue_handler_test.go b/queue_handler_test.go index 9b9660b..d0f7fe4 100644 --- a/queue_handler_test.go +++ b/queue_handler_test.go @@ -3,8 +3,10 @@ package main import ( "encoding/json" "io/ioutil" + "os" "testing" + "github.com/Financial-Times/cm-annotations-ontology/validator" "github.com/Financial-Times/kafka-client-go/v3" "github.com/Financial-Times/annotations-rw-neo4j/v4/forwarder" @@ -19,7 +21,7 @@ type QueueHandlerTestSuite struct { headers map[string]string body []byte message kafka.FTMessage - queueMessage queueMessage + queueMessage map[string]interface{} annotationsService *mockAnnotationsService forwarder *mockForwarder originMap map[string]string @@ -29,10 +31,13 @@ type QueueHandlerTestSuite struct { bookmark string messageType string log *logger.UPPLogger + validator jsonValidator } func (suite *QueueHandlerTestSuite) SetupTest() { var err error + os.Setenv("JSON_SCHEMAS_PATH", "./schemas") + os.Setenv("JSON_SCHEMA_NAME", "annotations-pac.json;annotations-next-video.json;annotations-v2.json") suite.log = logger.NewUPPInfoLogger("annotations-rw") suite.tid = "tid_sample" suite.originSystem = "http://cmdb.ft.com/systems/pac" @@ -47,6 +52,8 @@ func (suite *QueueHandlerTestSuite) SetupTest() { suite.annotationsService = new(mockAnnotationsService) suite.originMap, suite.lifecycleMap, suite.messageType, err = readConfigMap("annotation-config.json") + suite.validator = validator.NewSchemaValidator(suite.log).GetJSONValidator() + assert.NoError(suite.T(), err, "Unexpected config error") } @@ -55,10 +62,11 @@ func TestQueueHandlerTestSuite(t *testing.T) { } func (suite *QueueHandlerTestSuite) TestQueueHandler_Ingest() { - suite.annotationsService.On("Write", suite.queueMessage.UUID, annotationLifecycle, platformVersion, suite.queueMessage.Annotations).Return(suite.bookmark, nil) - suite.forwarder.On("SendMessage", suite.tid, suite.originSystem, suite.bookmark, platformVersion, suite.queueMessage.UUID, suite.queueMessage.Annotations).Return(nil) + suite.annotationsService.On("Write", suite.queueMessage["uuid"], annotationLifecycle, platformVersion, suite.queueMessage["annotations"]).Return(suite.bookmark, nil) + suite.forwarder.On("SendMessage", suite.tid, suite.originSystem, suite.bookmark, platformVersion, suite.queueMessage["uuid"], suite.queueMessage["annotations"]).Return(nil) qh := &queueHandler{ + validator: suite.validator, annotationsService: suite.annotationsService, consumer: mockConsumer{message: suite.message}, forwarder: suite.forwarder, @@ -69,14 +77,15 @@ func (suite *QueueHandlerTestSuite) TestQueueHandler_Ingest() { } qh.Ingest() - suite.annotationsService.AssertCalled(suite.T(), "Write", suite.queueMessage.UUID, annotationLifecycle, platformVersion, suite.queueMessage.Annotations) - suite.forwarder.AssertCalled(suite.T(), "SendMessage", suite.tid, suite.originSystem, suite.bookmark, platformVersion, suite.queueMessage.UUID, suite.queueMessage.Annotations) + suite.annotationsService.AssertCalled(suite.T(), "Write", suite.queueMessage["uuid"], annotationLifecycle, platformVersion, suite.queueMessage["annotations"]) + suite.forwarder.AssertCalled(suite.T(), "SendMessage", suite.tid, suite.originSystem, suite.bookmark, platformVersion, suite.queueMessage["uuid"], suite.queueMessage["annotations"]) } func (suite *QueueHandlerTestSuite) TestQueueHandler_Ingest_ProducerNil() { - suite.annotationsService.On("Write", suite.queueMessage.UUID, annotationLifecycle, platformVersion, suite.queueMessage.Annotations).Return(suite.bookmark, nil) + suite.annotationsService.On("Write", suite.queueMessage["uuid"], annotationLifecycle, platformVersion, suite.queueMessage["annotations"]).Return(suite.bookmark, nil) qh := queueHandler{ + validator: suite.validator, annotationsService: suite.annotationsService, consumer: mockConsumer{message: suite.message}, forwarder: nil, @@ -87,7 +96,7 @@ func (suite *QueueHandlerTestSuite) TestQueueHandler_Ingest_ProducerNil() { } qh.Ingest() - suite.annotationsService.AssertCalled(suite.T(), "Write", suite.queueMessage.UUID, annotationLifecycle, platformVersion, suite.queueMessage.Annotations) + suite.annotationsService.AssertCalled(suite.T(), "Write", suite.queueMessage["uuid"], annotationLifecycle, platformVersion, suite.queueMessage["annotations"]) suite.forwarder.AssertNumberOfCalls(suite.T(), "SendMessage", 0) } @@ -96,6 +105,7 @@ func (suite *QueueHandlerTestSuite) TestQueueHandler_Ingest_JsonError() { message := kafka.NewFTMessage(suite.headers, string(body)) qh := &queueHandler{ + validator: suite.validator, annotationsService: suite.annotationsService, consumer: mockConsumer{message: message}, forwarder: suite.forwarder, @@ -114,6 +124,7 @@ func (suite *QueueHandlerTestSuite) TestQueueHandler_Ingest_InvalidOrigin() { message := kafka.NewFTMessage(suite.headers, string(suite.body)) qh := &queueHandler{ + validator: suite.validator, annotationsService: suite.annotationsService, consumer: mockConsumer{message: message}, forwarder: suite.forwarder, diff --git a/schemas/annotations-next-video.json b/schemas/annotations-next-video.json new file mode 100644 index 0000000..d28e99e --- /dev/null +++ b/schemas/annotations-next-video.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://cm-delivery-prod.ft.com/schema/annotations-next-video+json", + "title": "Next Video Annotations", + "type": "object", + "description": "Schema for Next Video Annotations", + "properties": { + "id": { + "type": "string", + "pattern": ".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$", + "description": "ID of the related concept" + }, + "predicate": { + "type": "string", + "description": "Predicate of the annotation", + "enum": [ + "mentions", + "isClassifiedBy", + "implicitlyClassifiedBy", + "about", + "isPrimarilyClassifiedBy", + "majorMentions", + "hasAuthor", + "hasContributor", + "hasDisplayTag", + "hasBrand" + ] + }, + "relevanceScore": { + "type": "number", + "description": "Relevance score of the annotation" + }, + "confidenceScore": { + "type": "number", + "description": "Confidence score of the annotation" + } + }, + "required": [ + "id", + "predicate", + "relevanceScore", + "confidenceScore" + ], + "additionalProperties": false +} diff --git a/schemas/annotations-pac.json b/schemas/annotations-pac.json new file mode 100644 index 0000000..2fb300f --- /dev/null +++ b/schemas/annotations-pac.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://cm-delivery-prod.ft.com/schema/annotations-pac+json", + "title": "PAC Annotations", + "type": "object", + "description": "Schema for PAC Annotations", + "properties": { + "id": { + "type": "string", + "pattern": ".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$", + "description": "ID of the related concept" + }, + "predicate": { + "type": "string", + "description": "Predicate of the annotation", + "enum": [ + "http://www.ft.com/ontology/annotation/mentions", + "http://www.ft.com/ontology/classification/isClassifiedBy", + "http://www.ft.com/ontology/implicitlyClassifiedBy", + "http://www.ft.com/ontology/annotation/about", + "http://www.ft.com/ontology/isPrimarilyClassifiedBy", + "http://www.ft.com/ontology/majorMentions", + "http://www.ft.com/ontology/annotation/hasAuthor", + "http://www.ft.com/ontology/hasContributor", + "http://www.ft.com/ontology/hasDisplayTag", + "http://www.ft.com/ontology/hasBrand" + ] + } + }, + "required": [ + "id", + "predicate" + ], + "additionalProperties": false +} diff --git a/schemas/annotations-v2.json b/schemas/annotations-v2.json new file mode 100644 index 0000000..960255c --- /dev/null +++ b/schemas/annotations-v2.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://cm-delivery-prod.ft.com/schema/annotations-v2+json", + "title": "V2 Annotations", + "type": "object", + "description": "Schema for V2 Annotations", + "properties": { + "id": { + "type": "string", + "pattern": ".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$", + "description": "ID of the related concept" + }, + "prefLabel": { + "type": "string", + "description": "PrefLabel of the related concept" + }, + "types": { + "type": "array", + "description": "Types of the related concept", + "items": { + "type": "string", + "enum": [ + "http://www.ft.com/ontology/core/Thing", + "http://www.ft.com/ontology/concept/Concept", + "http://www.ft.com/ontology/person/Person", + "http://www.ft.com/ontology/organisation/Organisation" + ] + } + }, + "predicate": { + "type": "string", + "description": "Predicate of the annotation", + "enum": [ + "mentions" + ] + }, + "relevanceScore": { + "type": "number", + "description": "Relevance score of the annotation" + }, + "confidenceScore": { + "type": "number", + "description": "Confidence score of the annotation" + }, + "annotatedBy": { + "type": "string", + "description": "The entity that created the annotation" + }, + "annotatedDate": { + "type": "string", + "format": "date-time", + "description": "The creation date of the annotation" + }, + "annotatedDateEpoch": { + "type": "number", + "description": "The creation date of the annotation in Unix Epoch Time" + } + }, + "required": [ + "id", + "predicate", + "relevanceScore", + "confidenceScore", + "annotatedBy", + "annotatedDate", + "annotatedDateEpoch" + ], + "additionalProperties": false +}