Skip to content

Commit

Permalink
Integrate the cm-annotations-ontology library
Browse files Browse the repository at this point in the history
  • Loading branch information
@asparuhft
asparuhft committed Sep 12, 2023
1 parent a7244d3 commit 199cc63
Show file tree
Hide file tree
Showing 27 changed files with 518 additions and 733 deletions.
2 changes: 2 additions & 0 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ RUN BUILDINFO_PACKAGE="${ORG_PATH}/service-status-go/buildinfo." \
&& BUILDER="builder=$(go version)" \
&& LDFLAGS="-s -w -X '"${BUILDINFO_PACKAGE}$VERSION"' -X '"${BUILDINFO_PACKAGE}$DATETIME"' -X '"${BUILDINFO_PACKAGE}$REPOSITORY"' -X '"${BUILDINFO_PACKAGE}$REVISION"' -X '"${BUILDINFO_PACKAGE}$BUILDER"'" \
&& git config --global url."https://${GITHUB_USERNAME}:${GITHUB_TOKEN}@github.com".insteadOf "https://github.com" \
&& mkdir -p /artifacts/schemas/ \
&& cp -r /${SRC_FOLDER}/schemas /artifacts/schemas \
&& CGO_ENABLED=0 go build -mod=readonly -a -o /artifacts/${PROJECT} -ldflags="${LDFLAGS}"

COPY ./suggestion-config.json /artifacts/suggestion-config.json
Expand Down
209 changes: 29 additions & 180 deletions annotations/cypher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,22 @@ import (
"errors"
"fmt"
"net/url"
"path"
"regexp"
"strings"

cmneo4j "github.com/Financial-Times/cm-neo4j-driver"
)
"github.com/Financial-Times/cm-annotations-ontology/model"

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")
"github.com/Financial-Times/cm-annotations-ontology/neo4j"

var UnsupportedPredicateErr = errors.New("Unsupported predicate")
cmneo4j "github.com/Financial-Times/cm-neo4j-driver"
)

// Service interface. Compatible with the baserwftapp service EXCEPT for
// 1) the Write function, which has signature Write(thing interface{}) error...
// 2) the DecodeJson function, which has signature DecodeJSON(*json.Decoder) (thing interface{}, identity string, err error)
// The problem is that we have a list of things, and the uuid is for a related OTHER thing
// TODO - move to implement a shared defined Service interface?
type Service interface {
Write(contentUUID string, annotationLifecycle string, platformVersion string, thing interface{}) (bookmark string, err error)
Write(contentUUID string, annotationLifecycle string, platformVersion string, anns interface{}) (bookmark string, err error)
Read(contentUUID string, bookmark string, annotationLifecycle string) (thing interface{}, found bool, err error)
Delete(contentUUID string, annotationLifecycle string) (found bool, bookmark string, err error)
Check() (err error)
Expand All @@ -37,10 +35,6 @@ type service struct {
publicAPIURL string
}

const (
nextVideoAnnotationsLifecycle = "annotations-next-video"
)

// NewCypherAnnotationsService instantiate driver
func NewCypherAnnotationsService(driver *cmneo4j.Driver, publicAPIURL string) (Service, error) {
_, err := url.ParseRequestURI(publicAPIURL)
Expand All @@ -53,54 +47,34 @@ func NewCypherAnnotationsService(driver *cmneo4j.Driver, publicAPIURL string) (S

// DecodeJSON decodes to a list of annotations, for ease of use this is a struct itself
func (s service) DecodeJSON(dec *json.Decoder) (interface{}, error) {
a := Annotations{}
a := model.Annotations{}
err := dec.Decode(&a)
return a, err
}

func (s service) Read(contentUUID string, bookmark string, annotationLifecycle string) (thing interface{}, found bool, err error) {
results := []Annotation{}
statement := `
MATCH (c:Thing{uuid:$contentUUID})-[rel{lifecycle:$annotationLifecycle}]->(cc:Thing)
RETURN
cc.uuid as id,
cc.preflabel as prefLabel,
labels(cc) as types,
type(rel) as predicate,
rel.relevanceScore as relevanceScore,
rel.confidenceScore as confidenceScore,
rel.annotatedBy as annotatedBy,
rel.annotatedDate as annotatedDate
ORDER BY id`

query := []*cmneo4j.Query{{
Cypher: statement,
Params: map[string]interface{}{
"contentUUID": contentUUID,
"annotationLifecycle": annotationLifecycle,
},
Result: &results,
}}
func (s service) Read(contentUUID string, bookmark string, annotationLifecycle string) (ann interface{}, found bool, err error) {
query, results := neo4j.GetReadQuery(contentUUID, annotationLifecycle)
_, err = s.driver.ReadMultiple(query, []string{bookmark})
if errors.Is(err, cmneo4j.ErrNoResultsFound) {
return Annotations{}, false, nil
return model.Annotations{}, false, nil
}
if err != nil {
return Annotations{}, false, fmt.Errorf("error executing delete queries: %w", err)
return model.Annotations{}, false, fmt.Errorf("error executing delete queries: %w", err)
}

for idx := range results {
mapToResponseFormat(&results[idx], s.publicAPIURL)
mappedResults := *results
for idx := range mappedResults {
mapToResponseFormat(&mappedResults[idx], s.publicAPIURL)
}

return Annotations(results), true, nil
return results, true, nil
}

// Delete removes all the annotations for this content. Ignore the nodes on either end -
// may leave nodes that are only 'things' inserted by this writer: clean up
// as a result of this will need to happen externally if required
func (s service) Delete(contentUUID string, annotationLifecycle string) (bool, string, error) {
query := buildDeleteQuery(contentUUID, annotationLifecycle, true)
query := neo4j.BuildDeleteQuery(contentUUID, annotationLifecycle, true)

bookmark, err := s.driver.WriteMultiple([]*cmneo4j.Query{query}, nil)
if err != nil {
Expand All @@ -117,23 +91,25 @@ func (s service) Delete(contentUUID string, annotationLifecycle string) (bool, s

// Write a set of annotations associated with a piece of content. Any annotations
// already there will be removed
func (s service) Write(contentUUID string, annotationLifecycle string, platformVersion string, thing interface{}) (string, error) {
annotationsToWrite, ok := thing.(Annotations)
if ok == false {
return "", errors.New("thing is not of type Annotations")
}
func (s service) Write(contentUUID string, annotationLifecycle string, platformVersion string, anns interface{}) (string, error) {
if contentUUID == "" {
return "", errors.New("content uuid is required")
}

if err := validateAnnotations(&annotationsToWrite); err != nil {
return "", err
queries := append([]*cmneo4j.Query{}, neo4j.BuildDeleteQuery(contentUUID, annotationLifecycle, false))

annotations, ok := anns.([]interface{})
if !ok {
return "", errors.New("error in casting annotations")
}

queries := append([]*cmneo4j.Query{}, buildDeleteQuery(contentUUID, annotationLifecycle, false))
for _, annotationToWrite := range annotations {
annotation, ok := annotationToWrite.(map[string]interface{})
if !ok {
return "", errors.New("error in casting annotation")
}

for _, annotationToWrite := range annotationsToWrite {
query, err := createAnnotationQuery(contentUUID, annotationToWrite, platformVersion, annotationLifecycle)
query, err := neo4j.CreateAnnotationQuery(contentUUID, annotation, platformVersion, annotationLifecycle)
if err != nil {
return "", fmt.Errorf("create annotation query failed: %w", err)
}
Expand All @@ -153,21 +129,7 @@ func (s service) Check() error {
}

func (s service) Count(annotationLifecycle string, bookmark string, platformVersion string) (int, error) {
var results []struct {
Count int `json:"c"`
}

query := []*cmneo4j.Query{{
Cypher: `MATCH ()-[r{platformVersion:$platformVersion}]->()
WHERE r.lifecycle = $lifecycle
OR r.lifecycle IS NULL
RETURN count(r) as c`,
Params: map[string]interface{}{
"platformVersion": platformVersion,
"lifecycle": annotationLifecycle,
},
Result: &results,
}}
query, results := neo4j.Count(annotationLifecycle, platformVersion)

_, err := s.driver.ReadMultiple(query, []string{bookmark})
if errors.Is(err, cmneo4j.ErrNoResultsFound) {
Expand All @@ -187,123 +149,10 @@ func (s service) Initialise() error {
return err
}

func createAnnotationRelationship(relation string) (statement string) {
stmt := `
MERGE (content:Thing{uuid:$contentID})
MERGE (concept:Thing{uuid:$conceptID})
MERGE (content)-[pred:%s {lifecycle:$annotationLifecycle}]->(concept)
SET pred=$annProps
`
statement = fmt.Sprintf(stmt, relation)
return statement
}

func getRelationshipFromPredicate(predicate string) (string, error) {
r, ok := relations[extractPredicateFromURI(predicate)]
if !ok {
return "", UnsupportedPredicateErr
}
return r, nil
}

func createAnnotationQuery(contentUUID string, ann Annotation, platformVersion string, annotationLifecycle string) (*cmneo4j.Query, error) {
thingID, err := extractUUIDFromURI(ann.ID)
if err != nil {
return nil, err
}

params := map[string]interface{}{}
params["platformVersion"] = platformVersion
params["lifecycle"] = annotationLifecycle

if ann.AnnotatedBy != "" {
params["annotatedBy"], err = extractUUIDFromURI(ann.AnnotatedBy)
if err != nil {
return nil, err
}
}
if ann.AnnotatedDate != "" {
params["annotatedDateEpoch"] = ann.AnnotatedDateEpoch
params["annotatedDate"] = ann.AnnotatedDate
}
if ann.RelevanceScore != 0.0 {
params["relevanceScore"] = ann.RelevanceScore
}
if ann.ConfidenceScore != 0.0 {
params["confidenceScore"] = ann.ConfidenceScore
}

relation, err := getRelationshipFromPredicate(ann.Predicate)
if err != nil {
return nil, err
}

query := &cmneo4j.Query{
Cypher: createAnnotationRelationship(relation),
Params: map[string]interface{}{
"contentID": contentUUID,
"conceptID": thingID,
"annotationLifecycle": annotationLifecycle,
"annProps": params,
},
}

return query, nil
}

func buildDeleteQuery(contentUUID string, annotationLifecycle string, includeStats bool) *cmneo4j.Query {
statement := `OPTIONAL MATCH (:Thing{uuid:$contentID})-[r{lifecycle:$annotationLifecycle}]->(t:Thing)
DELETE r`
query := &cmneo4j.Query{
Cypher: statement,
Params: map[string]interface{}{
"contentID": contentUUID,
"annotationLifecycle": annotationLifecycle,
},
IncludeSummary: includeStats,
}
return query
}

func validateAnnotations(annotations *Annotations) error {
//TODO - for consistency, we should probably just not create the annotation?
for _, annotation := range *annotations {
if annotation.ID == "" {
return ValidationError{fmt.Sprintf("Concept uuid missing for annotation %+v", annotation)}
}
}
return nil
}

// ValidationError is thrown when the annotations are not valid because mandatory information is missing
type ValidationError struct {
Msg string
}

func (v ValidationError) Error() string {
return v.Msg
}

func mapToResponseFormat(ann *Annotation, publicAPIURL string) {
func mapToResponseFormat(ann *model.Annotation, publicAPIURL string) {
ann.ID = thingURL(ann.ID, publicAPIURL)
if ann.AnnotatedBy != "" {
ann.AnnotatedBy = thingURL(ann.AnnotatedBy, publicAPIURL)
}
}

func thingURL(uuid, baseURL string) string {
return strings.TrimRight(baseURL, "/") + "/things/" + uuid
}

func extractUUIDFromURI(uri string) (string, error) {
result := uuidExtractRegex.FindStringSubmatch(uri)
if len(result) == 2 {
return result[1], nil
}
return "", fmt.Errorf("couldn't extract uuid from uri %s", uri)
}

func extractPredicateFromURI(uri string) string {
_, result := path.Split(uri)
return result
}
Loading

0 comments on commit 199cc63

Please sign in to comment.