Finished off support of both new and concordance models

people/cypher.go

@@ -83,7 +83,9 @@ type neoReadStruct struct {
 
 func (pcw CypherDriver) Read(uuid string) (Person, bool, error) {
 	person := Person{}
-	results := []neoReadStruct{}
+	results := []struct {
+		Rs []neoReadStruct
+	}{}
 
 	query := &neoism.CypherQuery{
 		Statement: `
@@ -92,32 +94,31 @@ func (pcw CypherDriver) Read(uuid string) (Person, bool, error) {
 						MATCH (canonical)<-[:EQUIVALENT_TO]-(p:Person)
                         OPTIONAL MATCH (p)<-[:HAS_MEMBER]-(m:Membership)
                         OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
-                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:Concept)
-                        WITH    p,
+                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:MembershipRole)
+                        WITH    canonical,
                                 { id:o.uuid, types:labels(o), prefLabel:o.prefLabel} as o,
                                 { id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}] } as m,
                                 { id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvents:[{startedAt:rr.inceptionDate}, {endedAt:rr.terminationDate}] } as r
-                        WITH p, m, o, collect(r) as r ORDER BY o.uuid DESC
-                        WITH p, collect({m:m, o:o, r:r}) as m
-                        WITH m, { id:p.uuid, types:labels(p), prefLabel:p.prefLabel, labels:p.aliases,
-												     birthYear:p.birthYear, salutation:p.salutation, emailAddress:p.emailAddress,
-														 twitterHandle:p.twitterHandle, facebookProfile:p.facebookProfile, linkedinProfile:p.linkedinProfile,
-														 imageURL:p.imageURL, Description:p.description, descriptionXML:p.descriptionXML} as p
+                        WITH canonical, m, o, collect(r) as r ORDER BY o.uuid DESC
+                        WITH canonical, collect({m:m, o:o, r:r}) as m
+                        WITH m, { ID:canonical.prefUUID, types:labels(canonical), prefLabel:canonical.prefLabel, labels:canonical.aliases,
+								birthYear:canonical.birthYear, salutation:canonical.salutation, emailAddress:canonical.emailAddress,
+								twitterHandle:canonical.twitterHandle, facebookProfile:canonical.facebookProfile, linkedinProfile:canonical.linkedinProfile,
+								imageURL:canonical.imageURL, Description:canonical.description, descriptionXML:canonical.descriptionXML} as p
                         RETURN collect ({p:p, m:m}) as rs
                         `,
 		Parameters: neoism.Props{"uuid": uuid},
 		Result:     &results,
 	}
 
 	err := pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
-
 	if err != nil {
 		log.WithError(err).WithField("UUID", uuid).Info("Error Querying Neo4J for a Person")
 		return Person{}, true, err
 	}
 
 	// TODO Sort this out by using pointers
-	if results[0].P.ID == "" {
+	if (len(results[0].Rs) == 0  || results[0].Rs[0].P.ID == "" ){
 		p, f, e := pcw.ReadOldConcordanceModel(uuid)
 		return p, f, e
 	}
@@ -130,7 +131,7 @@ func (pcw CypherDriver) Read(uuid string) (Person, bool, error) {
 		return Person{}, true, errors.New(errMsg)
 	}
 
-	person = neoReadStructToPerson(results[0], pcw.env)
+	person = neoReadStructToPerson(results[0].Rs[0], pcw.env)
 	return person, true, nil
 }
 
@@ -146,7 +147,7 @@ func (pcw CypherDriver) ReadOldConcordanceModel(uuid string) (person Person, fou
                         MATCH (identifier)-[:IDENTIFIES]->(p:Person)
                         OPTIONAL MATCH (p)<-[:HAS_MEMBER]-(m:Membership)
                         OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
-                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:Concept)
+                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:MembershipRole)
                         WITH    p,
                                 { id:o.uuid, types:labels(o), prefLabel:o.prefLabel} as o,
                                 { id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}] } as m,
@@ -164,7 +165,6 @@ func (pcw CypherDriver) ReadOldConcordanceModel(uuid string) (person Person, fou
 	}
 
 	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
-
 	if err != nil || len(results) == 0 || len(results[0].Rs) == 0 {
 		return Person{}, false, err
 	} else if len(results) != 1 && len(results[0].Rs) != 1 {
@@ -197,57 +197,64 @@ func neoReadStructToPerson(neo neoReadStruct, env string) Person {
 	public.FacebookProfile = neo.P.FacebookProfile
 	public.ImageURL = neo.P.ImageURL
 
-	if len(neo.M) == 1 && (neo.M[0].M.ID == "") {
-		public.Memberships = make([]Membership, 0, 0)
-	} else {
-		public.Memberships = make([]Membership, len(neo.M))
-		for mIdx, neoMem := range neo.M {
-			membership := Membership{}
-			membership.Title = neoMem.M.PrefLabel
-			membership.Types = mapper.TypeURIs(neoMem.M.Types)
-			membership.DirectType = filterToMostSpecificType(neoMem.M.Types)
-			membership.Organisation = Organisation{}
-			membership.Organisation.Thing = Thing{}
-			membership.Organisation.ID = mapper.IDURL(neoMem.O.ID)
-			membership.Organisation.APIURL = mapper.APIURL(neoMem.O.ID, neoMem.O.Types, env)
-			membership.Organisation.Types = mapper.TypeURIs(neoMem.O.Types)
-			membership.Organisation.DirectType = filterToMostSpecificType(neoMem.O.Types)
-			membership.Organisation.PrefLabel = neoMem.O.PrefLabel
-			if len(neoMem.O.Labels) > 0 {
-				membership.Organisation.Labels = &neoMem.O.Labels
-			}
-			if a, b := changeEvent(neoMem.M.ChangeEvents); a == true {
-				membership.ChangeEvents = b
-			}
-			membership.Roles = make([]Role, len(neoMem.R))
-			for rIdx, neoRole := range neoMem.R {
-				role := Role{}
-				role.Thing = Thing{}
-				role.ID = mapper.IDURL(neoRole.ID)
-				role.APIURL = mapper.APIURL(neoRole.ID, neoRole.Types, env)
-				role.Types = mapper.TypeURIs(neoRole.Types)
-				role.DirectType = filterToMostSpecificType(neoRole.Types)
-				role.PrefLabel = neoRole.PrefLabel
-				if a, b := changeEvent(neoRole.ChangeEvents); a == true {
-					role.ChangeEvents = b
+	if len(neo.M) > 0 {
+		memberships := []Membership{}
+		for _, neoMem := range neo.M {
+			if neoMem.M.ID != "" && neoMem.O.ID != "" && len(neoMem.R) > 0  {
+				membership := Membership{}
+				membership.Title = neoMem.M.PrefLabel
+				membership.Types = mapper.TypeURIs(neoMem.M.Types)
+				membership.DirectType = filterToMostSpecificType(neoMem.M.Types)
+				membership.Organisation = Organisation{}
+				membership.Organisation.Thing = Thing{}
+				membership.Organisation.ID = mapper.IDURL(neoMem.O.ID)
+				membership.Organisation.APIURL = mapper.APIURL(neoMem.O.ID, neoMem.O.Types, env)
+				membership.Organisation.Types = mapper.TypeURIs(neoMem.O.Types)
+				membership.Organisation.DirectType = filterToMostSpecificType(neoMem.O.Types)
+				membership.Organisation.PrefLabel = neoMem.O.PrefLabel
+				if len(neoMem.O.Labels) > 0 {
+					membership.Organisation.Labels = neoMem.O.Labels
+				}
+				if a, b := changeEvent(neoMem.M.ChangeEvents); a == true {
+					membership.ChangeEvents = b
 				}
 
-				membership.Roles[rIdx] = role
+				roles := []Role{}
+				for _, neoRole := range neoMem.R {
+					if neoRole.ID != "" {
+						role := Role{}
+						role.Thing = Thing{}
+						role.ID = mapper.IDURL(neoRole.ID)
+						role.APIURL = mapper.APIURL(neoRole.ID, neoRole.Types, env)
+						role.Types = mapper.TypeURIs(neoRole.Types)
+						role.DirectType = filterToMostSpecificType(neoRole.Types)
+						role.PrefLabel = neoRole.PrefLabel
+						if a, b := changeEvent(neoRole.ChangeEvents); a == true {
+							role.ChangeEvents = b
+						}
+						roles = append(roles, role)
+					}
+				}
+				if (len(roles) > 0) {
+					membership.Roles = roles
+					memberships = append(memberships, membership)
+				}
 			}
-			public.Memberships[mIdx] = membership
+			public.Memberships = memberships
 		}
 	}
+
 	return public
 }
 
-func changeEvent(neoChgEvts []neoChangeEvent) (bool, *[]ChangeEvent) {
+func changeEvent(neoChgEvts []neoChangeEvent) (bool, []ChangeEvent) {
 	var results []ChangeEvent
 	currentLayout := "2006-01-02T15:04:05.999Z"
 	layout := "2006-01-02T15:04:05Z"
 
 	if neoChgEvts[0].StartedAt == "" && neoChgEvts[1].EndedAt == "" {
 		results = make([]ChangeEvent, 0, 0)
-		return false, &results
+		return false, results
 	}
 	for _, neoChgEvt := range neoChgEvts {
 		if neoChgEvt.StartedAt != "" {
@@ -259,7 +266,7 @@ func changeEvent(neoChgEvts []neoChangeEvent) (bool, *[]ChangeEvent) {
 			results = append(results, ChangeEvent{EndedAt: t.Format(layout)})
 		}
 	}
-	return true, &results
+	return true, results
 }
 
 func filterToMostSpecificType(unfilteredTypes []string) string {

people/cypher_test.go

@@ -154,7 +154,7 @@ func TestNeoReadPersonWithAlternateUPPID(t *testing.T) {
 
 // New Model and backwards compatibility tests
 func TestNewModelWithFullyNewModelMembershipRelatedConcepts(t *testing.T) {
-	// New model to new model fully Type Org and Person and Role
+	//New model to new model fully Type Org and Person and Role
 
 	defer cleanDB(db, t, "7d0738b1-0ea2-47cb-bb82-e86744b389f0", "184cbe9b-b630-40d5-a5d0-99ecabd7fd86", "7ceeafe5-9f9a-4315-b3da-a5b4b69c013a", "8cdff2ba-3062-471e-b98a-7ee961239cd2")
 	writeJSONToConceptsService(t, "./fixtures/newModel/MembershipRole-SmartyPants-7d0738b1-0ea2-47cb-bb82-e86744b389f0.json")
@@ -166,12 +166,35 @@ func TestNewModelWithFullyNewModelMembershipRelatedConcepts(t *testing.T) {
 	readConceptAndCompare(t, person, "7ceeafe5-9f9a-4315-b3da-a5b4b69c013a")
 }
 
-func TestNewModelWithFullyOldModelMembershipRelatedConcepts(t *testing.T) {
-	// New model to old model fully Type Org - Some form of hybrid of both old and new
-}
+// TODO: When we concord to Factset we will need to handle a mixture of old model and new model
 
 func TestNewModelWithThingOnlyMembershipRelatedConceptsDoesNotReturnMembership(t *testing.T) {
 	// New model to org/person/role that is only a Thing - No membership should be returned
+
+	defer cleanDB(db, t, "ef0921e4-c862-43ac-8936-f345b9fb131a", "7ceeafe5-9f9a-4315-b3da-a5b4b69c013a", "0ee8e7b7-bac9-4db1-b94b-5605ce1d2907", "ac4be3c3-6dc1-4966-9cc5-ac824780f631")
+
+	writeJSONToService(t, membershipsDriver, "./fixtures/oldModel/Membership-Shirley-Rooney-ef0921e4-c862-43ac-8936-f345b9fb131a.json")
+	writeJSONToConceptsService(t, "./fixtures/newModel/Person-Shirley-Rooney-7ceeafe5-9f9a-4315-b3da-a5b4b69c013a.json")
+
+	person := Person{
+		Thing: Thing{
+			ID: "http://api.ft.com/things/7ceeafe5-9f9a-4315-b3da-a5b4b69c013a",
+			APIURL: "http://api.ft.com/people/7ceeafe5-9f9a-4315-b3da-a5b4b69c013a",
+			PrefLabel: "Shirley Rooney",
+		},
+		Types: []string{
+			"http://www.ft.com/ontology/core/Thing",
+			"http://www.ft.com/ontology/concept/Concept",
+			"http://www.ft.com/ontology/person/Person",
+		},
+		Memberships: []Membership{},
+		DirectType: "http://www.ft.com/ontology/person/Person",
+		TwitterHandle: "@something",
+		EmailAddress: "test@example.com",
+		DescriptionXML:"Some text containing <strong>markup</strong>",
+
+	}
+	readConceptAndCompare(t, person, "7ceeafe5-9f9a-4315-b3da-a5b4b69c013a")
 }
 
 func readConceptAndCompare(t *testing.T, expected Person, uuid string) {
@@ -183,11 +206,11 @@ func readConceptAndCompare(t *testing.T, expected Person, uuid string) {
 	assert.NotNil(t, actual)
 
 	sort.Slice(expected.Memberships, func(i, j int) bool {
-		return expected.Memberships[i].Title < expected.Memberships[j].Title
+		return expected.Memberships[i].Organisation.ID < expected.Memberships[j].Organisation.ID
 	})
 
 	sort.Slice(actual.Memberships, func(i, j int) bool {
-		return actual.Memberships[i].Title < actual.Memberships[j].Title
+		return actual.Memberships[i].Organisation.ID < actual.Memberships[j].Organisation.ID
 	})
 
 	assert.Equal(t, expected.Memberships, actual.Memberships, "Expected Memberships differ from actual \nExpected: %v \nActual: %v", expected.Memberships, actual.Memberships)
@@ -200,6 +223,34 @@ func readConceptAndCompare(t *testing.T, expected Person, uuid string) {
 		return actual.Labels[i] < actual.Labels[j]
 	})
 
+	for _, membership := range actual.Memberships {
+		sort.Slice(membership.Roles, func(i, j int) bool {
+			return membership.Roles[i].ID < membership.Roles[j].ID
+		})
+
+		sort.Slice(membership.Types, func(i, j int) bool {
+			return membership.Types[i] < membership.Types[j]
+		})
+
+		sort.Slice(membership.ChangeEvents, func(i, j int) bool {
+			return membership.ChangeEvents[i].StartedAt < membership.ChangeEvents[j].StartedAt
+		})
+	}
+
+	for _, membership := range expected.Memberships {
+		sort.Slice(membership.Roles, func(i, j int) bool {
+			return membership.Roles[i].ID < membership.Roles[j].ID
+		})
+
+		sort.Slice(membership.Types, func(i, j int) bool {
+			return membership.Types[i] < membership.Types[j]
+		})
+
+		sort.Slice(membership.ChangeEvents, func(i, j int) bool {
+			return membership.ChangeEvents[i].StartedAt < membership.ChangeEvents[j].StartedAt
+		})
+	}
+
 	assert.Equal(t, expected.Labels, actual.Labels, "Expected labels differ from actual \nExpected: %v \nActual: %v", expected.Labels, actual.Labels)
 	assert.Equal(t, expected.ID, actual.ID, "Expected labels differ from actual \nExpected: %v \nActual: %v", expected.ID, actual.ID)
 	assert.Equal(t, expected.APIURL, actual.APIURL, "Expected API URL differ from actual \nExpected: %v \nActual: %v", expected.APIURL, actual.APIURL)

people/fixtures/newModel/Membership-Hybrid-New-Old-cf79efd4-a0b6-4fdf-b9fd-d375a60c202e.json

@@ -0,0 +1,20 @@
+{
+  "prefUUID": "cf79efd4-a0b6-4fdf-b9fd-d375a60c202e",
+  "prefLabel": "Head of Old Organisations",
+  "type": "Membership",
+  "membershipRoles" : ["7d0738b1-0ea2-47cb-bb82-e86744b389f0"],
+  "organisationUUID": "638fc0c1-c4d9-4be4-b6d9-c97a057e7d1b",
+  "personUUID": "7ceeafe5-9f9a-4315-b3da-a5b4b69c013a",
+  "sourceRepresentations": [
+    {
+      "uuid": "cf79efd4-a0b6-4fdf-b9fd-d375a60c202e",
+      "prefLabel": "Head of Old Organisations",
+      "authority": "Smartlogic",
+      "authorityValue": "cf79efd4-a0b6-4fdf-b9fd-d375a60c202e",
+      "type": "Membership",
+      "membershipRoles" : ["7d0738b1-0ea2-47cb-bb82-e86744b389f0"],
+      "organisationUUID": "638fc0c1-c4d9-4be4-b6d9-c97a057e7d1b",
+      "personUUID": "7ceeafe5-9f9a-4315-b3da-a5b4b69c013a"
+    }
+  ]
+}

people/fixtures/newModel/MembershipRole-SmartyPants-7d0738b1-0ea2-47cb-bb82-e86744b389f0.json

@@ -7,7 +7,7 @@
       "uuid": "7d0738b1-0ea2-47cb-bb82-e86744b389f0",
       "prefLabel": "Smarty Pants",
       "type": "MembershipRole",
-      "authority": "SmartLogic",
+      "authority": "Smartlogic",
       "authorityValue": "7d0738b1-0ea2-47cb-bb82-e86744b389f0"
     }
   ]

people/fixtures/newModel/Organisation-RooneyRoosters-184cbe9b-b630-40d5-a5d0-99ecabd7fd86.json

@@ -16,7 +16,7 @@
       "uuid": "184cbe9b-b630-40d5-a5d0-99ecabd7fd86",
       "prefLabel": "Rooney Roosters",
       "type": "Organisation",
-      "authority": "SmartLogic",
+      "authority": "Smartlogic",
       "authorityValue": "184cbe9b-b630-40d5-a5d0-99ecabd7fd86",
       "emailAddress": "test@example.com",
       "twitterHandle": "@something",

people/fixtures/newModel/Person-Shirley-Rooney-7ceeafe5-9f9a-4315-b3da-a5b4b69c013a.json

@@ -22,7 +22,7 @@
     }, {
       "uuid": "7ceeafe5-9f9a-4315-b3da-a5b4b69c013a",
       "prefLabel": "Shirley Rooney",
-      "authority": "SmartLogic",
+      "authority": "Smartlogic",
       "authorityValue": "7ceeafe5-9f9a-4315-b3da-a5b4b69c013a",
       "type": "Person",
       "emailAddress": "test@example.com",

people/fixtures/oldModel/Membership-Shirley-Rooney-ef0921e4-c862-43ac-8936-f345b9fb131a.json

@@ -0,0 +1,17 @@
+{
+  "uuid": "ef0921e4-c862-43ac-8936-f345b9fb131a",
+  "prefLabel": "Smarty Pants Market Strategist",
+  "personUuid": "7ceeafe5-9f9a-4315-b3da-a5b4b69c013a",
+  "organisationUuid": "ac4be3c3-6dc1-4966-9cc5-ac824780f631",
+  "inceptionDate": "2008-01-16T00:00:00.000Z",
+  "alternativeIdentifiers" : {
+    "uuids" : ["ef0921e4-c862-43ac-8936-f345b9fb131a"],
+    "factsetIdentifier" : "463453543"
+  },
+  "membershipRoles": [
+    {
+      "roleUuid": "0ee8e7b7-bac9-4db1-b94b-5605ce1d2907",
+      "inceptionDate": "2008-01-16T00:00:00.000Z"
+    }
+  ]
+}

people/fixtures/oldModel/Organisation-Child_Of_A-ac4be3c3-6dc1-4966-9cc5-ac824780f631.json

@@ -1,5 +1,6 @@
 {
   "uuid": "ac4be3c3-6dc1-4966-9cc5-ac824780f631",
+  "prefLabel": "ABN AMRO Rothschild LLC",
   "properName": "ABN AMRO Rothschild LLC",
   "alternativeIdentifiers" : {
     "uuids" : ["ac4be3c3-6dc1-4966-9cc5-ac824780f631", "6a088943-cc59-4fbb-a7b9-8792942afb0f"],