Skip to content

Commit

Permalink
Display UPP identifiers in the output. Little bit of optimising query (…
Browse files Browse the repository at this point in the history 
…#20)

* Display UPP identifiers in the output.  Little bit of optimising query

* Better naming of model properties

* De-duping the concordance sorting code

* Changed name of concordance sort method
  • Loading branch information
David Smith committed Oct 11, 2017
1 parent dfa8d69 commit 6cffb26
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 57 deletions.
63 changes: 27 additions & 36 deletions concordances/cypher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,9 @@ func (pcw CypherDriver) CheckConnectivity() error {
}

type neoReadStruct struct {
UUID string `json:"UUID"`
PrefUUID string `json:"prefUUID"`
NewModelTypes []string `json:"newModelTypes"`
NewModelIdentifiers neoIdentifier `json:"newModelIdentifiers"`
OldModelTypes []string `json:"oldModelTypes"`
OldModelIdentifiers neoIdentifier `json:"oldModelIdentifiers"`
UUID string `json:"uuid"`
Types []string `json:"types"`
Identifier neoIdentifier `json:"identifier"`
}

type neoIdentifier struct {
Expand All @@ -52,15 +49,12 @@ func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Conc
Statement: `
MATCH (p:Concept)<-[:IDENTIFIES]-(i:UPPIdentifier)
WHERE i.value in {identifiers}
MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
WHERE NOT ids:UPPIdentifier
OPTIONAL MATCH (p:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
OPTIONAL MATCH (leafNode:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
OPTIONAL MATCH (leafNode:Concept)<-[:IDENTIFIES]-(leafId:Identifier)
WHERE NOT leafId:UPPIdentifier
RETURN canonical.prefUUID as prefUUID, p.uuid AS UUID, labels(p) AS oldModelTypes, labels(p) AS newModelTypes,
{labels:labels(ids), value:ids.value} AS oldModelIdentifiers,
{labels:labels(leafId), value:leafId.value} AS newModelIdentifiers
MATCH (p)-[:IDENTIFIES]-(ids:Identifier)
OPTIONAL MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
OPTIONAL MATCH (leafNode:Concept)-[:EQUIVALENT_TO]->(canonical)
OPTIONAL MATCH (leafNode)<-[:IDENTIFIES]-(leafId:Identifier)
WITH COALESCE(canonical.prefUUID, p.uuid) AS uuid, COALESCE(labels(canonical), labels(p)) AS types, COALESCE(leafId, ids) as nodeIds
RETURN DISTINCT uuid, types, {labels:labels(nodeIds), value:nodeIds.value} as identifier
`,
Parameters: neoism.Props{"identifiers": identifiers},
Result: &results,
Expand All @@ -87,21 +81,20 @@ func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Conc
func (pcw CypherDriver) ReadByAuthority(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
results := []neoReadStruct{}

authorityProperty := mapAuthorityToAuthorityProperty(authority)
if authorityProperty == "" {
authorityLabel := mapAuthorityToIdentifierLabel(authority)
if authorityLabel == "" {
return Concordances{}, false, nil
}

query := &neoism.CypherQuery{
Statement: `
MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
WHERE ids.value in {identifierValues} AND NOT ids:UPPIdentifier
OPTIONAL MATCH (p:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
RETURN canonical.prefUUID as prefUUID, p.uuid AS UUID, labels(p) AS oldModelTypes, labels(canonical) AS newModelTypes,
{labels:labels(ids), value:ids.value} AS oldModelIdentifiers,
{labels:labels(ids), value:ids.value} AS newModelIdentifiers`,
Statement: fmt.Sprintf(`
MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier:%s)
WHERE ids.value in {identifierValues}
OPTIONAL MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
RETURN COALESCE(canonical.prefUUID, p.uuid) AS uuid, COALESCE(labels(canonical), labels(p)) AS types, {labels:labels(ids), value:ids.value} as identifier`, authorityLabel),

Parameters: neoism.Props{
"identifierValues": identifierValues,
"authority": authorityProperty,
},
Result: &results,
}
Expand Down Expand Up @@ -146,17 +139,11 @@ func neoReadStructToConcordances(neo []neoReadStruct, env string) (concordances
var con = Concordance{}
var concept = Concept{}

if neoCon.PrefUUID != "" {
log.Debugf("New concept model with prefUUID: %v", neoCon.PrefUUID)
concept.ID = mapper.IDURL(neoCon.PrefUUID)
concept.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.NewModelTypes, env)
con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NewModelIdentifiers.Labels), IdentifierValue: neoCon.NewModelIdentifiers.Value}
} else {
log.Debugf("Old concept model with UUID: %v", neoCon.UUID)
concept.ID = mapper.IDURL(neoCon.UUID)
concept.APIURL = mapper.APIURL(neoCon.UUID, neoCon.OldModelTypes, env)
con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.OldModelIdentifiers.Labels), IdentifierValue: neoCon.OldModelIdentifiers.Value}
}
log.Debug(neoCon)

concept.ID = mapper.IDURL(neoCon.UUID)
concept.APIURL = mapper.APIURL(neoCon.UUID, neoCon.Types, env)
con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.Identifier.Labels), IdentifierValue: neoCon.Identifier.Value}

con.Concept = concept
concordances.Concordance = append(concordances.Concordance, con)
Expand Down Expand Up @@ -208,6 +195,10 @@ func mapAuthorityToAuthorityProperty(authority string) string {

func mapAuthorityToIdentifierLabel(authority string) (label string) {
switch authority {
case UP_AUTHORITY:
return UP_ID_NODE_LABEL
case SL_AUTHORITY:
return SL_ID_NODE_LABEL
case TME_AUTHORITY:
return TME_ID_NODE_LABEL
case FS_AUTHORITY:
Expand Down
91 changes: 70 additions & 21 deletions concordances/cypher_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,14 @@ import (

"reflect"

"sort"

"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
"github.com/Financial-Times/concepts-rw-neo4j/concepts"
"github.com/Financial-Times/neo-utils-go/neoutils"
"github.com/Financial-Times/organisations-rw-neo4j/organisations"
"github.com/jmcvetta/neoism"
"github.com/stretchr/testify/assert"
"sort"
)

var concordedBrandSmartlogic = Concordance{
Expand All @@ -26,6 +27,15 @@ var concordedBrandSmartlogic = Concordance{
IdentifierValue: "b20801ac-5a76-43cf-b816-8c3b2f7133ad"},
}

var concordedBrandSmartlogicUPP = Concordance{
Concept{
ID: "http://api.ft.com/things/b20801ac-5a76-43cf-b816-8c3b2f7133ad",
APIURL: "http://api.ft.com/brands/b20801ac-5a76-43cf-b816-8c3b2f7133ad"},
Identifier{
Authority: "http://api.ft.com/system/UPP",
IdentifierValue: "b20801ac-5a76-43cf-b816-8c3b2f7133ad"},
}

var concordedBrandTME = Concordance{
Concept{
ID: "http://api.ft.com/things/b20801ac-5a76-43cf-b816-8c3b2f7133ad",
Expand All @@ -35,6 +45,15 @@ var concordedBrandTME = Concordance{
IdentifierValue: "VGhlIFJvbWFu-QnJhbmRz"},
}

var concordedBrandTMEUPP = Concordance{
Concept{
ID: "http://api.ft.com/things/b20801ac-5a76-43cf-b816-8c3b2f7133ad",
APIURL: "http://api.ft.com/brands/b20801ac-5a76-43cf-b816-8c3b2f7133ad"},
Identifier{
Authority: "http://api.ft.com/system/UPP",
IdentifierValue: "70f4732b-7f7d-30a1-9c29-0cceec23760e"},
}

var mainOrganisationLEI = Concordance{
Concept{
ID: "http://api.ft.com/things/21a5cc0-d326-4e62-b84a-d840c2209fee",
Expand All @@ -53,6 +72,15 @@ var childOrganisationFactset = Concordance{
IdentifierValue: "003JLG-E"},
}

var childOrganisationFactsetUPP = Concordance{
Concept{
ID: "http://api.ft.com/things/f21a5cc0-d326-4e62-b84a-d840c2209fee",
APIURL: "http://api.ft.com/organisations/f21a5cc0-d326-4e62-b84a-d840c2209fee"},
Identifier{
Authority: "http://api.ft.com/system/UPP",
IdentifierValue: "f21a5cc0-d326-4e62-b84a-d840c2209fee"},
}

var childOrganisationLEI = Concordance{
Concept{
ID: "http://api.ft.com/things/f21a5cc0-d326-4e62-b84a-d840c2209fee",
Expand All @@ -71,6 +99,15 @@ var mainOrganisationTME = Concordance{
IdentifierValue: "TnN0ZWluX09OX0ZvcnR1bmVDb21wYW55X05XUw==-T04="},
}

var mainOrganisationTMEUPP = Concordance{
Concept{
ID: "http://api.ft.com/things/3e844449-b27f-40d4-b696-2ce9b6137133",
APIURL: "http://api.ft.com/organisations/3e844449-b27f-40d4-b696-2ce9b6137133"},
Identifier{
Authority: "http://api.ft.com/system/UPP",
IdentifierValue: "TnN0ZWluX09OX0ZvcnR1bmVDb21wYW55X05XUw==-T04="},
}

var unconcordedBrandTME = Concordance{
Concept{
ID: "http://api.ft.com/things/ad56856a-7d38-48e2-a131-7d104f17e8f6",
Expand All @@ -80,6 +117,15 @@ var unconcordedBrandTME = Concordance{
IdentifierValue: "UGFydHkgcGVvcGxl-QnJhbmRz"},
}

var unconcordedBrandTMEUPP = Concordance{
Concept{
ID: "http://api.ft.com/things/ad56856a-7d38-48e2-a131-7d104f17e8f6",
APIURL: "http://api.ft.com/brands/ad56856a-7d38-48e2-a131-7d104f17e8f6"},
Identifier{
Authority: "http://api.ft.com/system/UPP",
IdentifierValue: "ad56856a-7d38-48e2-a131-7d104f17e8f6"},
}

func TestNeoReadByConceptID_NewModel_Unconcorded(t *testing.T) {
assert := assert.New(t)
db := getDatabaseConnection(t, assert)
Expand All @@ -94,9 +140,9 @@ func TestNeoReadByConceptID_NewModel_Unconcorded(t *testing.T) {
conc, found, err := undertest.ReadByConceptID([]string{"ad56856a-7d38-48e2-a131-7d104f17e8f6"})
assert.NoError(err)
assert.True(found)
assert.Equal(1, len(conc.Concordance))
assert.Equal(2, len(conc.Concordance))

readConceptAndCompare(t, Concordances{[]Concordance{unconcordedBrandTME}}, conc, "TestNeoReadByConceptID_NewModel_Unconcorded")
readConceptAndCompare(t, Concordances{[]Concordance{unconcordedBrandTME, unconcordedBrandTMEUPP}}, conc, "TestNeoReadByConceptID_NewModel_Unconcorded")
}

func TestNeoReadByConceptID_NewModel_Concorded(t *testing.T) {
Expand All @@ -112,9 +158,9 @@ func TestNeoReadByConceptID_NewModel_Concorded(t *testing.T) {
conc, found, err := undertest.ReadByConceptID([]string{"b20801ac-5a76-43cf-b816-8c3b2f7133ad"})
assert.NoError(err)
assert.True(found)
assert.Equal(2, len(conc.Concordance))
assert.Equal(4, len(conc.Concordance))

readConceptAndCompare(t, Concordances{[]Concordance{concordedBrandSmartlogic, concordedBrandTME}}, conc, "TestNeoReadByConceptID_NewModel_Concorded")
readConceptAndCompare(t, Concordances{[]Concordance{concordedBrandSmartlogic, concordedBrandSmartlogicUPP, concordedBrandTME, concordedBrandTMEUPP}}, conc, "TestNeoReadByConceptID_NewModel_Concorded")
}

func TestNeoReadByConceptID_NewModel_And_OldModel(t *testing.T) {
Expand All @@ -134,9 +180,9 @@ func TestNeoReadByConceptID_NewModel_And_OldModel(t *testing.T) {
conc, found, err := undertest.ReadByConceptID([]string{"b20801ac-5a76-43cf-b816-8c3b2f7133ad", "f21a5cc0-d326-4e62-b84a-d840c2209fee"})
assert.NoError(err)
assert.True(found)
assert.Equal(4, len(conc.Concordance))
assert.Equal(7, len(conc.Concordance))

sliceConcordances := []Concordance{childOrganisationFactset, childOrganisationLEI, concordedBrandTME, concordedBrandSmartlogic}
sliceConcordances := []Concordance{childOrganisationFactset, childOrganisationFactsetUPP, childOrganisationLEI, concordedBrandTME, concordedBrandTMEUPP, concordedBrandSmartlogic, concordedBrandSmartlogicUPP}
readConceptAndCompare(t, Concordances{sliceConcordances}, conc, "TestNeoReadByConceptID_NewModel_And_OldModel")
}

Expand Down Expand Up @@ -215,7 +261,7 @@ func TestNeoReadByConceptIDToConcordancesMandatoryFields(t *testing.T) {
assert.True(found)
assert.NotEmpty(cs.Concordance)

readConceptAndCompare(t, Concordances{[]Concordance{childOrganisationFactset, childOrganisationLEI}}, cs, "TestNeoReadByConceptIDToConcordancesMandatoryFields")
readConceptAndCompare(t, Concordances{[]Concordance{childOrganisationFactset, childOrganisationFactsetUPP, childOrganisationLEI}}, cs, "TestNeoReadByConceptIDToConcordancesMandatoryFields")
}

func TestNeoReadByAuthorityToConcordancesMandatoryFields(t *testing.T) {
Expand Down Expand Up @@ -276,9 +322,9 @@ func TestNeoReadByConceptIdReturnMultipleConcordancesForMultipleIdentifiers(t *t
assert.NoError(err)
assert.True(found)
assert.NotEmpty(cs.Concordance)
assert.Equal(len(cs.Concordance), 2)
assert.Equal(3, len(cs.Concordance))

readConceptAndCompare(t, Concordances{[]Concordance{childOrganisationFactset, childOrganisationLEI}}, cs, "TestNeoReadByConceptIdReturnMultipleConcordancesForMultipleIdentifiers")
readConceptAndCompare(t, Concordances{[]Concordance{childOrganisationFactset, childOrganisationFactsetUPP, childOrganisationLEI}}, cs, "TestNeoReadByConceptIdReturnMultipleConcordancesForMultipleIdentifiers")
}

func TestNeoReadByAuthorityEmptyConcordancesWhenUnsupportedAuthority(t *testing.T) {
Expand All @@ -300,20 +346,23 @@ func TestNeoReadByAuthorityEmptyConcordancesWhenUnsupportedAuthority(t *testing.
}

func readConceptAndCompare(t *testing.T, expected Concordances, actual Concordances, testName string) {
sort.Slice(expected.Concordance, func(i, j int) bool {
return expected.Concordance[i].Concept.ID < expected.Concordance[j].Concept.ID
})
sort.Slice(expected.Concordance, func(i, j int) bool {
return expected.Concordance[i].Identifier.IdentifierValue < expected.Concordance[j].Identifier.IdentifierValue
})

sort.Slice(actual.Concordance, func(i, j int) bool {
return actual.Concordance[i].Concept.ID < actual.Concordance[j].Concept.ID
sortConcordances(expected.Concordance)
sortConcordances(actual.Concordance)

assert.True(t, reflect.DeepEqual(expected, actual), fmt.Sprintf("Actual aggregated concept differs from expected: Test: %v \n Expected: %v \n Actual: %v", testName, expected, actual))
}

func sortConcordances(concordanceList []Concordance) {
sort.SliceStable(concordanceList, func(i, j int) bool {
return concordanceList[i].Concept.ID < concordanceList[j].Concept.ID
})
sort.Slice(actual.Concordance, func(i, j int) bool {
return actual.Concordance[i].Identifier.IdentifierValue < actual.Concordance[j].Identifier.IdentifierValue
sort.SliceStable(concordanceList, func(i, j int) bool {
return concordanceList[i].Identifier.Authority < concordanceList[j].Identifier.Authority
})
sort.SliceStable(concordanceList, func(i, j int) bool {
return concordanceList[i].Identifier.IdentifierValue < concordanceList[j].Identifier.IdentifierValue
})
assert.True(t, reflect.DeepEqual(expected, actual), fmt.Sprintf("Actual aggregated concept differs from expected: Test: %v \n Expected: %v \n Actual: %v", testName, expected, actual))
}

func getDatabaseConnection(t *testing.T, assert *assert.Assertions) neoutils.NeoConnection {
Expand Down

0 comments on commit 6cffb26

Please sign in to comment.