Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Display UPP identifiers in the output. Little bit of optimising query #20

Merged
4 commits merged into from
Oct 11, 2017
Merged

Display UPP identifiers in the output. Little bit of optimising query #20

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
75d45f3
Display UPP identifiers in the output. Little bit of optimising query
Oct 9, 2017
83dc19e
Better naming of model properties
Oct 10, 2017
e612a10
De-duping the concordance sorting code
Oct 10, 2017
c9cb94a
Changed name of concordance sort method
Oct 10, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 27 additions & 36 deletions concordances/cypher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,9 @@ func (pcw CypherDriver) CheckConnectivity() error {
}

type neoReadStruct struct {
UUID string `json:"UUID"`
PrefUUID string `json:"prefUUID"`
NewModelTypes []string `json:"newModelTypes"`
NewModelIdentifiers neoIdentifier `json:"newModelIdentifiers"`
OldModelTypes []string `json:"oldModelTypes"`
OldModelIdentifiers neoIdentifier `json:"oldModelIdentifiers"`
UUID string `json:"UUID"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be "uuid" to be idiomatic json?

Types []string `json:"types"`
Identifiers neoIdentifier `json:"identifiers"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Logically identifiers would represent multiple ids? Not just one neoIdentifier? If neoIdentifier as a struct represents more than one id, that should be called neoIdentifiers instead

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, agreed. Hangover from the previous model.

}

type neoIdentifier struct {
Expand All @@ -52,15 +49,12 @@ func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Conc
Statement: `
MATCH (p:Concept)<-[:IDENTIFIES]-(i:UPPIdentifier)
WHERE i.value in {identifiers}
MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
WHERE NOT ids:UPPIdentifier
OPTIONAL MATCH (p:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
OPTIONAL MATCH (leafNode:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
OPTIONAL MATCH (leafNode:Concept)<-[:IDENTIFIES]-(leafId:Identifier)
WHERE NOT leafId:UPPIdentifier
RETURN canonical.prefUUID as prefUUID, p.uuid AS UUID, labels(p) AS oldModelTypes, labels(p) AS newModelTypes,
{labels:labels(ids), value:ids.value} AS oldModelIdentifiers,
{labels:labels(leafId), value:leafId.value} AS newModelIdentifiers
MATCH (p)-[:IDENTIFIES]-(ids:Identifier)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this no longer required to be a Concept?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The Concept constraint is defined in the first line of the query, so it's not needed again.

OPTIONAL MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
OPTIONAL MATCH (leafNode:Concept)-[:EQUIVALENT_TO]->(canonical)
OPTIONAL MATCH (leafNode)<-[:IDENTIFIES]-(leafId:Identifier)
WITH COALESCE(canonical.prefUUID, p.uuid) AS UUID, COALESCE(labels(canonical), labels(p)) AS types, COALESCE(leafId, ids) as nodeIds
RETURN DISTINCT UUID, types, {labels:labels(nodeIds), value:nodeIds.value} as identifiers
`,
Parameters: neoism.Props{"identifiers": identifiers},
Result: &results,
Expand All @@ -87,21 +81,20 @@ func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Conc
func (pcw CypherDriver) ReadByAuthority(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
results := []neoReadStruct{}

authorityProperty := mapAuthorityToAuthorityProperty(authority)
if authorityProperty == "" {
authorityLabel := mapAuthorityToIdentifierLabel(authority)
if authorityLabel == "" {
return Concordances{}, false, nil
}

query := &neoism.CypherQuery{
Statement: `
MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
WHERE ids.value in {identifierValues} AND NOT ids:UPPIdentifier
OPTIONAL MATCH (p:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
RETURN canonical.prefUUID as prefUUID, p.uuid AS UUID, labels(p) AS oldModelTypes, labels(canonical) AS newModelTypes,
{labels:labels(ids), value:ids.value} AS oldModelIdentifiers,
{labels:labels(ids), value:ids.value} AS newModelIdentifiers`,
Statement: fmt.Sprintf(`
MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier:%s)
WHERE ids.value in {identifierValues}
OPTIONAL MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
RETURN COALESCE(canonical.prefUUID, p.uuid) AS UUID, COALESCE(labels(canonical), labels(p)) AS types, {labels:labels(ids), value:ids.value} as identifiers`, authorityLabel),

Parameters: neoism.Props{
"identifierValues": identifierValues,
"authority": authorityProperty,
},
Result: &results,
}
Expand Down Expand Up @@ -146,17 +139,11 @@ func neoReadStructToConcordances(neo []neoReadStruct, env string) (concordances
var con = Concordance{}
var concept = Concept{}

if neoCon.PrefUUID != "" {
log.Debugf("New concept model with prefUUID: %v", neoCon.PrefUUID)
concept.ID = mapper.IDURL(neoCon.PrefUUID)
concept.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.NewModelTypes, env)
con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NewModelIdentifiers.Labels), IdentifierValue: neoCon.NewModelIdentifiers.Value}
} else {
log.Debugf("Old concept model with UUID: %v", neoCon.UUID)
concept.ID = mapper.IDURL(neoCon.UUID)
concept.APIURL = mapper.APIURL(neoCon.UUID, neoCon.OldModelTypes, env)
con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.OldModelIdentifiers.Labels), IdentifierValue: neoCon.OldModelIdentifiers.Value}
}
log.Debug(neoCon)

concept.ID = mapper.IDURL(neoCon.UUID)
concept.APIURL = mapper.APIURL(neoCon.UUID, neoCon.Types, env)
con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.Identifiers.Labels), IdentifierValue: neoCon.Identifiers.Value}

con.Concept = concept
concordances.Concordance = append(concordances.Concordance, con)
Expand Down Expand Up @@ -208,6 +195,10 @@ func mapAuthorityToAuthorityProperty(authority string) string {

func mapAuthorityToIdentifierLabel(authority string) (label string) {
switch authority {
case UP_AUTHORITY:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should probably be a map

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's no performance improvement by changing it to a map, and I think this is slightly more readable than defining the map and then doing the lookup.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Disagree - you're doing a key value lookup, which is exactly what maps exist for. Adding new mappings will require 2 lines for the constants, plus an addition to this switch statement, which is inefficient

return UP_ID_NODE_LABEL
case SL_AUTHORITY:
return SL_ID_NODE_LABEL
case TME_AUTHORITY:
return TME_ID_NODE_LABEL
case FS_AUTHORITY:
Expand Down
80 changes: 66 additions & 14 deletions concordances/cypher_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,14 @@ import (

"reflect"

"sort"

"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
"github.com/Financial-Times/concepts-rw-neo4j/concepts"
"github.com/Financial-Times/neo-utils-go/neoutils"
"github.com/Financial-Times/organisations-rw-neo4j/organisations"
"github.com/jmcvetta/neoism"
"github.com/stretchr/testify/assert"
"sort"
)

var concordedBrandSmartlogic = Concordance{
Expand All @@ -26,6 +27,15 @@ var concordedBrandSmartlogic = Concordance{
IdentifierValue: "b20801ac-5a76-43cf-b816-8c3b2f7133ad"},
}

var concordedBrandSmartlogicUPP = Concordance{
Concept{
ID: "http://api.ft.com/things/b20801ac-5a76-43cf-b816-8c3b2f7133ad",
APIURL: "http://api.ft.com/brands/b20801ac-5a76-43cf-b816-8c3b2f7133ad"},
Identifier{
Authority: "http://api.ft.com/system/UPP",
IdentifierValue: "b20801ac-5a76-43cf-b816-8c3b2f7133ad"},
}

var concordedBrandTME = Concordance{
Concept{
ID: "http://api.ft.com/things/b20801ac-5a76-43cf-b816-8c3b2f7133ad",
Expand All @@ -35,6 +45,15 @@ var concordedBrandTME = Concordance{
IdentifierValue: "VGhlIFJvbWFu-QnJhbmRz"},
}

var concordedBrandTMEUPP = Concordance{
Concept{
ID: "http://api.ft.com/things/b20801ac-5a76-43cf-b816-8c3b2f7133ad",
APIURL: "http://api.ft.com/brands/b20801ac-5a76-43cf-b816-8c3b2f7133ad"},
Identifier{
Authority: "http://api.ft.com/system/UPP",
IdentifierValue: "70f4732b-7f7d-30a1-9c29-0cceec23760e"},
}

var mainOrganisationLEI = Concordance{
Concept{
ID: "http://api.ft.com/things/21a5cc0-d326-4e62-b84a-d840c2209fee",
Expand All @@ -53,6 +72,15 @@ var childOrganisationFactset = Concordance{
IdentifierValue: "003JLG-E"},
}

var childOrganisationFactsetUPP = Concordance{
Concept{
ID: "http://api.ft.com/things/f21a5cc0-d326-4e62-b84a-d840c2209fee",
APIURL: "http://api.ft.com/organisations/f21a5cc0-d326-4e62-b84a-d840c2209fee"},
Identifier{
Authority: "http://api.ft.com/system/UPP",
IdentifierValue: "f21a5cc0-d326-4e62-b84a-d840c2209fee"},
}

var childOrganisationLEI = Concordance{
Concept{
ID: "http://api.ft.com/things/f21a5cc0-d326-4e62-b84a-d840c2209fee",
Expand All @@ -71,6 +99,15 @@ var mainOrganisationTME = Concordance{
IdentifierValue: "TnN0ZWluX09OX0ZvcnR1bmVDb21wYW55X05XUw==-T04="},
}

var mainOrganisationTMEUPP = Concordance{
Concept{
ID: "http://api.ft.com/things/3e844449-b27f-40d4-b696-2ce9b6137133",
APIURL: "http://api.ft.com/organisations/3e844449-b27f-40d4-b696-2ce9b6137133"},
Identifier{
Authority: "http://api.ft.com/system/UPP",
IdentifierValue: "TnN0ZWluX09OX0ZvcnR1bmVDb21wYW55X05XUw==-T04="},
}

var unconcordedBrandTME = Concordance{
Concept{
ID: "http://api.ft.com/things/ad56856a-7d38-48e2-a131-7d104f17e8f6",
Expand All @@ -80,6 +117,15 @@ var unconcordedBrandTME = Concordance{
IdentifierValue: "UGFydHkgcGVvcGxl-QnJhbmRz"},
}

var unconcordedBrandTMEUPP = Concordance{
Concept{
ID: "http://api.ft.com/things/ad56856a-7d38-48e2-a131-7d104f17e8f6",
APIURL: "http://api.ft.com/brands/ad56856a-7d38-48e2-a131-7d104f17e8f6"},
Identifier{
Authority: "http://api.ft.com/system/UPP",
IdentifierValue: "ad56856a-7d38-48e2-a131-7d104f17e8f6"},
}

func TestNeoReadByConceptID_NewModel_Unconcorded(t *testing.T) {
assert := assert.New(t)
db := getDatabaseConnection(t, assert)
Expand All @@ -94,9 +140,9 @@ func TestNeoReadByConceptID_NewModel_Unconcorded(t *testing.T) {
conc, found, err := undertest.ReadByConceptID([]string{"ad56856a-7d38-48e2-a131-7d104f17e8f6"})
assert.NoError(err)
assert.True(found)
assert.Equal(1, len(conc.Concordance))
assert.Equal(2, len(conc.Concordance))

readConceptAndCompare(t, Concordances{[]Concordance{unconcordedBrandTME}}, conc, "TestNeoReadByConceptID_NewModel_Unconcorded")
readConceptAndCompare(t, Concordances{[]Concordance{unconcordedBrandTME, unconcordedBrandTMEUPP}}, conc, "TestNeoReadByConceptID_NewModel_Unconcorded")
}

func TestNeoReadByConceptID_NewModel_Concorded(t *testing.T) {
Expand All @@ -112,9 +158,9 @@ func TestNeoReadByConceptID_NewModel_Concorded(t *testing.T) {
conc, found, err := undertest.ReadByConceptID([]string{"b20801ac-5a76-43cf-b816-8c3b2f7133ad"})
assert.NoError(err)
assert.True(found)
assert.Equal(2, len(conc.Concordance))
assert.Equal(4, len(conc.Concordance))

readConceptAndCompare(t, Concordances{[]Concordance{concordedBrandSmartlogic, concordedBrandTME}}, conc, "TestNeoReadByConceptID_NewModel_Concorded")
readConceptAndCompare(t, Concordances{[]Concordance{concordedBrandSmartlogic, concordedBrandSmartlogicUPP, concordedBrandTME, concordedBrandTMEUPP}}, conc, "TestNeoReadByConceptID_NewModel_Concorded")
}

func TestNeoReadByConceptID_NewModel_And_OldModel(t *testing.T) {
Expand All @@ -134,9 +180,9 @@ func TestNeoReadByConceptID_NewModel_And_OldModel(t *testing.T) {
conc, found, err := undertest.ReadByConceptID([]string{"b20801ac-5a76-43cf-b816-8c3b2f7133ad", "f21a5cc0-d326-4e62-b84a-d840c2209fee"})
assert.NoError(err)
assert.True(found)
assert.Equal(4, len(conc.Concordance))
assert.Equal(7, len(conc.Concordance))

sliceConcordances := []Concordance{childOrganisationFactset, childOrganisationLEI, concordedBrandTME, concordedBrandSmartlogic}
sliceConcordances := []Concordance{childOrganisationFactset, childOrganisationFactsetUPP, childOrganisationLEI, concordedBrandTME, concordedBrandTMEUPP, concordedBrandSmartlogic, concordedBrandSmartlogicUPP}
readConceptAndCompare(t, Concordances{sliceConcordances}, conc, "TestNeoReadByConceptID_NewModel_And_OldModel")
}

Expand Down Expand Up @@ -215,7 +261,7 @@ func TestNeoReadByConceptIDToConcordancesMandatoryFields(t *testing.T) {
assert.True(found)
assert.NotEmpty(cs.Concordance)

readConceptAndCompare(t, Concordances{[]Concordance{childOrganisationFactset, childOrganisationLEI}}, cs, "TestNeoReadByConceptIDToConcordancesMandatoryFields")
readConceptAndCompare(t, Concordances{[]Concordance{childOrganisationFactset, childOrganisationFactsetUPP, childOrganisationLEI}}, cs, "TestNeoReadByConceptIDToConcordancesMandatoryFields")
}

func TestNeoReadByAuthorityToConcordancesMandatoryFields(t *testing.T) {
Expand Down Expand Up @@ -276,9 +322,9 @@ func TestNeoReadByConceptIdReturnMultipleConcordancesForMultipleIdentifiers(t *t
assert.NoError(err)
assert.True(found)
assert.NotEmpty(cs.Concordance)
assert.Equal(len(cs.Concordance), 2)
assert.Equal(3, len(cs.Concordance))

readConceptAndCompare(t, Concordances{[]Concordance{childOrganisationFactset, childOrganisationLEI}}, cs, "TestNeoReadByConceptIdReturnMultipleConcordancesForMultipleIdentifiers")
readConceptAndCompare(t, Concordances{[]Concordance{childOrganisationFactset, childOrganisationFactsetUPP, childOrganisationLEI}}, cs, "TestNeoReadByConceptIdReturnMultipleConcordancesForMultipleIdentifiers")
}

func TestNeoReadByAuthorityEmptyConcordancesWhenUnsupportedAuthority(t *testing.T) {
Expand All @@ -300,17 +346,23 @@ func TestNeoReadByAuthorityEmptyConcordancesWhenUnsupportedAuthority(t *testing.
}

func readConceptAndCompare(t *testing.T, expected Concordances, actual Concordances, testName string) {
sort.Slice(expected.Concordance, func(i, j int) bool {
sort.SliceStable(expected.Concordance, func(i, j int) bool {
return expected.Concordance[i].Concept.ID < expected.Concordance[j].Concept.ID
})
sort.Slice(expected.Concordance, func(i, j int) bool {
sort.SliceStable(expected.Concordance, func(i, j int) bool {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure I like this duplication for expected and actual concordance sorting, but can't offer an alternative (that doesn't involve implementing sort.Interface on Concordances in 3 diff ways)

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've extracted it so at least it's not repeated for both slices.

return expected.Concordance[i].Identifier.Authority < expected.Concordance[j].Identifier.Authority
})
sort.SliceStable(expected.Concordance, func(i, j int) bool {
return expected.Concordance[i].Identifier.IdentifierValue < expected.Concordance[j].Identifier.IdentifierValue
})

sort.Slice(actual.Concordance, func(i, j int) bool {
sort.SliceStable(actual.Concordance, func(i, j int) bool {
return actual.Concordance[i].Concept.ID < actual.Concordance[j].Concept.ID
})
sort.Slice(actual.Concordance, func(i, j int) bool {
sort.SliceStable(actual.Concordance, func(i, j int) bool {
return actual.Concordance[i].Identifier.Authority < actual.Concordance[j].Identifier.Authority
})
sort.SliceStable(actual.Concordance, func(i, j int) bool {
return actual.Concordance[i].Identifier.IdentifierValue < actual.Concordance[j].Identifier.IdentifierValue
})
assert.True(t, reflect.DeepEqual(expected, actual), fmt.Sprintf("Actual aggregated concept differs from expected: Test: %v \n Expected: %v \n Actual: %v", testName, expected, actual))
Expand Down