Skip to content

Commit

Permalink
predicate handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@peterschubert
peterschubert committed May 2, 2017
1 parent d176846 commit 5b56ffb
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 38 deletions.
8 changes: 0 additions & 8 deletions content_indexer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,10 @@ package main
import (
"encoding/json"
health "github.com/Financial-Times/go-fthealth/v1_1"
"github.com/Financial-Times/http-handlers-go/httphandlers"
"github.com/Financial-Times/message-queue-gonsumer/consumer"
status "github.com/Financial-Times/service-status-go/httphandlers"
log "github.com/Sirupsen/logrus"
"github.com/gorilla/mux"
"github.com/kr/pretty"
"github.com/rcrowley/go-metrics"
"net"
"net/http"
"os"
Expand Down Expand Up @@ -80,9 +77,6 @@ func (indexer *contentIndexer) start(appSystemCode string, appName string, index

func (indexer *contentIndexer) serveAdminEndpoints(appSystemCode string, appName string, port string, queueConfig consumer.QueueConfig) {
healthService := newHealthService(indexer.esServiceInstance, queueConfig.Topic, queueConfig.Addrs[0])
var monitoringRouter http.Handler = mux.NewRouter()
monitoringRouter = httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), monitoringRouter)
monitoringRouter = httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry, monitoringRouter)

serveMux := http.NewServeMux()

Expand All @@ -92,8 +86,6 @@ func (indexer *contentIndexer) serveAdminEndpoints(appSystemCode string, appName
serveMux.HandleFunc(status.GTGPath, status.NewGoodToGoHandler(healthService.gtgCheck))
serveMux.HandleFunc(status.BuildInfoPath, status.BuildInfoHandler)

serveMux.Handle("/", monitoringRouter)

if err := http.ListenAndServe(":"+port, serveMux); err != nil {
log.Fatalf("Unable to start: %v", err)
}
Expand Down
5 changes: 2 additions & 3 deletions content_indexer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,10 @@ func (hook *logHook) Levels() []logrus.Level {
func (hook *logHook) LastEntry() (l *logrus.Entry) {
hook.Lock()
defer hook.Unlock()
if i := len(hook.Entries) - 1; i < 0 {
return nil
} else {
if i := len(hook.Entries) - 1; i >= 0 {
return hook.Entries[i]
}
return nil
}

// Reset removes all Entries from this test hook.
Expand Down
88 changes: 61 additions & 27 deletions model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,67 +256,93 @@ func convertToESContentModel(enrichedContent enrichedContentModel, contentType s
esModel.PublishReference = enrichedContent.Content.PublishReference

for _, annotation := range enrichedContent.Metadata {
tmeIDs := []string{annotation.Thing.ID}
fallbackID := annotation.Thing.ID
tmeIDs := []string{fallbackID}
if len(annotation.Thing.TmeIDs) != 0 {
tmeIDs = append(tmeIDs, annotation.Thing.TmeIDs...)
} else {
log.Warnf("Indexing content with uuid %s - TME id missing for concept with id %s, using thing id instead",
&(enrichedContent.Content.UUID), annotation.Thing.ID)
&(enrichedContent.Content.UUID), fallbackID)
}
for _, taxonomy := range annotation.Thing.Types {
switch taxonomy {
case "http://www.ft.com/ontology/organisation/Organisation":
esModel.CmrOrgnames = append(esModel.CmrOrgnames, annotation.Thing.PrefLabel)
esModel.CmrOrgnamesIds = append(esModel.CmrOrgnamesIds, getCmrID(tmeOrganisations, tmeIDs))
esModel.CmrOrgnames = appendIfNotExists(esModel.CmrOrgnames, annotation.Thing.PrefLabel)
esModel.CmrOrgnamesIds = appendIfNotExists(esModel.CmrOrgnamesIds, getCmrID(tmeOrganisations, tmeIDs))
if annotation.Thing.Predicate == about {
esModel.CmrPrimarytheme = new(string)
*esModel.CmrPrimarytheme = annotation.Thing.PrefLabel
esModel.CmrPrimarythemeID = new(string)
*esModel.CmrPrimarythemeID = getCmrID(tmeOrganisations, tmeIDs)
}
case "http://www.ft.com/ontology/person/Person":
cmrID := getCmrID(tmePeople, tmeIDs)
if cmrID != annotation.Thing.ID {
esModel.CmrPeople = append(esModel.CmrPeople, annotation.Thing.PrefLabel)
esModel.CmrPeopleIds = append(esModel.CmrPeopleIds, cmrID)
authorCmrID := getCmrID(tmeAuthors, tmeIDs)
// if it's only author, skip adding to people
if cmrID != fallbackID || authorCmrID == fallbackID {
esModel.CmrPeople = appendIfNotExists(esModel.CmrPeople, annotation.Thing.PrefLabel)
esModel.CmrPeopleIds = appendIfNotExists(esModel.CmrPeopleIds, cmrID)
}
if annotation.Thing.Predicate == hasAuthor {
cmrID := getCmrID(tmeAuthors, tmeIDs)
if cmrID != annotation.Thing.ID {
esModel.CmrAuthors = append(esModel.CmrAuthors, annotation.Thing.PrefLabel)
esModel.CmrAuthorsIds = append(esModel.CmrAuthorsIds, cmrID)
if authorCmrID != fallbackID {
esModel.CmrAuthors = appendIfNotExists(esModel.CmrAuthors, annotation.Thing.PrefLabel)
esModel.CmrAuthorsIds = appendIfNotExists(esModel.CmrAuthorsIds, authorCmrID)
}
}
if annotation.Thing.Predicate == about {
esModel.CmrPrimarytheme = new(string)
*esModel.CmrPrimarytheme = annotation.Thing.PrefLabel
esModel.CmrPrimarythemeID = new(string)
*esModel.CmrPrimarythemeID = getCmrID(tmePeople, tmeIDs)
}
case "http://www.ft.com/ontology/company/Company":
esModel.CmrCompanynames = append(esModel.CmrCompanynames, annotation.Thing.PrefLabel)
esModel.CmrCompanynamesIds = append(esModel.CmrCompanynamesIds, getCmrID(tmeOrganisations, tmeIDs))
esModel.CmrCompanynames = appendIfNotExists(esModel.CmrCompanynames, annotation.Thing.PrefLabel)
esModel.CmrCompanynamesIds = appendIfNotExists(esModel.CmrCompanynamesIds, getCmrID(tmeOrganisations, tmeIDs))
case "http://www.ft.com/ontology/product/Brand":
esModel.CmrBrands = append(esModel.CmrBrands, annotation.Thing.PrefLabel)
esModel.CmrBrandsIds = append(esModel.CmrBrandsIds, getCmrID(tmeBrands, tmeIDs))
esModel.CmrBrands = appendIfNotExists(esModel.CmrBrands, annotation.Thing.PrefLabel)
esModel.CmrBrandsIds = appendIfNotExists(esModel.CmrBrandsIds, getCmrID(tmeBrands, tmeIDs))
case "http://www.ft.com/ontology/Subject":
esModel.CmrSubjects = append(esModel.CmrSubjects, annotation.Thing.PrefLabel)
esModel.CmrSubjectsIds = append(esModel.CmrSubjectsIds, getCmrID(tmeSubjects, tmeIDs))
esModel.CmrSubjects = appendIfNotExists(esModel.CmrSubjects, annotation.Thing.PrefLabel)
esModel.CmrSubjectsIds = appendIfNotExists(esModel.CmrSubjectsIds, getCmrID(tmeSubjects, tmeIDs))
case "http://www.ft.com/ontology/Section":
esModel.CmrSections = append(esModel.CmrSections, annotation.Thing.PrefLabel)
esModel.CmrSectionsIds = append(esModel.CmrSectionsIds, getCmrID(tmeSections, tmeIDs))
esModel.CmrSections = appendIfNotExists(esModel.CmrSections, annotation.Thing.PrefLabel)
esModel.CmrSectionsIds = appendIfNotExists(esModel.CmrSectionsIds, getCmrID(tmeSections, tmeIDs))
if annotation.Thing.Predicate == primaryClassification {
esModel.CmrPrimarysection = new(string)
*esModel.CmrPrimarysection = annotation.Thing.PrefLabel
esModel.CmrPrimarysectionID = new(string)
*esModel.CmrPrimarysectionID = getCmrID(tmeSections, tmeIDs)
}
case "http://www.ft.com/ontology/Topic":
esModel.CmrTopics = append(esModel.CmrTopics, annotation.Thing.PrefLabel)
esModel.CmrTopicsIds = append(esModel.CmrTopicsIds, getCmrID(tmeTopics, tmeIDs))
esModel.CmrTopics = appendIfNotExists(esModel.CmrTopics, annotation.Thing.PrefLabel)
esModel.CmrTopicsIds = appendIfNotExists(esModel.CmrTopicsIds, getCmrID(tmeTopics, tmeIDs))
if annotation.Thing.Predicate == about {
esModel.CmrPrimarytheme = new(string)
*esModel.CmrPrimarytheme = annotation.Thing.PrefLabel
esModel.CmrPrimarythemeID = new(string)
*esModel.CmrPrimarythemeID = getCmrID(tmeTopics, tmeIDs)
}
case "http://www.ft.com/ontology/Location":
esModel.CmrRegions = append(esModel.CmrRegions, annotation.Thing.PrefLabel)
esModel.CmrRegionsIds = append(esModel.CmrRegionsIds, getCmrID(tmeRegions, tmeIDs))
esModel.CmrRegions = appendIfNotExists(esModel.CmrRegions, annotation.Thing.PrefLabel)
esModel.CmrRegionsIds = appendIfNotExists(esModel.CmrRegionsIds, getCmrID(tmeRegions, tmeIDs))
if annotation.Thing.Predicate == about {
esModel.CmrPrimarytheme = new(string)
*esModel.CmrPrimarytheme = annotation.Thing.PrefLabel
esModel.CmrPrimarythemeID = new(string)
*esModel.CmrPrimarythemeID = getCmrID(tmeRegions, tmeIDs)
}
case "http://www.ft.com/ontology/Genre":
esModel.CmrGenres = append(esModel.CmrGenres, annotation.Thing.PrefLabel)
esModel.CmrGenreIds = append(esModel.CmrGenreIds, getCmrID(tmeGenres, tmeIDs))
esModel.CmrGenres = appendIfNotExists(esModel.CmrGenres, annotation.Thing.PrefLabel)
esModel.CmrGenreIds = appendIfNotExists(esModel.CmrGenreIds, getCmrID(tmeGenres, tmeIDs))
case "http://www.ft.com/ontology/SpecialReport":
esModel.CmrSpecialreports = append(esModel.CmrSpecialreports, annotation.Thing.PrefLabel)
esModel.CmrSpecialreportsIds = append(esModel.CmrSpecialreportsIds, getCmrID(tmeSpecialReports, tmeIDs))
esModel.CmrSpecialreports = appendIfNotExists(esModel.CmrSpecialreports, annotation.Thing.PrefLabel)
esModel.CmrSpecialreportsIds = appendIfNotExists(esModel.CmrSpecialreportsIds, getCmrID(tmeSpecialReports, tmeIDs))
if annotation.Thing.Predicate == primaryClassification {
esModel.CmrPrimarysection = new(string)
*esModel.CmrPrimarysection = annotation.Thing.PrefLabel
esModel.CmrPrimarysectionID = new(string)
*esModel.CmrPrimarysectionID = getCmrID(tmeSpecialReports, tmeIDs)
}
}
}
}
Expand All @@ -331,3 +357,11 @@ func getCmrID(taxonomy string, tmeIDs []string) string {
}
return tmeIDs[0]
}
func appendIfNotExists(s []string, e string) []string {
for _, a := range s {
if a == e {
return s
}
}
return append(s, e)
}

0 comments on commit 5b56ffb

Please sign in to comment.