UPPSF-1100 Added spark and cct origins and authorities

cmd/content-rw-elasticsearch/main.go

@@ -3,15 +3,15 @@
 package main
 
 import (
-	nethttp "net/http"
+	"net/http"
 	"os"
 	"time"
 
 	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
 	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
 	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/es"
 	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/health"
-	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/http"
+	pkghttp "github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/http"
 	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/mapper"
 	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/message"
 	"github.com/Financial-Times/go-logger/v2"
@@ -129,15 +129,15 @@ func main() {
 			Endpoint:  *esEndpoint,
 		}
 
-		httpClient := http.NewHTTPClient()
+		httpClient := pkghttp.NewHTTPClient()
 
 		appConfig, err := config.ParseConfig("app.yml")
 		if err != nil {
 			log.Fatal(err)
 		}
 
 		esService := es.NewService(*indexName)
-		
+
 		concordanceAPIService := concept.NewConcordanceAPIService(*publicConcordancesEndpoint, httpClient)
 
 		mapperHandler := mapper.NewMapperHandler(
@@ -160,9 +160,9 @@ func main() {
 
 		healthService := health.NewHealthService(&queueConfig, esService, httpClient, concordanceAPIService, *appSystemCode, log)
 		//
-		serveMux := nethttp.NewServeMux()
+		serveMux := http.NewServeMux()
 		serveMux = healthService.AttachHTTPEndpoints(serveMux, *appName, config.AppDescription)
-		http.StartServer(log, serveMux, *port)
+		pkghttp.StartServer(log, serveMux, *port)
 
 		handler.Stop()
 	}

configs/app.yml

@@ -19,19 +19,32 @@ predicates:
   hasAuthor: "http://www.ft.com/ontology/annotation/hasAuthor"
   hasContributor: "http://www.ft.com/ontology/hasContributor"
 
-content:
-  origin:
-    methode: "methode-web-pub"
-    wordpress: "wordpress"
-    video: "next-video-editor"
-    pac: "http://cmdb.ft.com/systems/pac"
-
-authorities:
-  article: "http://api.ft.com/system/FTCOM-METHODE"
-  blog: "http://api.ft.com/system/FT-LABS-WP"
-  video: "http://api.ft.com/system/NEXT-VIDEO-EDITOR"
+contentMetadata:
+  methode:
+    origin: "methode-web-pub"
+    authority: "http://api.ft.com/system/FTCOM-METHODE"
+    contentType: "article"
+  wordpress:
+    origin: "wordpress"
+    authority: "http://api.ft.com/system/FT-LABS-WP"
+    contentType: "blog"
+  video:
+    origin: "next-video-editor"
+    authority: "http://api.ft.com/system/NEXT-VIDEO-EDITOR"
+    contentType: "video"
+  cct:
+    origin: "http://cmdb.ft.com/systems/cct"
+    authority: "http://api.ft.com/system/cct"
+    contentType: "article"
+  spark:
+    origin: "http://cmdb.ft.com/systems/spark"
+    authority: "http://api.ft.com/system/spark"
+    contentType: "article"
+  pac:
+    origin: "http://cmdb.ft.com/systems/pac"
+    contentType: "article"
 
-esContentTypeMap:
+esContentTypeMetadata:
   article:
     collection: "FTCom"
     format: "Articles"

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/Financial-Times/go-logger/v2 v2.0.1
 	github.com/Financial-Times/message-queue-gonsumer v0.0.0-20180518165041-cd41937c7566
 	github.com/Financial-Times/service-status-go v0.0.0-20160323111542-3f5199736a3d
+	github.com/Financial-Times/transactionid-utils-go v0.2.0
 	github.com/Financial-Times/uuid-utils-go v0.0.0-20170516110427-e22658edd0f1
 	github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9
 	github.com/fortytw2/leaktest v1.3.0 // indirect

go.sum

@@ -10,6 +10,8 @@ github.com/Financial-Times/message-queue-gonsumer v0.0.0-20180518165041-cd41937c
 github.com/Financial-Times/message-queue-gonsumer v0.0.0-20180518165041-cd41937c7566/go.mod h1:A88i3psx3Zm80Ai2OYTrwzKkZGKj+x5KL02z+YrRd10=
 github.com/Financial-Times/service-status-go v0.0.0-20160323111542-3f5199736a3d h1:USNBTIof6vWGM49SYrxvC5Y8NqyDL3YuuYmID81ORZQ=
 github.com/Financial-Times/service-status-go v0.0.0-20160323111542-3f5199736a3d/go.mod h1:7zULC9rrq6KxFkpB3Y5zNVaEwrf1g2m3dvXJBPDXyvM=
+github.com/Financial-Times/transactionid-utils-go v0.2.0 h1:YcET5Hd1fUGWWpQSVszYUlAc15ca8tmjRetUuQKRqEQ=
+github.com/Financial-Times/transactionid-utils-go v0.2.0/go.mod h1:tPAcAFs/dR6Q7hBDGNyUyixHRvg/n9NW/JTq8C58oZ0=
 github.com/Financial-Times/uuid-utils-go v0.0.0-20170516110427-e22658edd0f1 h1:FXM7cqqPyGh2QZ8BRJA16Gr65/+/91KEFSPKyRM+Nd8=
 github.com/Financial-Times/uuid-utils-go v0.0.0-20170516110427-e22658edd0f1/go.mod h1:i62wLwNq+NmRCQpZS5BLTKsOVYsTOxs9bSx7FgtxXwM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -168,6 +170,7 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=

pkg/config/config.go

@@ -24,23 +24,33 @@ const (
 	AudioType   = "audio"
 )
 
-type ContentTypeMap map[string]schema.ContentType
+type ESContentTypeMetadataMap map[string]schema.ContentType
 type Map map[string]string
+type ContentMetadataMap map[string]ContentMetadata
+
+type ContentMetadata struct {
+	Origin      string
+	Authority   string
+	ContentType string
+}
 
 func (c Map) Get(key string) string {
 	return c[strings.ToLower(key)]
 }
 
-func (c ContentTypeMap) Get(key string) schema.ContentType {
+func (c ESContentTypeMetadataMap) Get(key string) schema.ContentType {
+	return c[strings.ToLower(key)]
+}
+
+func (c ContentMetadataMap) Get(key string) ContentMetadata {
 	return c[strings.ToLower(key)]
 }
 
 type AppConfig struct {
-	Predicates     Map
-	ConceptTypes   Map
-	Origins        Map
-	Authorities    Map
-	ContentTypeMap ContentTypeMap
+	Predicates               Map
+	ConceptTypes             Map
+	ContentMetadataMap       ContentMetadataMap
+	ESContentTypeMetadataMap ESContentTypeMetadataMap
 }
 
 func ParseConfig(configFileName string) (AppConfig, error) {
@@ -55,22 +65,25 @@ func ParseConfig(configFileName string) (AppConfig, error) {
 		return AppConfig{}, err
 	}
 
-	origins := v.Sub("content").GetStringMapString("origin")
+	var contentMetadataMap ContentMetadataMap
+	err = v.UnmarshalKey("contentMetadata", &contentMetadataMap)
+	if err != nil {
+		return AppConfig{}, fmt.Errorf("unable to unmarshal %w", err)
+	}
+
 	predicates := v.GetStringMapString("predicates")
 	concepts := v.GetStringMapString("conceptTypes")
-	authorities := v.GetStringMapString("authorities")
-	var contentTypeMap ContentTypeMap
-	err = v.UnmarshalKey("esContentTypeMap", &contentTypeMap)
+	var contentTypeMetadataMap ESContentTypeMetadataMap
+	err = v.UnmarshalKey("esContentTypeMetadata", &contentTypeMetadataMap)
 	if err != nil {
 		return AppConfig{}, fmt.Errorf("unable to unmarshal %w", err)
 	}
 
 	return AppConfig{
-		Predicates:     predicates,
-		ConceptTypes:   concepts,
-		Origins:        origins,
-		Authorities:    authorities,
-		ContentTypeMap: contentTypeMap,
+		Predicates:               predicates,
+		ConceptTypes:             concepts,
+		ContentMetadataMap:       contentMetadataMap,
+		ESContentTypeMetadataMap: contentTypeMetadataMap,
 	}, nil
 }
 

pkg/es/client.go

@@ -32,7 +32,7 @@ func (a AWSSigningTransport) RoundTrip(req *http.Request) (*http.Response, error
 	return a.HTTPClient.Do(awsauth.Sign4(req, a.Credentials))
 }
 
-func NewClient(config AccessConfig, c *http.Client) (Client, error) {
+func NewClient(config AccessConfig, c *http.Client, log *logger.UPPLogger) (Client, error) {
 	signingTransport := AWSSigningTransport{
 		Credentials: awsauth.Credentials{
 			AccessKeyID:     config.AccessKey,
@@ -47,6 +47,7 @@ func NewClient(config AccessConfig, c *http.Client) (Client, error) {
 		elastic.SetScheme("https"),
 		elastic.SetHttpClient(signingClient),
 		elastic.SetSniff(false), // needs to be disabled due to EAS behavior. Healthcheck still operates as normal.
-		elastic.SetErrorLog(logger.NewUnstructuredLogger()),
+		// elastic.SetErrorLog(logger.NewUnstructuredLogger()),
+		elastic.SetErrorLog(log),
 	)
 }

pkg/es/service.go

@@ -107,6 +107,8 @@ func (s *ElasticsearchService) SetClient(client Client) {
 }
 
 func (s *ElasticsearchService) WriteData(conceptType string, uuid string, payload interface{}) (*elastic.IndexResult, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
 	return s.ElasticClient.Index().
 		Index(s.IndexName).
 		Type(conceptType).
@@ -116,6 +118,8 @@ func (s *ElasticsearchService) WriteData(conceptType string, uuid string, payloa
 }
 
 func (s *ElasticsearchService) DeleteData(conceptType string, uuid string) (*elastic.DeleteResult, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
 	return s.ElasticClient.Delete().
 		Index(s.IndexName).
 		Type(conceptType).

pkg/mapper/mapper.go

@@ -173,9 +173,9 @@ func (h *Handler) populateContentRelatedFields(model *schema.IndexModel, enriche
 	model.InternalContentType = new(string)
 	*model.InternalContentType = contentType
 	model.Category = new(string)
-	*model.Category = h.Config.ContentTypeMap.Get(contentType).Category
+	*model.Category = h.Config.ESContentTypeMetadataMap.Get(contentType).Category
 	model.Format = new(string)
-	*model.Format = h.Config.ContentTypeMap.Get(contentType).Format
+	*model.Format = h.Config.ESContentTypeMetadataMap.Get(contentType).Format
 	model.UID = &(enrichedContent.Content.UUID)
 	model.LeadHeadline = new(string)
 	*model.LeadHeadline = html.TransformText(enrichedContent.Content.Title,