Fix writing Audio in elasticsearch when kafka header is missing; Fix schema checker

.circleci/config.yml

@@ -23,8 +23,6 @@ jobs:
             GO111MODULE=off go get -u github.com/myitcv/gobin
             curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.18.0
             wget https://raw.githubusercontent.com/Financial-Times/upp-coding-standard/v1.0.0/golangci-config/.golangci.yml
-            STATIK_VERSION=$(go list -mod=readonly -m all | grep statik | cut -d ' ' -f2)
-            gobin github.com/rakyll/statik@${STATIK_VERSION}
       - run:
           name: Make result folders
           command: |
@@ -35,7 +33,6 @@ jobs:
           command: |
             GOPRIVATE="github.com/Financial-Times"
             git config --global url."https://$GITHUB_USERNAME:$GITHUB_TOKEN@github.com/".insteadOf "https://github.com/"
-            go generate ./cmd/content-rw-elasticsearch
             go build -mod=readonly -v ./cmd/content-rw-elasticsearch
       - run:
           name: Run Linters
@@ -80,8 +77,6 @@ jobs:
             npm install -g --unsafe-perm --loglevel warn --user 0 --no-progress dredd@11.2.19
             rm -rf /var/lib/apt/lists/*
             GO111MODULE=off go get -u github.com/myitcv/gobin
-            STATIK_VERSION=$(go list -mod=readonly -m all | grep statik | cut -d ' ' -f2)
-            gobin github.com/rakyll/statik@${STATIK_VERSION}
       - run:
           name: Load ersatz-fixtures.yml to ersatz image
           command: "curl -X POST --data-binary @_ft/ersatz-fixtures.yml -H \"Content-type: text/x-yaml\" http://localhost:9000/__configure"
@@ -90,7 +85,6 @@ jobs:
           command: |
             GOPRIVATE="github.com/Financial-Times"
             git config --global url."https://$GITHUB_USERNAME:$GITHUB_TOKEN@github.com/".insteadOf "https://github.com/"
-            go generate ./cmd/content-rw-elasticsearch
             go build -mod=readonly -v ./cmd/content-rw-elasticsearch
       - run:
           name: Dredd API Testing

Dockerfile

@@ -12,23 +12,17 @@ ARG GITHUB_TOKEN
 COPY . ${SRC_FOLDER}
 WORKDIR ${SRC_FOLDER}
 
-# Install statik cli tool in GOPATH in order to successfully execute the go generate command
-RUN echo "machine github.com login $GITHUB_USERNAME password $GITHUB_TOKEN" > ~/.netrc \
-  && GOPRIVATE="github.com/Financial-Times" \
-  && GO111MODULE=off go get -u github.com/myitcv/gobin \
-  # Get statik version from go.mod of the project
-  && STATIK_VERSION="$(go list -mod=readonly -m all | grep statik | cut -d ' ' -f2)" \
-  && gobin github.com/rakyll/statik@${STATIK_VERSION} \
-  && go generate ./cmd/${PROJECT}
-
 # Build app
-RUN VERSION="version=$(git describe --tag --always 2> /dev/null)" \
+RUN echo "machine github.com login $GITHUB_USERNAME password $GITHUB_TOKEN" > ~/.netrc \
+  && VERSION="version=$(git describe --tag --always 2> /dev/null)" \
   && DATETIME="dateTime=$(date -u +%Y%m%d%H%M%S)" \
   && REPOSITORY="repository=$(git config --get remote.origin.url)" \
   && REVISION="revision=$(git rev-parse HEAD)" \
   && BUILDER="builder=$(go version)" \
   && LDFLAGS="-X '"${BUILDINFO_PACKAGE}$VERSION"' -X '"${BUILDINFO_PACKAGE}$DATETIME"' -X '"${BUILDINFO_PACKAGE}$REPOSITORY"' -X '"${BUILDINFO_PACKAGE}$REVISION"' -X '"${BUILDINFO_PACKAGE}$BUILDER"'" \
   && CGO_ENABLED=0 go build -mod=readonly -a -o /artifacts/${PROJECT}/${PROJECT} -ldflags="${LDFLAGS}" ./cmd/${PROJECT} \
+  && mkdir -p /artifacts/configs/ \
+  && cp -r ${SRC_FOLDER}/configs /artifacts/configs \
   && echo "Build flags: ${LDFLAGS}"
 
 # Multi-stage build - copy only the certs and the binary into the image

Makefile

@@ -4,19 +4,12 @@ STATIK_VERSION=$(shell go list -m all | grep statik | cut -d ' ' -f2)
 
 all: clean build-readonly test
 
-install:
-	GO111MODULE="off" go get -u github.com/myitcv/gobin
-	gobin github.com/rakyll/statik@${STATIK_VERSION}
 
-generate:
-	@echo ">>> Embedding static resources in binary..."
-	go generate ./cmd/${PROJECT_NAME}
-
-build: generate
+build:
 	@echo ">>> Building Application..."
 	go build -v ./cmd/${PROJECT_NAME}
 
-build-readonly: generate
+build-readonly:
 	@echo ">>> Building Application with -mod=readonly..."
 	go build -mod=readonly -v ./cmd/${PROJECT_NAME}
 

README.md

@@ -8,28 +8,12 @@ Indexes V2 content in Elasticsearch for use by SAPI V1
 
 ## Project Local Execution
 
-Before executing any of the proposed ways run
-
-```sh
-make install
-```
-
-to install external tools needed and then
-
 ```sh
 make all
 ```
 
 to run tests and a clean build of the project.
 
----
-
-**NOTE**
-
-Each time you modify any file under the `configs` directory, please run `make generate` in order to regenerate
-the `statik` package which embeds the files in the binary.
-
----
 
 ### Docker Compose
 

cmd/content-rw-elasticsearch/main.go

@@ -1,5 +1,3 @@
-//go:generate statik -src=../../configs -dest ../../
-
 package main
 
 import (

configs/app.yml

@@ -55,6 +55,6 @@ esContentTypeMetadata:
     format: "Videos"
     category: "video"
   audio:
-    collection: "FTAudios"
+    collection: "FTPodcasts"
     format: "Audios"
     category: "audio"

go.mod

@@ -15,7 +15,6 @@ require (
 	github.com/hashicorp/go-version v1.0.0 // indirect
 	github.com/jawher/mow.cli v1.0.4
 	github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c
-	github.com/rakyll/statik v0.1.7
 	github.com/smartystreets/go-aws-auth v0.0.0-20170504205021-8ef1316913ee
 	github.com/smartystreets/gunit v1.1.3 // indirect
 	github.com/spf13/viper v1.6.2

go.sum

@@ -109,8 +109,6 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
-github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=

helm/content-rw-elasticsearch/values.yaml

@@ -19,4 +19,4 @@ env:
   KAFKA_CONCURRENT_PROCESSING: ""
   PUBLIC_CONCORDANCES_ENDPOINT: ""
   INTERNAL_CONTENT_API_URL: ""
-  ELASTICSEARCH_SAPI_INDEX: "ft"
+  ELASTICSEARCH_SAPI_INDEX: "ft-new"

pkg/config/config.go

@@ -4,12 +4,12 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
-	// This blank import is required in order to read the embedded config files
-	_ "github.com/Financial-Times/content-rw-elasticsearch/v2/statik"
-	"github.com/rakyll/statik/fs"
 	"github.com/spf13/viper"
 )
 
@@ -24,6 +24,8 @@ const (
 	AudioType   = "audio"
 
 	PACOrigin = "http://cmdb.ft.com/systems/pac"
+
+	ContentTypeAudio = "Audio"
 )
 
 type ESContentTypeMetadataMap map[string]schema.ContentType
@@ -56,7 +58,7 @@ type AppConfig struct {
 }
 
 func ParseConfig(configFileName string) (AppConfig, error) {
-	contents, err := ReadEmbeddedResource(configFileName)
+	contents, err := ReadConfigFile(configFileName)
 	if err != nil {
 		return AppConfig{}, err
 	}
@@ -89,21 +91,16 @@ func ParseConfig(configFileName string) (AppConfig, error) {
 	}, nil
 }
 
-func ReadEmbeddedResource(fileName string) ([]byte, error) {
-	statikFS, err := fs.New()
+func ReadConfigFile(fileName string) ([]byte, error) {
+	path, err := filepath.Abs(fmt.Sprintf("./configs/%s", fileName))
 	if err != nil {
-		return nil, err
+		log.Fatal(err)
 	}
-	// Access individual files by their paths.
-	f, err := statikFS.Open("/" + fileName)
+	file, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		log.Fatal(err)
 	}
-	defer f.Close()
+	defer file.Close()
 
-	contents, err := ioutil.ReadAll(f)
-	if err != nil {
-		return nil, err
-	}
-	return contents, err
+	return ioutil.ReadAll(file)
 }

pkg/es/service.go

@@ -52,17 +52,18 @@ func (s *ElasticsearchService) GetSchemaHealth() (string, error) {
 	if referenceIndex == nil {
 		referenceIndex = new(elasticIndex)
 
-		referenceJSON, err := config.ReadEmbeddedResource("referenceSchema.json")
+		referenceJSON, err := config.ReadConfigFile("referenceSchema.json")
 		if err != nil {
 			return "", err
 		}
 
-		fullReferenceJSON := []byte(fmt.Sprintf(`{"ft": %s}`, string(referenceJSON)))
+		fullReferenceJSON := []byte(fmt.Sprintf(`{"%s": %s}`, s.IndexName, string(referenceJSON)))
 		err = json.Unmarshal(fullReferenceJSON, &referenceIndex.index)
 		if err != nil {
 			return "", err
 		}
 	}
+
 	if referenceIndex.index[s.IndexName] == nil || referenceIndex.index[s.IndexName].Settings == nil || referenceIndex.index[s.IndexName].Mappings == nil {
 		return "not ok, wrong referenceIndex", nil
 	}

pkg/mapper/mapper_test.go

@@ -115,7 +115,7 @@ func TestConvertToESContentModel(t *testing.T) {
 	}
 
 	log := logger.NewUPPLogger(config.AppName, config.AppDefaultLogLevel)
-	appConfig, err := config.ParseConfig("app.yml")
+	appConfig, err := config.ParseConfig("../../../configs/app.yml")
 	if err != nil {
 		log.Fatal(err)
 	}

pkg/message/message_handler.go

@@ -2,6 +2,7 @@ package message
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"strings"
 	"time"
@@ -100,7 +101,7 @@ func (h *Handler) handleMessage(msg consumer.Message) {
 
 	uuid := combinedPostPublicationEvent.UUID
 	log = log.WithUUID(uuid)
-	log.Info("Processing combined post publication event")
+	log.Info(fmt.Sprintf("Processing combined post publication event with type %s", combinedPostPublicationEvent.Content.Type))
 
 	contentType := h.readContentType(msg, combinedPostPublicationEvent)
 	if contentType == "" && msg.Headers[originHeader] != config.PACOrigin {
@@ -136,7 +137,7 @@ func (h *Handler) handleMessage(msg consumer.Message) {
 
 func (h *Handler) readContentType(msg consumer.Message, event schema.EnrichedContent) string {
 	typeHeader := msg.Headers[contentTypeHeader]
-	if strings.Contains(typeHeader, audioContentTypeHeader) {
+	if strings.Contains(typeHeader, audioContentTypeHeader) || event.Content.Type == config.ContentTypeAudio {
 		return config.AudioType
 	}
 	if strings.Contains(typeHeader, articleContentTypeHeader) {

pkg/message/message_handler_test.go

@@ -151,7 +151,7 @@ func mockMapperHandler(concordanceAPIMock *concordanceAPIMock, log *logger.UPPLo
 }
 
 func initAppConfig() config.AppConfig {
-	appConfig, err := config.ParseConfig("app.yml")
+	appConfig, err := config.ParseConfig("../../../configs/app.yml")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -288,7 +288,7 @@ func TestHandleWriteMessageAudio(t *testing.T) {
 	input := modifyTestInputAuthority("NEXT-VIDEO-EDITOR")
 
 	serviceMock := &esServiceMock{}
-	serviceMock.On("WriteData", "FTAudios", "aae9611e-f66c-4fe4-a6c6-2e2bdea69060", mock.Anything).Return(&elastic.IndexResult{}, nil)
+	serviceMock.On("WriteData", "FTPodcasts", "aae9611e-f66c-4fe4-a6c6-2e2bdea69060", mock.Anything).Return(&elastic.IndexResult{}, nil)
 	concordanceAPIMock := new(concordanceAPIMock)
 	concordanceAPIMock.On("GetConcepts", mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return(map[string]concept.Model{}, nil)
 
@@ -299,6 +299,25 @@ func TestHandleWriteMessageAudio(t *testing.T) {
 	concordanceAPIMock.AssertExpectations(t)
 }
 
+func TestHandleWriteMessageAudioWithoutHeader(t *testing.T) {
+
+	inputJSON := tst.ReadTestResource("exampleAudioModel.json")
+	input := strings.Replace(string(inputJSON), "FTCOM-METHODE", "NEXT-VIDEO-EDITOR", 1)
+
+	serviceMock := &esServiceMock{}
+	serviceMock.On("WriteData", "FTPodcasts", "aae9611e-f66c-4fe4-a6c6-2e2bdea69060", mock.Anything).Return(&elastic.IndexResult{}, nil)
+
+	concordanceAPIMock := new(concordanceAPIMock)
+
+	_, handler := mockMessageHandler(defaultESClient, serviceMock, concordanceAPIMock)
+	handler.handleMessage(consumer.Message{
+		Body:    input,
+		Headers: map[string]string{},
+	})
+
+	serviceMock.AssertExpectations(t)
+}
+
 func TestHandleWriteMessageArticleByHeaderType(t *testing.T) {
 	input := modifyTestInputAuthority("invalid")
 

test/testdata/exampleAudioModel.json

@@ -0,0 +1,58 @@
+{
+  "uuid": "aae9611e-f66c-4fe4-a6c6-2e2bdea69060",
+  "content": {
+    "accessLevel": "subscribed",
+    "alternativeStandfirsts": {
+      "promotionalStandfirst": "promotionalStandfirst"
+    },
+    "alternativeTitles": {
+      "contentPackageTitle": null,
+      "promotionalTitle": "promotionalTitle"
+    },
+    "body": "example body",
+    "byline": "byline",
+    "canBeDistributed": "yes",
+    "canBeSyndicated": "yes",
+    "comments": {
+      "enabled": true
+    },
+    "contentPackage": null,
+    "copyright": null,
+    "description": null,
+    "editorialDesk": "/FT/MarketNews",
+    "externalBinaryUrl": null,
+    "firstPublishedDate": "2017-06-26T04:00:17.000Z",
+    "identifiers": [
+      {
+        "authority": "http://api.ft.com/system/FTCOM-METHODE",
+        "identifierValue": "aae9611e-f66c-4fe4-a6c6-2e2bdea69060"
+      }
+    ],
+    "internalBinaryUrl": null,
+    "lastModified": "2018-04-04T12:58:00.347Z",
+    "mainImage": "5546cbc4-d4f7-47f9-a158-03856a0d3706",
+    "masterSource": null,
+    "mediaType": null,
+    "members": null,
+    "pixelHeight": null,
+    "pixelWidth": null,
+    "publishReference": "tid_123",
+    "publishedDate": "2017-06-26T04:00:17.000Z",
+    "rightsGroup": null,
+    "standfirst": "",
+    "standout": {
+      "editorsChoice": false,
+      "exclusive": false,
+      "scoop": false
+    },
+    "storyPackage": null,
+    "title": "Example audio",
+    "type": "Audio",
+    "uuid": "aae9611e-f66c-4fe4-a6c6-2e2bdea69060",
+    "webUrl": null
+  },
+  "metadata": [],
+  "contentUri": "http://methode-article-mapper.svc.ft.com/content/aae9611e-f66c-4fe4-a6c6-2e2bdea69060",
+  "lastModified": "2020-07-06T12:58:00.347Z",
+  "markedDeleted": "false"
+}