UPPSF-1150 Fixed shutdown hook, some naming, added snyk scanning and …

…dredd config in circleci

.circleci/config.yml

@@ -1,60 +1,71 @@
-version: 2
+version: 2.1
+
+orbs:
+  golang-ci: financial-times/golang-ci@1.0.0
+
 jobs:
   build:
     working_directory: /go/src/github.com/Financial-Times/content-rw-elasticsearch
     docker:
-    - image: golang:1
-      environment:
-        CIRCLE_TEST_REPORTS: /tmp/test-results
-        CIRCLE_COVERAGE_REPORT: /tmp/coverage-results
+      - image: golang:1
+        environment:
+          CIRCLE_TEST_REPORTS: /tmp/test-results
+          CIRCLE_COVERAGE_REPORT: /tmp/coverage-results
     steps:
-    - checkout
-    - run:
-        name: External dependencies
-        command: |
-          GO111MODULE=off go get -u github.com/mattn/goveralls
-          GO111MODULE=off go get -u github.com/jstemmer/go-junit-report
-          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.18.0
-          wget https://raw.githubusercontent.com/Financial-Times/upp-coding-standard/v1.0.0/golangci-config/.golangci.yml
-    - run:
-        name: Make result folders
-        command: |
-          mkdir -p ${CIRCLE_TEST_REPORTS}
-          mkdir -p ${CIRCLE_COVERAGE_REPORT}
-    - run:
-        name: Go build
-        command: go build -mod=readonly -v ./cmd/content-rw-elasticsearch
-    - run:
-        name: Run Linters
-        command: golangci-lint run --config=.golangci.yml --new-from-rev=master
-    - run:
-        name: Run tests
-        command: go test -race -cover -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out ./... | go-junit-report > ${CIRCLE_TEST_REPORTS}/junit.xml
-    - run:
-        name: Upload coverage
-        command: goveralls -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out -service=circle-ci -repotoken=${COVERALLS_TOKEN}
-    - store_test_results:
-        path: /tmp/test-results
+      - checkout
+      - run:
+          name: External dependencies
+          command: |
+            GO111MODULE=off go get -u github.com/mattn/goveralls
+            GO111MODULE=off go get -u github.com/jstemmer/go-junit-report
+            curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.18.0
+            wget https://raw.githubusercontent.com/Financial-Times/upp-coding-standard/v1.0.0/golangci-config/.golangci.yml
+      - run:
+          name: Make result folders
+          command: |
+            mkdir -p ${CIRCLE_TEST_REPORTS}
+            mkdir -p ${CIRCLE_COVERAGE_REPORT}
+      - run:
+          name: Go build
+          command: go build -mod=readonly -v ./cmd/content-rw-elasticsearch
+      - run:
+          name: Run Linters
+          command: golangci-lint run --config=.golangci.yml --new-from-rev=master
+      - run:
+          name: Run tests
+          command: go test -race -mod=readonly -cover -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out ./... | go-junit-report > ${CIRCLE_TEST_REPORTS}/junit.xml
+      - run:
+          name: Upload coverage
+          command: goveralls -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out -service=circle-ci -repotoken=${COVERALLS_TOKEN}
+      - store_test_results:
+          path: /tmp/test-results
   dockerfile:
     working_directory: /content-rw-elasticsearch
     docker:
-    - image: docker:18.03
+      - image: docker:18.03
     steps:
-    - checkout
-    - setup_docker_engine
-    - run:
-        name: Build Dockerfile
-        command: docker build .
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Build Dockerfile
+          command: docker build .
   dredd:
     working_directory: /go/src/github.com/Financial-Times/content-rw-elasticsearch
     docker:
-      - image: bankrs/golang-dredd:go1.13.7-dredd11.2.19
+      - image: bankrs/golang:1
         environment:
           GOPATH: /go
           ELASTICSEARCH_SAPI_ENDPOINT: http://localhost:9000
       - image: peteclarkft/ersatz:stable
     steps:
       - checkout
+      - run:
+          name: Download dredd
+          command: |
+            curl -sL https://deb.nodesource.com/setup_11.x | bash -
+            DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs=11.\*
+            npm install -g --unsafe-perm --loglevel warn --user 0 --no-progress dredd@11.2.19
+            rm -rf /var/lib/apt/lists/*
       - run:
           name: Load ersatz-fixtures.yml to ersatz image
           command: "curl -X POST --data-binary @_ft/ersatz-fixtures.yml -H \"Content-type: text/x-yaml\" http://localhost:9000/__configure"
@@ -65,11 +76,16 @@ jobs:
           name: Dredd API Testing
           command: dredd
 workflows:
-  version: 2
+  version: 2.1
   test-and-build-docker:
     jobs:
       - build
       - dredd
       - dockerfile:
           requires:
-            - build
+            - build
+  scanning:
+    jobs:
+      - golang-ci/scan:
+          name: scan-dependencies
+          context: org-global

Dockerfile

@@ -25,7 +25,4 @@ WORKDIR /
 COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=0 /artifacts/* /
 
-## Copy things specific for this service
-#COPY configs /configs
-
 CMD [ "/content-rw-elasticsearch" ]

configs/config.yml

@@ -1,4 +1,4 @@
-concepts:
+conceptTypes:
   organisation: "http://www.ft.com/ontology/organisation/Organisation"
   person: "http://www.ft.com/ontology/person/Person"
   company: "http://www.ft.com/ontology/company/Company"

pkg/config/config.go

@@ -2,7 +2,6 @@ package config
 
 import (
 	"fmt"
-	"log"
 	"os"
 	"path"
 	"strings"
@@ -37,7 +36,7 @@ func (c ContentTypeMap) Get(key string) schema.ContentType {
 
 type AppConfig struct {
 	Predicates     Map
-	Concepts       Map
+	ConceptTypes   Map
 	Origins        Map
 	Authorities    Map
 	ContentTypeMap ContentTypeMap
@@ -48,11 +47,11 @@ func ParseConfig(configFilePath string) (AppConfig, error) {
 	v.SetConfigType("yaml")
 	v.SetConfigFile(joinPath(ProjectRoot, configFilePath))
 	if err := v.ReadInConfig(); err != nil {
-		log.Fatal(err)
+		return AppConfig{}, err
 	}
 	origins := v.Sub("content").GetStringMapString("origin")
 	predicates := v.GetStringMapString("predicates")
-	concepts := v.GetStringMapString("concepts")
+	concepts := v.GetStringMapString("conceptTypes")
 	authorities := v.GetStringMapString("authorities")
 	var contentTypeMap ContentTypeMap
 	err := v.UnmarshalKey("esContentTypeMap", &contentTypeMap)
@@ -61,7 +60,7 @@ func ParseConfig(configFilePath string) (AppConfig, error) {
 	}
 	return AppConfig{
 		Predicates:     predicates,
-		Concepts:       concepts,
+		ConceptTypes:   concepts,
 		Origins:        origins,
 		Authorities:    authorities,
 		ContentTypeMap: contentTypeMap,

pkg/health/healthcheck.go

@@ -16,8 +16,8 @@ import (
 )
 
 const (
-	PathHealth        = "/__health"
-	PathHealthDetails = "/__health-details"
+	pathHealth        = "/__health"
+	pathHealthDetails = "/__health-details"
 )
 
 type Service struct {
@@ -57,8 +57,8 @@ func (s *Service) AttachHTTPEndpoints(serveMux *http.ServeMux, appName string, a
 		Description: appDescription,
 		Checks:      s.Checks,
 	}
-	serveMux.HandleFunc(PathHealth, fthealth.Handler(hc))
-	serveMux.HandleFunc(PathHealthDetails, s.healthDetails)
+	serveMux.HandleFunc(pathHealth, fthealth.Handler(hc))
+	serveMux.HandleFunc(pathHealthDetails, s.healthDetails)
 	serveMux.HandleFunc(status.GTGPath, status.NewGoodToGoHandler(s.gtgCheck))
 	serveMux.HandleFunc(status.BuildInfoPath, status.BuildInfoHandler)
 

pkg/http/server.go

@@ -1,11 +1,13 @@
 package http
 
 import (
+	"context"
 	"net/http"
 	"os"
 	"os/signal"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/Financial-Times/go-logger/v2"
 )
@@ -25,7 +27,10 @@ func StartServer(log *logger.UPPLogger, serveMux *http.ServeMux, port string) {
 	waitForSignal()
 	log.Info("[Shutdown] Application is shutting down")
 
-	if err := server.Close(); err != nil {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if err := server.Shutdown(ctx); err != nil {
 		log.WithError(err).Error("Unable to stop http server")
 	}
 	wg.Wait()

pkg/mapper/mapper.go

@@ -94,15 +94,15 @@ func (h *Handler) populateAnnotationRelatedFields(annotation schema.Thing, model
 	hasAuthor := h.Config.Predicates.Get("hasAuthor")
 	hasContributor := h.Config.Predicates.Get("hasContributor")
 	for _, taxonomy := range annotation.Types {
-		concepts := h.Config.Concepts
+		conceptTypes := h.Config.ConceptTypes
 		switch taxonomy {
-		case concepts.Get("organisation"):
+		case conceptTypes.Get("organisation"):
 			model.CmrOrgnames = appendIfNotExists(model.CmrOrgnames, annotation.PrefLabel)
 			model.CmrOrgnamesIds = prepareElasticField(model.CmrOrgnamesIds, annIDs)
 			if annotation.Predicate == about {
 				setPrimaryTheme(model, annotation.PrefLabel, getCmrIDWithFallback(tmeOrganisations, annIDs))
 			}
-		case concepts.Get("person"):
+		case conceptTypes.Get("person"):
 			_, personFound := getCmrID(tmePeople, annIDs)
 			authorCmrID, authorFound := getCmrID(tmeAuthors, annIDs)
 			// if it's only author, skip adding to people
@@ -120,25 +120,25 @@ func (h *Handler) populateAnnotationRelatedFields(annotation schema.Thing, model
 			if annotation.Predicate == about {
 				setPrimaryTheme(model, annotation.PrefLabel, getCmrIDWithFallback(tmePeople, annIDs))
 			}
-		case concepts.Get("company"):
+		case conceptTypes.Get("company"):
 			model.CmrCompanynames = appendIfNotExists(model.CmrCompanynames, annotation.PrefLabel)
 			model.CmrCompanynamesIds = prepareElasticField(model.CmrCompanynamesIds, annIDs)
-		case concepts.Get("brand"):
+		case conceptTypes.Get("brand"):
 			model.CmrBrands = appendIfNotExists(model.CmrBrands, annotation.PrefLabel)
 			model.CmrBrandsIds = prepareElasticField(model.CmrBrandsIds, annIDs)
-		case concepts.Get("topic"):
+		case conceptTypes.Get("topic"):
 			model.CmrTopics = appendIfNotExists(model.CmrTopics, annotation.PrefLabel)
 			model.CmrTopicsIds = prepareElasticField(model.CmrTopicsIds, annIDs)
 			if annotation.Predicate == about {
 				setPrimaryTheme(model, annotation.PrefLabel, getCmrIDWithFallback(tmeTopics, annIDs))
 			}
-		case concepts.Get("location"):
+		case conceptTypes.Get("location"):
 			model.CmrRegions = appendIfNotExists(model.CmrRegions, annotation.PrefLabel)
 			model.CmrRegionsIds = prepareElasticField(model.CmrRegionsIds, annIDs)
 			if annotation.Predicate == about {
 				setPrimaryTheme(model, annotation.PrefLabel, getCmrIDWithFallback(tmeRegions, annIDs))
 			}
-		case concepts.Get("genre"):
+		case conceptTypes.Get("genre"):
 			model.CmrGenres = appendIfNotExists(model.CmrGenres, annotation.PrefLabel)
 			model.CmrGenreIds = prepareElasticField(model.CmrGenreIds, annIDs)
 		}