From 0dd44ebed6cbeb0bce86be1fca7f39d0b3800e33 Mon Sep 17 00:00:00 2001 From: Tsvetan Dimitrov Date: Thu, 20 Feb 2020 18:29:39 +0200 Subject: [PATCH] UPPSF-1150 Fixed shutdown hook, some naming, added snyk scanning and dredd config in circleci --- .circleci/config.yml | 98 +++++++++++++++++++++++---------------- Dockerfile | 3 -- configs/config.yml | 2 +- pkg/config/config.go | 9 ++-- pkg/health/healthcheck.go | 8 ++-- pkg/http/server.go | 7 ++- pkg/mapper/mapper.go | 16 +++---- 7 files changed, 80 insertions(+), 63 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e0fcfb6..792ab5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,60 +1,71 @@ -version: 2 +version: 2.1 + +orbs: + golang-ci: financial-times/golang-ci@1.0.0 + jobs: build: working_directory: /go/src/github.com/Financial-Times/content-rw-elasticsearch docker: - - image: golang:1 - environment: - CIRCLE_TEST_REPORTS: /tmp/test-results - CIRCLE_COVERAGE_REPORT: /tmp/coverage-results + - image: golang:1 + environment: + CIRCLE_TEST_REPORTS: /tmp/test-results + CIRCLE_COVERAGE_REPORT: /tmp/coverage-results steps: - - checkout - - run: - name: External dependencies - command: | - GO111MODULE=off go get -u github.com/mattn/goveralls - GO111MODULE=off go get -u github.com/jstemmer/go-junit-report - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.18.0 - wget https://raw.githubusercontent.com/Financial-Times/upp-coding-standard/v1.0.0/golangci-config/.golangci.yml - - run: - name: Make result folders - command: | - mkdir -p ${CIRCLE_TEST_REPORTS} - mkdir -p ${CIRCLE_COVERAGE_REPORT} - - run: - name: Go build - command: go build -mod=readonly -v ./cmd/content-rw-elasticsearch - - run: - name: Run Linters - command: golangci-lint run --config=.golangci.yml --new-from-rev=master - - run: - name: Run tests - command: go test -race -cover -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out ./... | go-junit-report > ${CIRCLE_TEST_REPORTS}/junit.xml - - run: - name: Upload coverage - command: goveralls -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out -service=circle-ci -repotoken=${COVERALLS_TOKEN} - - store_test_results: - path: /tmp/test-results + - checkout + - run: + name: External dependencies + command: | + GO111MODULE=off go get -u github.com/mattn/goveralls + GO111MODULE=off go get -u github.com/jstemmer/go-junit-report + curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.18.0 + wget https://raw.githubusercontent.com/Financial-Times/upp-coding-standard/v1.0.0/golangci-config/.golangci.yml + - run: + name: Make result folders + command: | + mkdir -p ${CIRCLE_TEST_REPORTS} + mkdir -p ${CIRCLE_COVERAGE_REPORT} + - run: + name: Go build + command: go build -mod=readonly -v ./cmd/content-rw-elasticsearch + - run: + name: Run Linters + command: golangci-lint run --config=.golangci.yml --new-from-rev=master + - run: + name: Run tests + command: go test -race -mod=readonly -cover -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out ./... | go-junit-report > ${CIRCLE_TEST_REPORTS}/junit.xml + - run: + name: Upload coverage + command: goveralls -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out -service=circle-ci -repotoken=${COVERALLS_TOKEN} + - store_test_results: + path: /tmp/test-results dockerfile: working_directory: /content-rw-elasticsearch docker: - - image: docker:18.03 + - image: docker:18.03 steps: - - checkout - - setup_docker_engine - - run: - name: Build Dockerfile - command: docker build . + - checkout + - setup_remote_docker + - run: + name: Build Dockerfile + command: docker build . dredd: working_directory: /go/src/github.com/Financial-Times/content-rw-elasticsearch docker: - - image: bankrs/golang-dredd:go1.13.7-dredd11.2.19 + - image: golang:1 environment: GOPATH: /go ELASTICSEARCH_SAPI_ENDPOINT: http://localhost:9000 - image: peteclarkft/ersatz:stable steps: - checkout + - run: + name: Download dredd + command: | + curl -sL https://deb.nodesource.com/setup_11.x | bash - + DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs=11.\* + npm install -g --unsafe-perm --loglevel warn --user 0 --no-progress dredd@11.2.19 + rm -rf /var/lib/apt/lists/* - run: name: Load ersatz-fixtures.yml to ersatz image command: "curl -X POST --data-binary @_ft/ersatz-fixtures.yml -H \"Content-type: text/x-yaml\" http://localhost:9000/__configure" @@ -65,11 +76,16 @@ jobs: name: Dredd API Testing command: dredd workflows: - version: 2 + version: 2.1 test-and-build-docker: jobs: - build - dredd - dockerfile: requires: - - build \ No newline at end of file + - build + scanning: + jobs: + - golang-ci/scan: + name: scan-dependencies + context: org-global \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 441ddf1..1ed74f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,4 @@ WORKDIR / COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=0 /artifacts/* / -## Copy things specific for this service -#COPY configs /configs - CMD [ "/content-rw-elasticsearch" ] diff --git a/configs/config.yml b/configs/config.yml index c69cf27..e5a54b3 100644 --- a/configs/config.yml +++ b/configs/config.yml @@ -1,4 +1,4 @@ -concepts: +conceptTypes: organisation: "http://www.ft.com/ontology/organisation/Organisation" person: "http://www.ft.com/ontology/person/Person" company: "http://www.ft.com/ontology/company/Company" diff --git a/pkg/config/config.go b/pkg/config/config.go index d3e6a50..0df1c28 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,7 +2,6 @@ package config import ( "fmt" - "log" "os" "path" "strings" @@ -37,7 +36,7 @@ func (c ContentTypeMap) Get(key string) schema.ContentType { type AppConfig struct { Predicates Map - Concepts Map + ConceptTypes Map Origins Map Authorities Map ContentTypeMap ContentTypeMap @@ -48,11 +47,11 @@ func ParseConfig(configFilePath string) (AppConfig, error) { v.SetConfigType("yaml") v.SetConfigFile(joinPath(ProjectRoot, configFilePath)) if err := v.ReadInConfig(); err != nil { - log.Fatal(err) + return AppConfig{}, err } origins := v.Sub("content").GetStringMapString("origin") predicates := v.GetStringMapString("predicates") - concepts := v.GetStringMapString("concepts") + concepts := v.GetStringMapString("conceptTypes") authorities := v.GetStringMapString("authorities") var contentTypeMap ContentTypeMap err := v.UnmarshalKey("esContentTypeMap", &contentTypeMap) @@ -61,7 +60,7 @@ func ParseConfig(configFilePath string) (AppConfig, error) { } return AppConfig{ Predicates: predicates, - Concepts: concepts, + ConceptTypes: concepts, Origins: origins, Authorities: authorities, ContentTypeMap: contentTypeMap, diff --git a/pkg/health/healthcheck.go b/pkg/health/healthcheck.go index b01912f..46f3a99 100644 --- a/pkg/health/healthcheck.go +++ b/pkg/health/healthcheck.go @@ -16,8 +16,8 @@ import ( ) const ( - PathHealth = "/__health" - PathHealthDetails = "/__health-details" + pathHealth = "/__health" + pathHealthDetails = "/__health-details" ) type Service struct { @@ -57,8 +57,8 @@ func (s *Service) AttachHTTPEndpoints(serveMux *http.ServeMux, appName string, a Description: appDescription, Checks: s.Checks, } - serveMux.HandleFunc(PathHealth, fthealth.Handler(hc)) - serveMux.HandleFunc(PathHealthDetails, s.healthDetails) + serveMux.HandleFunc(pathHealth, fthealth.Handler(hc)) + serveMux.HandleFunc(pathHealthDetails, s.healthDetails) serveMux.HandleFunc(status.GTGPath, status.NewGoodToGoHandler(s.gtgCheck)) serveMux.HandleFunc(status.BuildInfoPath, status.BuildInfoHandler) diff --git a/pkg/http/server.go b/pkg/http/server.go index 0d900ee..c5be069 100644 --- a/pkg/http/server.go +++ b/pkg/http/server.go @@ -1,11 +1,13 @@ package http import ( + "context" "net/http" "os" "os/signal" "sync" "syscall" + "time" "github.com/Financial-Times/go-logger/v2" ) @@ -25,7 +27,10 @@ func StartServer(log *logger.UPPLogger, serveMux *http.ServeMux, port string) { waitForSignal() log.Info("[Shutdown] Application is shutting down") - if err := server.Close(); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := server.Shutdown(ctx); err != nil { log.WithError(err).Error("Unable to stop http server") } wg.Wait() diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index 084a99c..43b6ed5 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -94,15 +94,15 @@ func (h *Handler) populateAnnotationRelatedFields(annotation schema.Thing, model hasAuthor := h.Config.Predicates.Get("hasAuthor") hasContributor := h.Config.Predicates.Get("hasContributor") for _, taxonomy := range annotation.Types { - concepts := h.Config.Concepts + conceptTypes := h.Config.ConceptTypes switch taxonomy { - case concepts.Get("organisation"): + case conceptTypes.Get("organisation"): model.CmrOrgnames = appendIfNotExists(model.CmrOrgnames, annotation.PrefLabel) model.CmrOrgnamesIds = prepareElasticField(model.CmrOrgnamesIds, annIDs) if annotation.Predicate == about { setPrimaryTheme(model, annotation.PrefLabel, getCmrIDWithFallback(tmeOrganisations, annIDs)) } - case concepts.Get("person"): + case conceptTypes.Get("person"): _, personFound := getCmrID(tmePeople, annIDs) authorCmrID, authorFound := getCmrID(tmeAuthors, annIDs) // if it's only author, skip adding to people @@ -120,25 +120,25 @@ func (h *Handler) populateAnnotationRelatedFields(annotation schema.Thing, model if annotation.Predicate == about { setPrimaryTheme(model, annotation.PrefLabel, getCmrIDWithFallback(tmePeople, annIDs)) } - case concepts.Get("company"): + case conceptTypes.Get("company"): model.CmrCompanynames = appendIfNotExists(model.CmrCompanynames, annotation.PrefLabel) model.CmrCompanynamesIds = prepareElasticField(model.CmrCompanynamesIds, annIDs) - case concepts.Get("brand"): + case conceptTypes.Get("brand"): model.CmrBrands = appendIfNotExists(model.CmrBrands, annotation.PrefLabel) model.CmrBrandsIds = prepareElasticField(model.CmrBrandsIds, annIDs) - case concepts.Get("topic"): + case conceptTypes.Get("topic"): model.CmrTopics = appendIfNotExists(model.CmrTopics, annotation.PrefLabel) model.CmrTopicsIds = prepareElasticField(model.CmrTopicsIds, annIDs) if annotation.Predicate == about { setPrimaryTheme(model, annotation.PrefLabel, getCmrIDWithFallback(tmeTopics, annIDs)) } - case concepts.Get("location"): + case conceptTypes.Get("location"): model.CmrRegions = appendIfNotExists(model.CmrRegions, annotation.PrefLabel) model.CmrRegionsIds = prepareElasticField(model.CmrRegionsIds, annIDs) if annotation.Predicate == about { setPrimaryTheme(model, annotation.PrefLabel, getCmrIDWithFallback(tmeRegions, annIDs)) } - case concepts.Get("genre"): + case conceptTypes.Get("genre"): model.CmrGenres = appendIfNotExists(model.CmrGenres, annotation.PrefLabel) model.CmrGenreIds = prepareElasticField(model.CmrGenreIds, annIDs) }