UPPSF-1150 Added v2 to go.mod and all imports

- Removed unneeded mutexes
- Added install target in Makefile for installing statik circleci
- Deleted generated statik binary which will now be generated only
before build
- Renamed test/data to test/testdata to ignore being detected from go
tooling
- Added statik as an external dependency to Dockerfile and circleci
config

.circleci/config.yml

@@ -1,6 +1,7 @@
 version: 2.1
 
 orbs:
+#  snyk: snyk/snyk@0.0.8
   golang-ci: financial-times/golang-ci@1.0.0
 
 jobs:
@@ -18,16 +19,21 @@ jobs:
           command: |
             GO111MODULE=off go get -u github.com/mattn/goveralls
             GO111MODULE=off go get -u github.com/jstemmer/go-junit-report
+            GO111MODULE=off go get -u github.com/myitcv/gobin
             curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.18.0
             wget https://raw.githubusercontent.com/Financial-Times/upp-coding-standard/v1.0.0/golangci-config/.golangci.yml
+            STATIK_VERSION=$(go list -m all | grep statik | cut -d ' ' -f2)
+            gobin github.com/rakyll/statik@${STATIK_VERSION}
       - run:
           name: Make result folders
           command: |
             mkdir -p ${CIRCLE_TEST_REPORTS}
             mkdir -p ${CIRCLE_COVERAGE_REPORT}
       - run:
           name: Go build
-          command: go build -mod=readonly -v ./cmd/content-rw-elasticsearch
+          command: |
+            go generate ./cmd/content-rw-elasticsearch
+            go build -mod=readonly -v ./cmd/content-rw-elasticsearch
       - run:
           name: Run Linters
           command: golangci-lint run --config=.golangci.yml --new-from-rev=master
@@ -66,15 +72,31 @@ jobs:
             DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs=11.\*
             npm install -g --unsafe-perm --loglevel warn --user 0 --no-progress dredd@11.2.19
             rm -rf /var/lib/apt/lists/*
+            GO111MODULE=off go get -u github.com/myitcv/gobin
+            STATIK_VERSION=$(go list -m all | grep statik | cut -d ' ' -f2)
+            gobin github.com/rakyll/statik@${STATIK_VERSION}
       - run:
           name: Load ersatz-fixtures.yml to ersatz image
           command: "curl -X POST --data-binary @_ft/ersatz-fixtures.yml -H \"Content-type: text/x-yaml\" http://localhost:9000/__configure"
       - run:
           name: Go Build
-          command: go build -mod=readonly -v ./cmd/content-rw-elasticsearch
+          command: |
+            go generate ./cmd/content-rw-elasticsearch
+            go build -mod=readonly -v ./cmd/content-rw-elasticsearch
       - run:
           name: Dredd API Testing
           command: dredd
+  snykscan:
+    working_directory: /go/src/github.com/Financial-Times/content-rw-elasticsearch
+    docker:
+      - image: circleci/golang:1
+    steps:
+      - checkout
+      - snyk/scan:
+          monitor-on-build: false
+          severity-threshold: medium
+          fail-on-issues: true
+
 workflows:
   version: 2.1
   test-and-build-docker:
@@ -86,6 +108,16 @@ workflows:
             - build
   scanning:
     jobs:
+      - build
       - golang-ci/scan:
           name: scan-dependencies
-          context: org-global
+          context: cm-team-snyk
+          requires:
+            - build
+#  scanning:
+#    jobs:
+#      - build
+#      - snykscan:
+#          context: cm-team-snyk
+#          requires:
+#            - build

.gitignore

@@ -19,3 +19,4 @@ vendor/
 .idea
 
 /content-rw-elasticsearch
+statik

Dockerfile

@@ -16,6 +16,8 @@ RUN VERSION="version=$(git describe --tag --always 2> /dev/null)" \
   && REVISION="revision=$(git rev-parse HEAD)" \
   && BUILDER="builder=$(go version)" \
   && LDFLAGS="-X '"${BUILDINFO_PACKAGE}$VERSION"' -X '"${BUILDINFO_PACKAGE}$DATETIME"' -X '"${BUILDINFO_PACKAGE}$REPOSITORY"' -X '"${BUILDINFO_PACKAGE}$REVISION"' -X '"${BUILDINFO_PACKAGE}$BUILDER"'" \
+  && STATIK_VERSION="$(go list -m all | grep statik | cut -d ' ' -f2)" \
+  && go get github.com/rakyll/statik@${STATIK_VERSION} \
   && go generate ./cmd/${PROJECT} \
   && CGO_ENABLED=0 go build -mod=readonly -a -o /artifacts/${PROJECT}/${PROJECT} -ldflags="${LDFLAGS}" ./cmd/${PROJECT} \
   && echo "Build flags: ${LDFLAGS}"

Makefile

@@ -1,8 +1,12 @@
 PROJECT_NAME=content-rw-elasticsearch
+STATIK_VERSION=$(shell go list -m all | grep statik | cut -d ' ' -f2)
 .PHONY: all test clean
 
 all: clean test build-readonly
 
+install:
+	go get github.com/rakyll/statik@${STATIK_VERSION}
+
 build:
 	@echo ">>> Embedding static resources in binary..."
 	go generate ./cmd/${PROJECT_NAME}

cmd/content-rw-elasticsearch/main.go

@@ -8,14 +8,14 @@ import (
 	"sync"
 	"time"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/concept"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/config"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/es"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/health"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/http"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/mapper"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/message"
-	_ "github.com/Financial-Times/content-rw-elasticsearch/statik"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/es"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/health"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/http"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/mapper"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/message"
+	_ "github.com/Financial-Times/content-rw-elasticsearch/v2/statik"
 	"github.com/Financial-Times/go-logger/v2"
 	"github.com/Financial-Times/message-queue-gonsumer/consumer"
 	"github.com/jawher/mow.cli"

go.mod

@@ -1,6 +1,6 @@
-module github.com/Financial-Times/content-rw-elasticsearch
+module github.com/Financial-Times/content-rw-elasticsearch/v2
 
-go 1.13
+go 1.14
 
 require (
 	github.com/Financial-Times/go-fthealth v0.0.0-20180807113633-3d8eb430d5b5
@@ -15,7 +15,7 @@ require (
 	github.com/hashicorp/go-version v1.0.0 // indirect
 	github.com/jawher/mow.cli v1.0.4
 	github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c
-	github.com/rakyll/statik v0.1.6
+	github.com/rakyll/statik v0.1.7
 	github.com/smartystreets/go-aws-auth v0.0.0-20170504205021-8ef1316913ee
 	github.com/smartystreets/gunit v1.1.3 // indirect
 	github.com/spf13/viper v1.6.2

go.sum

@@ -1,6 +1,7 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Financial-Times/content-rw-elasticsearch v0.0.0-20191206140835-19cc9dd188b4 h1:m5D85GknuQBQ1zUOmIR6t1oWP2Q3lNWyLXXLNcTQ9qk=
 github.com/Financial-Times/go-fthealth v0.0.0-20180807113633-3d8eb430d5b5 h1:XH5h45aAyG1bAFBYmkgJkT4q13CbkCJ+gj9+rIfzuL8=
 github.com/Financial-Times/go-fthealth v0.0.0-20180807113633-3d8eb430d5b5/go.mod h1:gpAzq6W5rCheYlY32JOIxS/VjVcYHbC2PkMzQngHT9c=
 github.com/Financial-Times/go-logger/v2 v2.0.1 h1:iekEfSsUtlkg+YkXTZo+/fIN2VbZ2/3Hl9yolP3z5X8=
@@ -109,6 +110,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rakyll/statik v0.1.6 h1:uICcfUXpgqtw2VopbIncslhAmE5hwc4g20TEyEENBNs=
 github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
+github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=

pkg/concept/concordance_test.go

@@ -142,7 +142,6 @@ func TestConcordanceApiService_GetConceptsErrorOnNewRequest(t *testing.T) {
 	concepts, err := concordanceAPIService.GetConcepts("tid_test", []string{sampleID})
 
 	expect.Error(err)
-	expect.Equal("parse ://concordances: missing protocol scheme", err.Error())
 	expect.Nil(concepts)
 }
 
@@ -241,7 +240,6 @@ func TestConcordanceApiService_CheckHealthErrorOnNewRequest(t *testing.T) {
 	check, err := concordanceAPIService.HealthCheck()
 	expect.Error(err)
 	expect.Empty(check)
-	expect.Equal("parse ://__gtg: missing protocol scheme", err.Error())
 }
 
 func TestConcordanceApiService_CheckHealthErrorOnRequestDo(t *testing.T) {

pkg/config/config.go

@@ -6,9 +6,9 @@ import (
 	"io/ioutil"
 	"strings"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/schema"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
 	// This blank import is required in order to read the embedded config files
-	_ "github.com/Financial-Times/content-rw-elasticsearch/statik"
+	_ "github.com/Financial-Times/content-rw-elasticsearch/v2/statik"
 	"github.com/rakyll/statik/fs"
 	"github.com/spf13/viper"
 )

pkg/es/service.go

@@ -7,7 +7,7 @@ import (
 	"reflect"
 	"sync"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
 
 	"gopkg.in/olivere/elastic.v2"
 )
@@ -19,7 +19,7 @@ type elasticIndex struct {
 }
 
 type ElasticsearchService struct {
-	sync.RWMutex
+	mu            sync.RWMutex
 	ElasticClient Client
 	IndexName     string
 }
@@ -95,9 +95,14 @@ func (s *ElasticsearchService) GetSchemaHealth() (string, error) {
 	return "ok", nil
 }
 
+func (s *ElasticsearchService) GetClient() Client {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.ElasticClient
+}
 func (s *ElasticsearchService) SetClient(client Client) {
-	s.Lock()
-	defer s.Unlock()
+	s.mu.Lock()
+	defer s.mu.Unlock()
 	s.ElasticClient = client
 }
 

pkg/health/healthcheck.go

@@ -7,8 +7,8 @@ import (
 
 	status "github.com/Financial-Times/service-status-go/httphandlers"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/concept"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/es"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/es"
 	fthealth "github.com/Financial-Times/go-fthealth/v1_1"
 	"github.com/Financial-Times/go-logger/v2"
 	"github.com/Financial-Times/message-queue-gonsumer/consumer"

pkg/http/server.go

@@ -18,7 +18,7 @@ func StartServer(log *logger.UPPLogger, serveMux *http.ServeMux, port string) {
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
-		if err := server.ListenAndServe(); err != nil {
+		if err := server.ListenAndServe(); err != http.ErrServerClosed {
 			log.WithError(err).Error("HTTP server is closing")
 		}
 		wg.Done()

pkg/mapper/mapper.go

@@ -6,13 +6,13 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/config"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/schema"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
 
 	"fmt"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/concept"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/html"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/html"
 	"github.com/Financial-Times/go-logger/v2"
 	"github.com/Financial-Times/uuid-utils-go"
 )

pkg/mapper/mapper_test.go

@@ -5,12 +5,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/config"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/schema"
-	tst "github.com/Financial-Times/content-rw-elasticsearch/test"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
+	tst "github.com/Financial-Times/content-rw-elasticsearch/v2/test"
 	"github.com/Financial-Times/go-logger/v2"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/concept"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"

pkg/message/message_handler.go

@@ -7,11 +7,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/config"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/mapper"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/schema"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/mapper"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/es"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/es"
 	"github.com/Financial-Times/go-logger/v2"
 	"github.com/Financial-Times/message-queue-gonsumer/consumer"
 	"github.com/dchest/uniuri"
@@ -40,7 +40,6 @@ type Handler struct {
 	httpClient      *http.Client
 	esClient        ESClient
 	wg              *sync.WaitGroup
-	mu              sync.Mutex
 	log             *logger.UPPLogger
 }
 
@@ -67,30 +66,21 @@ func (h *Handler) Start(baseAPIURL string, accessConfig es.AccessConfig) {
 		}
 	}()
 
+	h.wg.Add(1)
 	go func() {
 		defer h.wg.Done()
 		for ec := range channel {
-			h.mu.Lock()
-			h.wg.Add(1)
-			h.mu.Unlock()
 			h.esService.SetClient(ec)
-			h.startMessageConsumer()
+			// this is a blocking method
+			h.messageConsumer.Start()
 		}
 	}()
 }
 
 func (h *Handler) Stop() {
-	h.mu.Lock()
 	if h.messageConsumer != nil {
 		h.messageConsumer.Stop()
 	}
-	h.mu.Unlock()
-
-}
-
-func (h *Handler) startMessageConsumer() {
-	// this is a blocking method
-	h.messageConsumer.Start()
 }
 
 func (h *Handler) handleMessage(msg consumer.Message) {

pkg/message/message_handler_test.go

@@ -9,15 +9,15 @@ import (
 	"testing"
 	"time"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/config"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/mapper"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/schema"
-	tst "github.com/Financial-Times/content-rw-elasticsearch/test"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/mapper"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
+	tst "github.com/Financial-Times/content-rw-elasticsearch/v2/test"
 	"github.com/Financial-Times/go-logger/v2"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/concept"
-	"github.com/Financial-Times/content-rw-elasticsearch/pkg/es"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/es"
 	"github.com/Financial-Times/message-queue-gonsumer/consumer"
 	"github.com/stretchr/testify/mock"
 	"gopkg.in/olivere/elastic.v2"
@@ -165,9 +165,7 @@ func TestStartClient(t *testing.T) {
 
 	expect.NotNil(handler.esService, "Elastic Service should be initialized")
 	expect.Equal("index", handler.esService.(*es.ElasticsearchService).IndexName, "Wrong index")
-	handler.esService.(*es.ElasticsearchService).Lock()
-	expect.NotNil(handler.esService.(*es.ElasticsearchService).ElasticClient, "Elastic client should be initialized")
-	handler.esService.(*es.ElasticsearchService).Unlock()
+	expect.NotNil(handler.esService.(*es.ElasticsearchService).GetClient(), "Elastic client should be initialized")
 }
 func TestStartClientError(t *testing.T) {
 	expect := assert.New(t)