Remove statik in favour of adding files to docker image

.circleci/config.yml

@@ -20,11 +20,8 @@ jobs:
             git config --global url."https://$GITHUB_USERNAME:$GITHUB_TOKEN@github.com/".insteadOf "https://github.com/"
             GO111MODULE=off go get -u github.com/mattn/goveralls
             GO111MODULE=off go get -u github.com/jstemmer/go-junit-report
-            GO111MODULE=off go get -u github.com/myitcv/gobin
             curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.18.0
             wget https://raw.githubusercontent.com/Financial-Times/upp-coding-standard/v1.0.0/golangci-config/.golangci.yml
-            STATIK_VERSION=$(go list -mod=readonly -m all | grep statik | cut -d ' ' -f2)
-            gobin github.com/rakyll/statik@${STATIK_VERSION}
       - run:
           name: Make result folders
           command: |
@@ -35,7 +32,6 @@ jobs:
           command: |
             GOPRIVATE="github.com/Financial-Times"
             git config --global url."https://$GITHUB_USERNAME:$GITHUB_TOKEN@github.com/".insteadOf "https://github.com/"
-            go generate ./cmd/content-rw-elasticsearch
             go build -mod=readonly -v ./cmd/content-rw-elasticsearch
       - run:
           name: Run Linters
@@ -45,7 +41,7 @@ jobs:
           command: go test -race -mod=readonly -cover -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out ./... | go-junit-report > ${CIRCLE_TEST_REPORTS}/junit.xml
       - run:
           name: Upload coverage
-          command: goveralls -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out -service=circle-ci -repotoken=${COVERALLS_TOKEN}
+          command: ls -las ${CIRCLE_COVERAGE_REPORT} && goveralls -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out -service=circle-ci -repotoken=${COVERALLS_TOKEN}
       - store_test_results:
           path: /tmp/test-results
   dockerfile:
@@ -79,9 +75,6 @@ jobs:
             DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs=11.\*
             npm install -g --unsafe-perm --loglevel warn --user 0 --no-progress dredd@11.2.19
             rm -rf /var/lib/apt/lists/*
-            GO111MODULE=off go get -u github.com/myitcv/gobin
-            STATIK_VERSION=$(go list -mod=readonly -m all | grep statik | cut -d ' ' -f2)
-            gobin github.com/rakyll/statik@${STATIK_VERSION}
       - run:
           name: Load ersatz-fixtures.yml to ersatz image
           command: "curl -X POST --data-binary @_ft/ersatz-fixtures.yml -H \"Content-type: text/x-yaml\" http://localhost:9000/__configure"
@@ -90,7 +83,6 @@ jobs:
           command: |
             GOPRIVATE="github.com/Financial-Times"
             git config --global url."https://$GITHUB_USERNAME:$GITHUB_TOKEN@github.com/".insteadOf "https://github.com/"
-            go generate ./cmd/content-rw-elasticsearch
             go build -mod=readonly -v ./cmd/content-rw-elasticsearch
       - run:
           name: Dredd API Testing

Dockerfile

@@ -12,23 +12,17 @@ ARG GITHUB_TOKEN
 COPY . ${SRC_FOLDER}
 WORKDIR ${SRC_FOLDER}
 
-# Install statik cli tool in GOPATH in order to successfully execute the go generate command
-RUN echo "machine github.com login $GITHUB_USERNAME password $GITHUB_TOKEN" > ~/.netrc \
-  && GOPRIVATE="github.com/Financial-Times" \
-  && GO111MODULE=off go get -u github.com/myitcv/gobin \
-  # Get statik version from go.mod of the project
-  && STATIK_VERSION="$(go list -mod=readonly -m all | grep statik | cut -d ' ' -f2)" \
-  && gobin github.com/rakyll/statik@${STATIK_VERSION} \
-  && go generate ./cmd/${PROJECT}
-
 # Build app
-RUN VERSION="version=$(git describe --tag --always 2> /dev/null)" \
+RUN echo "machine github.com login $GITHUB_USERNAME password $GITHUB_TOKEN" > ~/.netrc \
+  && VERSION="version=$(git describe --tag --always 2> /dev/null)" \
   && DATETIME="dateTime=$(date -u +%Y%m%d%H%M%S)" \
   && REPOSITORY="repository=$(git config --get remote.origin.url)" \
   && REVISION="revision=$(git rev-parse HEAD)" \
   && BUILDER="builder=$(go version)" \
   && LDFLAGS="-X '"${BUILDINFO_PACKAGE}$VERSION"' -X '"${BUILDINFO_PACKAGE}$DATETIME"' -X '"${BUILDINFO_PACKAGE}$REPOSITORY"' -X '"${BUILDINFO_PACKAGE}$REVISION"' -X '"${BUILDINFO_PACKAGE}$BUILDER"'" \
   && CGO_ENABLED=0 go build -mod=readonly -a -o /artifacts/${PROJECT}/${PROJECT} -ldflags="${LDFLAGS}" ./cmd/${PROJECT} \
+  && mkdir -p /artifacts/configs/ \
+  && cp -r ${SRC_FOLDER}/configs /artifacts/configs \
   && echo "Build flags: ${LDFLAGS}"
 
 # Multi-stage build - copy only the certs and the binary into the image

Makefile

@@ -1,18 +1,9 @@
 PROJECT_NAME=content-rw-elasticsearch
-STATIK_VERSION=$(shell go list -m all | grep statik | cut -d ' ' -f2)
 .PHONY: all test clean
 
 all: clean build-readonly test
 
-install:
-	GO111MODULE="off" go get -u github.com/myitcv/gobin
-	gobin github.com/rakyll/statik@${STATIK_VERSION}
-
-generate:
-	@echo ">>> Embedding static resources in binary..."
-	go generate ./cmd/${PROJECT_NAME}
-
-build: generate
+build: 
 	@echo ">>> Building Application..."
 	go build -v ./cmd/${PROJECT_NAME}
 

README.md

@@ -8,27 +8,10 @@ Indexes V2 content in Elasticsearch for use by SAPI V1
 
 ## Project Local Execution
 
-Before executing any of the proposed ways run
-
-```sh
-make install
-```
-
-to install external tools needed and then
-
 ```sh
-make all
+make build
 ```
 
-to run tests and a clean build of the project.
-
----
-
-**NOTE**
-
-Each time you modify any file under the `configs` directory, please run `make generate` in order to regenerate
-the `statik` package which embeds the files in the binary.
-
 ---
 
 ### Docker Compose

cmd/content-rw-elasticsearch/main.go

@@ -1,5 +1,3 @@
-//go:generate statik -src=../../configs -dest ../../
-
 package main
 
 import (

go.mod

@@ -15,11 +15,12 @@ require (
 	github.com/hashicorp/go-version v1.0.0 // indirect
 	github.com/jawher/mow.cli v1.0.4
 	github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c
-	github.com/rakyll/statik v0.1.7
 	github.com/smartystreets/go-aws-auth v0.0.0-20170504205021-8ef1316913ee
 	github.com/smartystreets/gunit v1.1.3 // indirect
 	github.com/spf13/viper v1.6.2
 	github.com/stretchr/testify v1.4.0
+	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
+	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect
 	golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e // indirect
 	gopkg.in/olivere/elastic.v2 v2.0.61
 )

go.sum

@@ -109,8 +109,6 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
-github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
@@ -159,15 +157,20 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -179,6 +182,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=

pkg/config/config.go

@@ -4,12 +4,12 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
-	// This blank import is required in order to read the embedded config files
-	_ "github.com/Financial-Times/content-rw-elasticsearch/v2/statik"
-	"github.com/rakyll/statik/fs"
 	"github.com/spf13/viper"
 )
 
@@ -56,7 +56,8 @@ type AppConfig struct {
 }
 
 func ParseConfig(configFileName string) (AppConfig, error) {
-	contents, err := ReadEmbeddedResource(configFileName)
+
+	contents, err := ReadConfigFile(configFileName)
 	if err != nil {
 		return AppConfig{}, err
 	}
@@ -89,21 +90,16 @@ func ParseConfig(configFileName string) (AppConfig, error) {
 	}, nil
 }
 
-func ReadEmbeddedResource(fileName string) ([]byte, error) {
-	statikFS, err := fs.New()
+func ReadConfigFile(fileName string) ([]byte, error) {
+	path, err := filepath.Abs(fmt.Sprintf("./configs/%s", fileName))
 	if err != nil {
-		return nil, err
+		log.Fatal(err)
 	}
-	// Access individual files by their paths.
-	f, err := statikFS.Open("/" + fileName)
+	file, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		log.Fatal(err)
 	}
-	defer f.Close()
+	defer file.Close()
 
-	contents, err := ioutil.ReadAll(f)
-	if err != nil {
-		return nil, err
-	}
-	return contents, err
+	return ioutil.ReadAll(file)
 }

pkg/es/service.go

@@ -8,7 +8,6 @@ import (
 	"sync"
 
 	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
-
 	"gopkg.in/olivere/elastic.v2"
 )
 
@@ -52,7 +51,7 @@ func (s *ElasticsearchService) GetSchemaHealth() (string, error) {
 	if referenceIndex == nil {
 		referenceIndex = new(elasticIndex)
 
-		referenceJSON, err := config.ReadEmbeddedResource("referenceSchema.json")
+		referenceJSON, err := config.ReadConfigFile("referenceSchema.json")
 		if err != nil {
 			return "", err
 		}

pkg/mapper/mapper_test.go

@@ -115,7 +115,7 @@ func TestConvertToESContentModel(t *testing.T) {
 	}
 
 	log := logger.NewUPPLogger(config.AppName, config.AppDefaultLogLevel)
-	appConfig, err := config.ParseConfig("app.yml")
+	appConfig, err := config.ParseConfig("../../../configs/app.yml")
 	if err != nil {
 		log.Fatal(err)
 	}

pkg/message/message_handler_test.go

@@ -151,7 +151,7 @@ func mockMapperHandler(concordanceAPIMock *concordanceAPIMock, log *logger.UPPLo
 }
 
 func initAppConfig() config.AppConfig {
-	appConfig, err := config.ParseConfig("app.yml")
+	appConfig, err := config.ParseConfig("../../../configs/app.yml")
 	if err != nil {
 		log.Fatal(err)
 	}