From 571372ab88699026304f75b2de260276d924804d Mon Sep 17 00:00:00 2001 From: Hristo Georgiev Date: Mon, 20 Jul 2020 18:31:40 +0300 Subject: [PATCH] Remove statik in favour of adding files to docker image --- .circleci/config.yml | 10 +--------- Dockerfile | 14 ++++---------- Makefile | 11 +---------- README.md | 19 +------------------ cmd/content-rw-elasticsearch/main.go | 2 -- go.mod | 3 ++- go.sum | 8 ++++++-- pkg/config/config.go | 28 ++++++++++++---------------- pkg/es/service.go | 3 +-- pkg/mapper/mapper_test.go | 2 +- pkg/message/message_handler_test.go | 2 +- 11 files changed, 30 insertions(+), 72 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 72dcd33..5351ca1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,11 +20,8 @@ jobs: git config --global url."https://$GITHUB_USERNAME:$GITHUB_TOKEN@github.com/".insteadOf "https://github.com/" GO111MODULE=off go get -u github.com/mattn/goveralls GO111MODULE=off go get -u github.com/jstemmer/go-junit-report - GO111MODULE=off go get -u github.com/myitcv/gobin curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.18.0 wget https://raw.githubusercontent.com/Financial-Times/upp-coding-standard/v1.0.0/golangci-config/.golangci.yml - STATIK_VERSION=$(go list -mod=readonly -m all | grep statik | cut -d ' ' -f2) - gobin github.com/rakyll/statik@${STATIK_VERSION} - run: name: Make result folders command: | @@ -35,7 +32,6 @@ jobs: command: | GOPRIVATE="github.com/Financial-Times" git config --global url."https://$GITHUB_USERNAME:$GITHUB_TOKEN@github.com/".insteadOf "https://github.com/" - go generate ./cmd/content-rw-elasticsearch go build -mod=readonly -v ./cmd/content-rw-elasticsearch - run: name: Run Linters @@ -45,7 +41,7 @@ jobs: command: go test -race -mod=readonly -cover -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out ./... | go-junit-report > ${CIRCLE_TEST_REPORTS}/junit.xml - run: name: Upload coverage - command: goveralls -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out -service=circle-ci -repotoken=${COVERALLS_TOKEN} + command: ls -las ${CIRCLE_COVERAGE_REPORT} && goveralls -coverprofile=${CIRCLE_COVERAGE_REPORT}/coverage.out -service=circle-ci -repotoken=${COVERALLS_TOKEN} - store_test_results: path: /tmp/test-results dockerfile: @@ -79,9 +75,6 @@ jobs: DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs=11.\* npm install -g --unsafe-perm --loglevel warn --user 0 --no-progress dredd@11.2.19 rm -rf /var/lib/apt/lists/* - GO111MODULE=off go get -u github.com/myitcv/gobin - STATIK_VERSION=$(go list -mod=readonly -m all | grep statik | cut -d ' ' -f2) - gobin github.com/rakyll/statik@${STATIK_VERSION} - run: name: Load ersatz-fixtures.yml to ersatz image command: "curl -X POST --data-binary @_ft/ersatz-fixtures.yml -H \"Content-type: text/x-yaml\" http://localhost:9000/__configure" @@ -90,7 +83,6 @@ jobs: command: | GOPRIVATE="github.com/Financial-Times" git config --global url."https://$GITHUB_USERNAME:$GITHUB_TOKEN@github.com/".insteadOf "https://github.com/" - go generate ./cmd/content-rw-elasticsearch go build -mod=readonly -v ./cmd/content-rw-elasticsearch - run: name: Dredd API Testing diff --git a/Dockerfile b/Dockerfile index 3b23328..15ee3c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,23 +12,17 @@ ARG GITHUB_TOKEN COPY . ${SRC_FOLDER} WORKDIR ${SRC_FOLDER} -# Install statik cli tool in GOPATH in order to successfully execute the go generate command -RUN echo "machine github.com login $GITHUB_USERNAME password $GITHUB_TOKEN" > ~/.netrc \ - && GOPRIVATE="github.com/Financial-Times" \ - && GO111MODULE=off go get -u github.com/myitcv/gobin \ - # Get statik version from go.mod of the project - && STATIK_VERSION="$(go list -mod=readonly -m all | grep statik | cut -d ' ' -f2)" \ - && gobin github.com/rakyll/statik@${STATIK_VERSION} \ - && go generate ./cmd/${PROJECT} - # Build app -RUN VERSION="version=$(git describe --tag --always 2> /dev/null)" \ +RUN echo "machine github.com login $GITHUB_USERNAME password $GITHUB_TOKEN" > ~/.netrc \ + && VERSION="version=$(git describe --tag --always 2> /dev/null)" \ && DATETIME="dateTime=$(date -u +%Y%m%d%H%M%S)" \ && REPOSITORY="repository=$(git config --get remote.origin.url)" \ && REVISION="revision=$(git rev-parse HEAD)" \ && BUILDER="builder=$(go version)" \ && LDFLAGS="-X '"${BUILDINFO_PACKAGE}$VERSION"' -X '"${BUILDINFO_PACKAGE}$DATETIME"' -X '"${BUILDINFO_PACKAGE}$REPOSITORY"' -X '"${BUILDINFO_PACKAGE}$REVISION"' -X '"${BUILDINFO_PACKAGE}$BUILDER"'" \ && CGO_ENABLED=0 go build -mod=readonly -a -o /artifacts/${PROJECT}/${PROJECT} -ldflags="${LDFLAGS}" ./cmd/${PROJECT} \ + && mkdir -p /artifacts/configs/ \ + && cp -r ${SRC_FOLDER}/configs /artifacts/configs \ && echo "Build flags: ${LDFLAGS}" # Multi-stage build - copy only the certs and the binary into the image diff --git a/Makefile b/Makefile index 3bff5a0..e87b031 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,9 @@ PROJECT_NAME=content-rw-elasticsearch -STATIK_VERSION=$(shell go list -m all | grep statik | cut -d ' ' -f2) .PHONY: all test clean all: clean build-readonly test -install: - GO111MODULE="off" go get -u github.com/myitcv/gobin - gobin github.com/rakyll/statik@${STATIK_VERSION} - -generate: - @echo ">>> Embedding static resources in binary..." - go generate ./cmd/${PROJECT_NAME} - -build: generate +build: @echo ">>> Building Application..." go build -v ./cmd/${PROJECT_NAME} diff --git a/README.md b/README.md index 486cd67..40cacec 100644 --- a/README.md +++ b/README.md @@ -8,27 +8,10 @@ Indexes V2 content in Elasticsearch for use by SAPI V1 ## Project Local Execution -Before executing any of the proposed ways run - -```sh -make install -``` - -to install external tools needed and then - ```sh -make all +make build ``` -to run tests and a clean build of the project. - ---- - -**NOTE** - -Each time you modify any file under the `configs` directory, please run `make generate` in order to regenerate -the `statik` package which embeds the files in the binary. - --- ### Docker Compose diff --git a/cmd/content-rw-elasticsearch/main.go b/cmd/content-rw-elasticsearch/main.go index 91357df..b11d464 100644 --- a/cmd/content-rw-elasticsearch/main.go +++ b/cmd/content-rw-elasticsearch/main.go @@ -1,5 +1,3 @@ -//go:generate statik -src=../../configs -dest ../../ - package main import ( diff --git a/go.mod b/go.mod index 60f5ac8..876cc82 100644 --- a/go.mod +++ b/go.mod @@ -15,11 +15,12 @@ require ( github.com/hashicorp/go-version v1.0.0 // indirect github.com/jawher/mow.cli v1.0.4 github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c - github.com/rakyll/statik v0.1.7 github.com/smartystreets/go-aws-auth v0.0.0-20170504205021-8ef1316913ee github.com/smartystreets/gunit v1.1.3 // indirect github.com/spf13/viper v1.6.2 github.com/stretchr/testify v1.4.0 + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect + golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e // indirect gopkg.in/olivere/elastic.v2 v2.0.61 ) diff --git a/go.sum b/go.sum index 1e6db1f..2980e36 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,6 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= -github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= @@ -159,6 +157,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -166,8 +166,11 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -179,6 +182,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= diff --git a/pkg/config/config.go b/pkg/config/config.go index f437a55..6708e1d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -4,12 +4,12 @@ import ( "bytes" "fmt" "io/ioutil" + "log" + "os" + "path/filepath" "strings" "github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema" - // This blank import is required in order to read the embedded config files - _ "github.com/Financial-Times/content-rw-elasticsearch/v2/statik" - "github.com/rakyll/statik/fs" "github.com/spf13/viper" ) @@ -56,7 +56,8 @@ type AppConfig struct { } func ParseConfig(configFileName string) (AppConfig, error) { - contents, err := ReadEmbeddedResource(configFileName) + + contents, err := ReadConfigFile(configFileName) if err != nil { return AppConfig{}, err } @@ -89,21 +90,16 @@ func ParseConfig(configFileName string) (AppConfig, error) { }, nil } -func ReadEmbeddedResource(fileName string) ([]byte, error) { - statikFS, err := fs.New() +func ReadConfigFile(fileName string) ([]byte, error) { + path, err := filepath.Abs(fmt.Sprintf("./configs/%s", fileName)) if err != nil { - return nil, err + log.Fatal(err) } - // Access individual files by their paths. - f, err := statikFS.Open("/" + fileName) + file, err := os.Open(path) if err != nil { - return nil, err + log.Fatal(err) } - defer f.Close() + defer file.Close() - contents, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - return contents, err + return ioutil.ReadAll(file) } diff --git a/pkg/es/service.go b/pkg/es/service.go index 47d826c..674c086 100644 --- a/pkg/es/service.go +++ b/pkg/es/service.go @@ -8,7 +8,6 @@ import ( "sync" "github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config" - "gopkg.in/olivere/elastic.v2" ) @@ -52,7 +51,7 @@ func (s *ElasticsearchService) GetSchemaHealth() (string, error) { if referenceIndex == nil { referenceIndex = new(elasticIndex) - referenceJSON, err := config.ReadEmbeddedResource("referenceSchema.json") + referenceJSON, err := config.ReadConfigFile("referenceSchema.json") if err != nil { return "", err } diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index f123069..041e26e 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -115,7 +115,7 @@ func TestConvertToESContentModel(t *testing.T) { } log := logger.NewUPPLogger(config.AppName, config.AppDefaultLogLevel) - appConfig, err := config.ParseConfig("app.yml") + appConfig, err := config.ParseConfig("../../../configs/app.yml") if err != nil { log.Fatal(err) } diff --git a/pkg/message/message_handler_test.go b/pkg/message/message_handler_test.go index fdfe2a0..bd3291b 100644 --- a/pkg/message/message_handler_test.go +++ b/pkg/message/message_handler_test.go @@ -151,7 +151,7 @@ func mockMapperHandler(concordanceAPIMock *concordanceAPIMock, log *logger.UPPLo } func initAppConfig() config.AppConfig { - appConfig, err := config.ParseConfig("app.yml") + appConfig, err := config.ParseConfig("../../../configs/app.yml") if err != nil { log.Fatal(err) }