Merge pull request #20 from Financial-Times/remove-kafka-topic-check

Removed kafka topic check

.gitignore

@@ -7,6 +7,7 @@
 # Folders
 _obj
 _test
+vendor/*/
 
 # Architecture specific extensions/prefixes
 *.[568vq]

Dockerfile

@@ -1,29 +1,31 @@
-FROM alpine:3.3
+FROM golang:1.8-alpine
 
-ADD *.go .git /concept-ingester/
+ENV PROJECT=concept-ingester
+COPY . /${PROJECT}-sources/
 
-RUN apk add --update bash \
-  && apk --update add git go \
-  && cd concept-ingester\
-  && git fetch origin 'refs/tags/*:refs/tags/*' \
-  && BUILDINFO_PACKAGE="github.com/Financial-Times/service-status-go/buildinfo." \
+RUN apk --no-cache --virtual .build-dependencies add git \
+  && ORG_PATH="github.com/Financial-Times" \
+  && REPO_PATH="${ORG_PATH}/${PROJECT}" \
+  && mkdir -p $GOPATH/src/${ORG_PATH} \
+  # Linking the project sources in the GOPATH folder
+  && ln -s /${PROJECT}-sources $GOPATH/src/${REPO_PATH} \
+  && cd $GOPATH/src/${REPO_PATH} \
+  && BUILDINFO_PACKAGE="${ORG_PATH}/${PROJECT}/vendor/${ORG_PATH}/service-status-go/buildinfo." \
   && VERSION="version=$(git describe --tag --always 2> /dev/null)" \
   && DATETIME="dateTime=$(date -u +%Y%m%d%H%M%S)" \
   && REPOSITORY="repository=$(git config --get remote.origin.url)" \
   && REVISION="revision=$(git rev-parse HEAD)" \
   && BUILDER="builder=$(go version)" \
   && LDFLAGS="-X '"${BUILDINFO_PACKAGE}$VERSION"' -X '"${BUILDINFO_PACKAGE}$DATETIME"' -X '"${BUILDINFO_PACKAGE}$REPOSITORY"' -X '"${BUILDINFO_PACKAGE}$REVISION"' -X '"${BUILDINFO_PACKAGE}$BUILDER"'" \
-  && cd .. \
-  && export GOPATH=/gopath \
-  && REPO_PATH="github.com/Financial-Times/concept-ingester" \
-  && mkdir -p $GOPATH/src/${REPO_PATH} \
-  && cp -r concept-ingester/* $GOPATH/src/${REPO_PATH} \
-  && cd $GOPATH/src/${REPO_PATH} \
-  && go get ./... \
-  && cd $GOPATH/src/${REPO_PATH} \
-  && echo ${LDFLAGS} \
+  && echo "Build flags: $LDFLAGS" \
+  && echo "Fetching dependencies..." \
+  && go get -u github.com/kardianos/govendor \
+  && $GOPATH/bin/govendor sync \
   && go build -ldflags="${LDFLAGS}" \
-  && mv concept-ingester /app \
-  && apk del go git \
+  && mv ${PROJECT} /${PROJECT} \
+  && apk del .build-dependencies \
   && rm -rf $GOPATH /var/cache/apk/*
-CMD [ "/app" ]
+
+WORKDIR /
+
+CMD [ "/concept-ingester" ]

circle.yml

@@ -1,16 +1,30 @@
+machine:
+  environment:
+    PROJECT_GOPATH: "${HOME}/.go_project"
+    PROJECT_PARENT_PATH: "${PROJECT_GOPATH}/src/github.com/${CIRCLE_PROJECT_USERNAME}"
+    PROJECT_PATH: "${PROJECT_PARENT_PATH}/${CIRCLE_PROJECT_REPONAME}"
+    GOPATH: "${HOME}/.go_workspace:${PROJECT_GOPATH}"
+
+checkout:
+  post:
+    - mkdir -p "${PROJECT_PARENT_PATH}"
+    - ln -sf "${HOME}/${CIRCLE_PROJECT_REPONAME}/" "${PROJECT_PATH}"
+
 dependencies:
   pre:
-    - go get github.com/axw/gocov/gocov; go get github.com/matm/gocov-html; go get -u github.com/jstemmer/go-junit-report
+    - go get -u github.com/kardianos/govendor
+  override:
+    - cd $PROJECT_PATH && govendor sync
+    - cd $PROJECT_PATH && go build -v
+
 test:
   pre:
-    - go get github.com/mattn/goveralls
+    - go get -u github.com/jstemmer/go-junit-report
+    - go get -u github.com/mattn/goveralls
+    - cd $PROJECT_PATH && wget https://raw.githubusercontent.com/Financial-Times/cookiecutter-upp-golang/master/coverage.sh && chmod +x coverage.sh
   override:
     - mkdir -p $CIRCLE_TEST_REPORTS/golang
-    - go test -race -v ./... | go-junit-report > $CIRCLE_TEST_REPORTS/golang/junit.xml
-    - go test -v -cover -race -coverprofile=$CIRCLE_ARTIFACTS/coverage.out ./...
-    - cd $CIRCLE_ARTIFACTS && sed -i '1d' *.out
-    - |
-      echo "mode: atomic" > $CIRCLE_ARTIFACTS/overall-coverage.result
-    - cd $CIRCLE_ARTIFACTS && cat *.out >> overall-coverage.result
+    - cd $PROJECT_PATH && govendor test -race -v +local | go-junit-report > $CIRCLE_TEST_REPORTS/golang/junit.xml
+    - cd $PROJECT_PATH && ./coverage.sh
   post:
-    - goveralls -coverprofile=$CIRCLE_ARTIFACTS/overall-coverage.result -service=circle-ci -repotoken=$COVERALLS_TOKEN
+    - goveralls -coverprofile=$CIRCLE_ARTIFACTS/coverage.out -service=circle-ci -repotoken=$COVERALLS_TOKEN

healthcheck.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+
+	fthealth "github.com/Financial-Times/go-fthealth/v1_1"
+	"github.com/Financial-Times/message-queue-gonsumer/consumer"
+	"github.com/Financial-Times/service-status-go/gtg"
+)
+
+type HealthCheck struct {
+	baseURLs          []string
+	elasticsearchConf *ElasticsearchWriterConfig
+	consumer          consumer.MessageConsumer
+	client            *http.Client
+}
+
+type ElasticsearchWriterConfig struct {
+	elasticsearchWriterUrl     string
+	includeElasticsearchWriter bool
+}
+
+func NewHealthCheck(c consumer.MessageConsumer, baseURLs []string, elasticsearchConf *ElasticsearchWriterConfig, client *http.Client) *HealthCheck {
+	return &HealthCheck{
+		consumer:          c,
+		baseURLs:          baseURLs,
+		elasticsearchConf: elasticsearchConf,
+		client:            client,
+	}
+}
+
+func (h *HealthCheck) Health() func(w http.ResponseWriter, r *http.Request) {
+	checks := []fthealth.Check{h.queueHealthCheck(), h.writerHealthCheck()}
+	if h.elasticsearchConf.includeElasticsearchWriter {
+		checks = append(checks, h.elasticHealthCheck())
+	}
+	hc := fthealth.HealthCheck{
+		SystemCode:  "concept-ingester",
+		Name:        "Concept Ingester",
+		Description: "Consumes Concept instances and forwards them to respective database writers for each concept type.",
+		Checks:      checks,
+	}
+	return fthealth.Handler(hc)
+}
+
+func (h *HealthCheck) queueHealthCheck() fthealth.Check {
+	return fthealth.Check{
+		Name:             "Message Queue Proxy Reachable",
+		Severity:         1,
+		BusinessImpact:   "Unable to connect to kafka proxy.",
+		TechnicalSummary: "Cannot connect to kafka-proxy. If this check fails, check that cluster is up and running, proxy is healthy.",
+		PanicGuide:       "https://dewey.ft.com/kafka-proxy.html",
+		Checker:          h.consumer.ConnectivityCheck,
+	}
+}
+
+func (h *HealthCheck) writerHealthCheck() fthealth.Check {
+	return fthealth.Check{
+		BusinessImpact:   "Unable to connect to one or more configured writers",
+		Name:             "Check connectivity to writers",
+		PanicGuide:       "https://dewey.ft.com/concept-ingester.html",
+		Severity:         1,
+		TechnicalSummary: `Cannot connect to one or more configured writers. If this check fails, check that cluster is up and running and each configured writer returns a healthy gtg`,
+		Checker:          h.checkCanConnectToWriters,
+	}
+}
+
+func (h *HealthCheck) elasticHealthCheck() fthealth.Check {
+	return fthealth.Check{
+		BusinessImpact:   "Unable to connect to  elasticsearch concept writer",
+		Name:             "Check connectivity to concept-rw-elasticsearch",
+		PanicGuide:       "https://dewey.ft.com/up-crwes.html",
+		Severity:         1,
+		TechnicalSummary: `Cannot connect to elasticsearch concept writer. If this check fails, check that the configured writer returns a healthy gtg`,
+		Checker:          h.checkCanConnectToElasticsearchWriter,
+	}
+}
+
+func (h *HealthCheck) GTG() gtg.Status {
+	checks := make([]gtg.StatusChecker, 0)
+
+	consumerCheck := func() gtg.Status {
+		return gtgCheck(h.consumer.ConnectivityCheck)
+	}
+	writersCheck := func() gtg.Status {
+		return gtgCheck(h.checkCanConnectToWriters)
+	}
+
+	checks = append(checks, consumerCheck, writersCheck)
+
+	if h.elasticsearchConf.includeElasticsearchWriter {
+		elasticsearchCheck := func() gtg.Status {
+			return gtgCheck(h.checkCanConnectToElasticsearchWriter)
+		}
+		checks = append(checks, elasticsearchCheck)
+	}
+
+	return gtg.FailFastParallelCheck(checks)()
+}
+
+func (h *HealthCheck) checkCanConnectToWriters() (string, error) {
+	err := h.checkWritersAvailability()
+	if err != nil {
+		return fmt.Sprintf("Healthcheck: Writer not available: %v", err.Error()), err
+	}
+	return "", nil
+}
+
+func (h *HealthCheck) checkCanConnectToElasticsearchWriter() (string, error) {
+	err := h.checkWriterAvailability(h.elasticsearchConf.elasticsearchWriterUrl)
+	if err != nil {
+		return fmt.Sprintf("Healthcheck: Elasticsearch Concept Writer not available: %v", err.Error()), err
+	}
+	return "", nil
+}
+
+func (h *HealthCheck) checkWritersAvailability() error {
+	for _, baseURL := range h.baseURLs {
+		err := h.checkWriterAvailability(baseURL)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (h *HealthCheck) checkWriterAvailability(baseURL string) error {
+	resp, err := h.client.Get(baseURL + "/__gtg")
+	if err != nil {
+		return fmt.Errorf("Error calling writer at %s : %v", baseURL+"/__gtg", err)
+	}
+	resp.Body.Close()
+	if resp != nil && resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("Writer %v returned status %d", baseURL+"/__gtg", resp.StatusCode)
+	}
+	return nil
+}
+
+func gtgCheck(handler func() (string, error)) gtg.Status {
+	if _, err := handler(); err != nil {
+		return gtg.Status{GoodToGo: false, Message: err.Error()}
+	}
+	return gtg.Status{GoodToGo: true}
+}