Merge branch 'release/0.24.0'

.gitignore

@@ -10,4 +10,6 @@ dp-search-data-extractor
 # Intellij
 .idea/
 *.iml
-*.iws
+*.iws
+
+.go/

.golangci.yml

@@ -5,7 +5,7 @@ run:
   concurrency: 4
 
   # timeout for analysis, e.g. 30s, 5m, default is 1m
-  timeout: 5m
+  timeout: 10m
 linters-settings:
   govet:
     check-shadowing: true
@@ -48,7 +48,6 @@ linters:
   # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
   disable-all: true
   enable:
-    - deadcode
     - depguard
     - dogsled
     - errcheck
@@ -65,12 +64,10 @@ linters:
     - ineffassign
     - nakedret
     - staticcheck
-    - structcheck
     - stylecheck
     - typecheck
     - unconvert
     - unused
-    - varcheck
     - whitespace
     - gocognit
     - prealloc
@@ -89,6 +86,6 @@ issues:
 # golangci.com configuration
 # https://github.com/golangci/golangci/wiki/Configuration
 service:
-  golangci-lint-version: 1.43.x # use the fixed version to not introduce new linters unexpectedly
+  golangci-lint-version: 1.50.x # use the fixed version to not introduce new linters unexpectedly
   prepare:
-    - echo "here I can run custom commands, but no preparation needed for this repo"
+    - echo "here I can run custom commands, but no preparation needed for this repo"

Dockerfile.concourse

@@ -1,4 +1,4 @@
-FROM onsdigital/dp-concourse-tools-ubuntu
+FROM onsdigital/dp-concourse-tools-ubuntu-20:ubuntu20.4-rc.1
 
 WORKDIR /app/
 

Dockerfile.local

@@ -0,0 +1,14 @@
+FROM golang:1.19-bullseye AS base
+
+ENV GOCACHE=/go/.go/cache GOPATH=/go/.go/path TZ=Europe/London
+
+# Install github.com/cespare/reflex
+RUN GOBIN=/bin go install github.com/cespare/reflex@v0.3.1
+RUN PATH=$PATH:/bin
+
+# Clean cache, as we want all modules in the container to be under /go/.go/path
+RUN go clean -modcache
+
+# Map between the working directories of dev and live
+RUN ln -s /go /dp-search-data-extractor
+WORKDIR /dp-search-data-extractor

Makefile

@@ -1,48 +1,78 @@
 BINPATH ?= build
 
+GREEN  := $(shell tput -Txterm setaf 2)
+YELLOW := $(shell tput -Txterm setaf 3)
+WHITE  := $(shell tput -Txterm setaf 7)
+CYAN   := $(shell tput -Txterm setaf 6)
+RESET  := $(shell tput -Txterm sgr0)
+
 BUILD_TIME=$(shell date +%s)
 GIT_COMMIT=$(shell git rev-parse HEAD)
 VERSION ?= $(shell git tag --points-at HEAD | grep ^v | head -n 1)
 
 LDFLAGS = -ldflags "-X main.BuildTime=$(BUILD_TIME) -X main.GitCommit=$(GIT_COMMIT) -X main.Version=$(VERSION)"
 
 .PHONY: all
-all: audit test build
-
-.PHONY: fmt
-fmt:
-	go fmt ./...
+all: delimiter-AUDIT audit delimiter-LINTERS lint delimiter-UNIT-TESTS test delimiter-COMPONENT-TESTS test-component delimiter-FINISH ## Runs multiple targets, audit, lint, test and test-component
 
 .PHONY: audit
-audit:
+audit: ## Runs checks for security vulnerabilities on dependencies (including transient ones)
 	go list -m all | nancy sleuth
 
-.PHONY: lint
-lint:
-	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.43.0
-	golangci-lint run ./...
-
 .PHONY: build
-build:
+build: ## Builds binary of application code and stores in bin directory as dp-search-data-extractor
 	go build -tags 'production' $(LDFLAGS) -o $(BINPATH)/dp-search-data-extractor
 
+.PHONY: convey
+convey: ## Runs unit test suite and outputs results on http://127.0.0.1:8080/
+	goconvey ./...
+
+.PHONY: delimiter-%
+delimiter-%:
+	@echo '===================${GREEN} $* ${RESET}==================='
+
 .PHONY: debug
-debug:
+debug: ## Used to build and run code locally in debug mode
 	go build -tags 'debug' $(LDFLAGS) -o $(BINPATH)/dp-search-data-extractor
 	HUMAN_LOG=1 DEBUG=1 $(BINPATH)/dp-search-data-extractor
 
-.PHONY: test
-test:
-	go test -count=1 -race -cover ./...
+.PHONY: debug-run
+debug-run: ## Used to run code locally in debug mode
+	HUMAN_LOG=1 DEBUG=1 go run -tags 'debug' -race $(LDFLAGS) main.go
+
+.PHONY: fmt
+fmt: ## Run Go formatting on code
+	go fmt ./...
+
+.PHONY: lint
+lint: ## Used in ci to run linters against Go code
+	golangci-lint run ./...
+
+.PHONY: lint-local
+lint-local: ## Use locally to run linters against Go code
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1
+	golangci-lint run ./...
 
 .PHONY: produce
-produce:
+produce: ## Runs a kafka producer to write message/messages to Kafka topic
 	HUMAN_LOG=1 go run cmd/producer/main.go
 
-.PHONY: convey
-convey:
-	goconvey ./...
+.PHONY: test
+test: ## Runs unit tests including checks for race conditions and returns coverage
+	go test -count=1 -race -cover ./...
 
 .PHONY: test-component
-test-component:
-	go test -cover -coverpkg=github.com/ONSdigital/dp-search-data-extractor/... -component
+test-component: ## Runs component test suite
+	go test -cover -race -coverpkg=github.com/ONSdigital/dp-search-data-extractor/... -component
+
+.PHONY: help
+help: ## Show help page for list of make targets
+	@echo ''
+	@echo 'Usage:'
+	@echo '  ${YELLOW}make${RESET} ${GREEN}<target>${RESET}'
+	@echo ''
+	@echo 'Targets:'
+	@awk 'BEGIN {FS = ":.*?## "} { \
+		if (/^[a-zA-Z_-]+:.*?##.*$$/) {printf "    ${YELLOW}%-20s${GREEN}%s${RESET}\n", $$1, $$2} \
+		else if (/^## .*$$/) {printf "  ${CYAN}%s${RESET}\n", substr($$1,4)} \
+		}' $(MAKEFILE_LIST)

README.md

@@ -1,31 +1,29 @@
 dp-search-data-extractor
 ================
+
 Service to retrieve published data to be used to update a search index
-This service calls /publisheddata endpoint on zebedee.
+This service calls /publisheddata endpoint on [zebedee](https://github.com/ONSdigital/zebedee) and metadata endpoint on [dataset API](https://github.com/ONSdigital/dp-dataset-api).
+
+This service listens to the "content-updated" kafka topic for events of type contentUpdatedEvent e.g. 
+see [schemas](schema) package.
 
-This service listens to the "content-published" kafka topic for events of type contentPublishedEvent e.g. 
-"data": {
-  "contentPublishedEvent": {
-  "CollectionID": "",
-  "DataType": "",
-  "URI": "businessindustryandtrade"
-  }
-}
+This service takes the uri, from the consumed event, and either calls ... 
+1. ... /publisheddata endpoint on zebedee. It passes in the URI as a path parameter e.g.
+  http://localhost:8082/publisheddata?uri=businessindustryandtrade
+1. ... /datasets/<id>/editions/<edition>/versions/<version>/metadata endpoint on dataset API, e.g.
+  http://localhost:22000/datasets/CPIH01/editions/timeseries/versions/1/metadata
 
-This service takes the URI, from the consumed event, and calls /publisheddata endpoint on zebedee. It passes in the URI as a path parameter e.g.
-http://localhost:8082/publisheddata?uri=businessindustryandtrade
-This service then logs whether Zebedee has retrieved the data successfully.
+See [search service architecture docs here](https://github.com/ONSdigital/dp-search-api/tree/develop/architecture#search-service-architecture)
 
 ###  healthcheck
 
         make debug and then 
         check http://localhost:25800/health
 
-**TODO** : we are simply writing the retrieved content to a file for now but in future this will be passed on to another service via an as yet unwritten kafka producer.
-
 ### Getting started
 
 * Run `make debug`
+* Run `make help` to see full list of make targets
 
 The service runs in the background consuming messages from Kafka.
 An example event can be created using the helper script, `make produce`.
@@ -43,6 +41,7 @@ An example event can be created using the helper script, `make produce`.
 | Environment variable           | Default                  | Description
 | ----------------------------   | -------------------------| -----------
 | BIND_ADDR                      | localhost:25800          | The host and port to bind to
+| DATASET_API_URL                | `http://localhost:22000` | The URL for the DatasetAPI
 | GRACEFUL_SHUTDOWN_TIMEOUT      | 5s                       | The graceful shutdown timeout in seconds (`time.Duration` format)
 | HEALTHCHECK_INTERVAL           | 30s                      | Time between self-healthchecks (`time.Duration` format)
 | HEALTHCHECK_CRITICAL_TIMEOUT   | 90s                      | Time to wait until an unhealthy dependent propagates its state to make this app unhealthy (`time.Duration` format)
@@ -57,11 +56,10 @@ An example event can be created using the helper script, `make produce`.
 | KAFKA_SEC_SKIP_VERIFY          | false                    | ignore server certificate issues if set to `true` [[1]](#notes_1)
 | KAFKA_CONTENT_UPDATED_GROUP    | dp-search-data-extractor | The consumer group this application to consume content-updated messages
 | KAFKA_CONTENT_UPDATED_TOPIC    | content-updated          | The name of the topic to consume messages from
-| ZEBEDEE_URL                    | `http://localhost:8082`  | The URL for the Zebedee
 | KEYWORDS_LIMITS                | -1                       | The keywords allowed, default no limit
-| DATASET_API_URL                | `http://localhost:22000` | The URL for the DatasetAPI
-| SERVICE_AUTH_TOKEN             | ""                       | The user auth token for the DatasetAPI
-
+| SERVICE_AUTH_TOKEN             | _unset_                  | The user auth token for the DatasetAPI
+| STOP_CONSUMING_ON_UNHEALTHY    | true                     | Application stops consuming kafka messages if application is in unhealthy state
+| ZEBEDEE_URL                    | `http://localhost:8082`  | The URL for the Zebedee
 
 **Notes:**
 
@@ -83,4 +81,4 @@ See [CONTRIBUTING](CONTRIBUTING.md) for details.
 
 Copyright © 2022, Office for National Statistics (https://www.ons.gov.uk)
 
-Released under MIT license, see [LICENSE](LICENSE.md) for details.
+Released under MIT license, see [LICENSE](LICENSE.md) for details.

ci/build.yml

@@ -6,7 +6,7 @@ image_resource:
   type: docker-image
   source:
     repository: golang
-    tag: 1.18.3
+    tag: 1.19.5
 
 inputs:
   - name: dp-search-data-extractor

ci/component.yml

@@ -6,7 +6,7 @@ image_resource:
   type: docker-image
   source:
     repository: golang
-    tag: 1.18.3
+    tag: 1.19.5
 
 inputs:
   - name: dp-search-data-extractor

ci/lint.yml

@@ -5,8 +5,8 @@ platform: linux
 image_resource:
   type: docker-image
   source:
-    repository: golang
-    tag: 1.18.3
+    repository: golangci/golangci-lint
+    tag: v1.50
 
 inputs:
   - name: dp-search-data-extractor

ci/unit.yml

@@ -6,7 +6,7 @@ image_resource:
   type: docker-image
   source:
     repository: golang
-    tag: 1.18.3
+    tag: 1.19.5
 
 inputs:
   - name: dp-search-data-extractor