From 9c79a604f774cd9714744bc30bf535dfbe2e1f7f Mon Sep 17 00:00:00 2001 From: ccremer Date: Tue, 27 Sep 2022 11:31:40 +0200 Subject: [PATCH] Add component --- component/Makefile | 85 +++++++++++++ component/Makefile.vars.mk | 47 ++++++++ component/class/defaults.yml | 15 +++ .../class/exoscale-metrics-collector.yml | 11 ++ component/component/app.jsonnet | 11 ++ component/component/main.jsonnet | 113 ++++++++++++++++++ component/jsonnetfile.json | 14 +++ component/jsonnetfile.lock.json | 16 +++ component/tests/defaults.yml | 9 ++ .../apps/exoscale-metrics-collector.yaml | 0 .../exoscale-metrics-collector/cronjob.yaml | 52 ++++++++ .../exoscale-metrics-collector/secrets.yaml | 13 ++ 12 files changed, 386 insertions(+) create mode 100644 component/Makefile create mode 100644 component/Makefile.vars.mk create mode 100644 component/class/defaults.yml create mode 100644 component/class/exoscale-metrics-collector.yml create mode 100644 component/component/app.jsonnet create mode 100644 component/component/main.jsonnet create mode 100644 component/jsonnetfile.json create mode 100644 component/jsonnetfile.lock.json create mode 100644 component/tests/defaults.yml create mode 100644 component/tests/golden/defaults/exoscale-metrics-collector/apps/exoscale-metrics-collector.yaml create mode 100644 component/tests/golden/defaults/exoscale-metrics-collector/exoscale-metrics-collector/cronjob.yaml create mode 100644 component/tests/golden/defaults/exoscale-metrics-collector/exoscale-metrics-collector/secrets.yaml diff --git a/component/Makefile b/component/Makefile new file mode 100644 index 0000000..9a926b2 --- /dev/null +++ b/component/Makefile @@ -0,0 +1,85 @@ +MAKEFLAGS += --warn-undefined-variables +SHELL := bash +.SHELLFLAGS := -eu -o pipefail -c +.DEFAULT_GOAL := all +.DELETE_ON_ERROR: +.SUFFIXES: + +include Makefile.vars.mk + +.PHONY: help +help: ## Show this help + @grep -E -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = "(: ).*?## "}; {gsub(/\\:/,":", $$1)}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +.PHONY: all +all: lint + +.PHONY: lint +lint: lint_jsonnet lint_yaml lint_adoc lint_kubent ## All-in-one linting + +.PHONY: lint_jsonnet +lint_jsonnet: $(JSONNET_FILES) ## Lint jsonnet files + $(JSONNET_DOCKER) $(JSONNETFMT_ARGS) --test -- $? + +.PHONY: lint_yaml +lint_yaml: ## Lint yaml files + $(YAMLLINT_DOCKER) -f parsable -c $(YAMLLINT_CONFIG) $(YAMLLINT_ARGS) -- . + +.PHONY: lint_adoc +lint_adoc: ## Lint documentation + $(VALE_CMD) $(VALE_ARGS) +.PHONY: lint_kubent +lint_kubent: ## Check for deprecated Kubernetes API versions + $(KUBENT_DOCKER) $(KUBENT_ARGS) -f $(KUBENT_FILES) + +.PHONY: format +format: format_jsonnet ## All-in-one formatting + +.PHONY: format_jsonnet +format_jsonnet: $(JSONNET_FILES) ## Format jsonnet files + $(JSONNET_DOCKER) $(JSONNETFMT_ARGS) -- $? + +.PHONY: docs-serve +docs-serve: ## Preview the documentation + $(ANTORA_PREVIEW_CMD) + +.PHONY: compile +.compile: + mkdir -p dependencies + $(COMPILE_CMD) + +.PHONY: test +test: commodore_args += -f tests/$(instance).yml +test: .compile ## Compile the component + +.PHONY: gen-golden +gen-golden: commodore_args += -f tests/$(instance).yml +gen-golden: clean .compile ## Update the reference version for target `golden-diff`. + @rm -rf tests/golden/$(instance) + @mkdir -p tests/golden/$(instance) + @cp -R compiled/. tests/golden/$(instance)/. + +.PHONY: golden-diff +golden-diff: commodore_args += -f tests/$(instance).yml +golden-diff: clean .compile ## Diff compile output against the reference version. Review output and run `make gen-golden golden-diff` if this target fails. + @git diff --exit-code --minimal --no-index -- tests/golden/$(instance) compiled/ + +.PHONY: golden-diff-all +golden-diff-all: recursive_target=golden-diff +golden-diff-all: $(test_instances) ## Run golden-diff for all instances. Note: this doesn't work when running make with multiple parallel jobs (-j != 1). + +.PHONY: gen-golden-all +gen-golden-all: recursive_target=gen-golden +gen-golden-all: $(test_instances) ## Run gen-golden for all instances. Note: this doesn't work when running make with multiple parallel jobs (-j != 1). + +.PHONY: lint_kubent_all +lint_kubent_all: recursive_target=lint_kubent +lint_kubent_all: $(test_instances) ## Lint deprecated Kubernetes API versions for all golden test instances. Will exit on first error. Note: this doesn't work when running make with multiple parallel jobs (-j != 1). + +.PHONY: $(test_instances) +$(test_instances): + $(MAKE) $(recursive_target) -e instance=$(basename $(@F)) + +.PHONY: clean +clean: ## Clean the project + rm -rf .cache compiled dependencies vendor helmcharts jsonnetfile*.json || true diff --git a/component/Makefile.vars.mk b/component/Makefile.vars.mk new file mode 100644 index 0000000..7afe10a --- /dev/null +++ b/component/Makefile.vars.mk @@ -0,0 +1,47 @@ +# Commodore takes the root dir name as the component name +COMPONENT_NAME ?= exoscale-metrics-collector +COMPONENT_SUBDIR ?= $(shell basename ${PWD}) + +compiled_path ?= compiled/$(COMPONENT_NAME)/$(COMPONENT_NAME) +root_volume ?= -v "$${PWD}/../:/$(COMPONENT_NAME)" +compiled_volume ?= -v "$${PWD}/$(compiled_path):/$(COMPONENT_NAME)" +commodore_args ?= --search-paths . -n $(COMPONENT_NAME) + +ifneq "$(shell which docker 2>/dev/null)" "" + DOCKER_CMD ?= $(shell which docker) + DOCKER_USERNS ?= "" +else + DOCKER_CMD ?= podman + DOCKER_USERNS ?= keep-id +endif +DOCKER_ARGS ?= run --rm -u "$$(id -u):$$(id -g)" --userns=$(DOCKER_USERNS) -w /$(COMPONENT_NAME)/$(COMPONENT_SUBDIR) -e HOME="/$(COMPONENT_NAME)" + +JSONNET_FILES ?= $(shell find . -type f -not -path './vendor/*' \( -name '*.*jsonnet' -or -name '*.libsonnet' \)) +JSONNETFMT_ARGS ?= --in-place --pad-arrays +JSONNET_IMAGE ?= docker.io/bitnami/jsonnet:latest +JSONNET_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=jsonnetfmt $(JSONNET_IMAGE) + +YAMLLINT_ARGS ?= --no-warnings +YAMLLINT_CONFIG ?= .yamllint.yml +YAMLLINT_IMAGE ?= docker.io/cytopia/yamllint:latest +YAMLLINT_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) $(YAMLLINT_IMAGE) + +VALE_CMD ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --volume "$${PWD}"/docs/modules:/pages docker.io/vshn/vale:2.1.1 +VALE_ARGS ?= --minAlertLevel=error --config=/pages/ROOT/pages/.vale.ini /pages + +ANTORA_PREVIEW_CMD ?= $(DOCKER_CMD) run --rm --publish 35729:35729 --publish 2020:2020 --volume "${PWD}/.git":/preview/antora/.git --volume "${PWD}/docs":/preview/antora/docs docker.io/vshn/antora-preview:3.0.1.1 --style=syn --antora=docs + + +COMMODORE_CMD ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) docker.io/projectsyn/commodore:latest +COMPILE_CMD ?= $(COMMODORE_CMD) component compile . $(commodore_args) +JB_CMD ?= $(DOCKER_CMD) $(DOCKER_ARGS) --entrypoint /usr/local/bin/jb docker.io/projectsyn/commodore:latest install +GOLDEN_FILES ?= $(shell find tests/golden/$(instance) -type f) + +KUBENT_FILES ?= $(shell echo "$(GOLDEN_FILES)" | sed 's/ /,/g') +KUBENT_ARGS ?= -c=false --helm2=false --helm3=false -e +# Use our own kubent image until the upstream image is available +KUBENT_IMAGE ?= docker.io/projectsyn/kubent:latest +KUBENT_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE) + +instance ?= defaults +test_instances = tests/defaults.yml diff --git a/component/class/defaults.yml b/component/class/defaults.yml new file mode 100644 index 0000000..121a2f8 --- /dev/null +++ b/component/class/defaults.yml @@ -0,0 +1,15 @@ +parameters: + exoscale_metrics_collector: + secrets: + exoscale: + stringData: + api_key: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/key}" + api_secret: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/secret}" + images: + collector: + registry: 'ghcr.io' + repository: 'vshn/exoscale-metrics-collector' + tag: 'v0.0.2' + # Times in UTC! Don't run job around midnight as exoscale API may return incomplete data + # default: Every day at minute 10 past hour 4, 10, and 16. + schedule: '10 4,10,16 * * *' diff --git a/component/class/exoscale-metrics-collector.yml b/component/class/exoscale-metrics-collector.yml new file mode 100644 index 0000000..b77e121 --- /dev/null +++ b/component/class/exoscale-metrics-collector.yml @@ -0,0 +1,11 @@ +parameters: + kapitan: + compile: + - input_paths: + - ${_base_directory}/component/app.jsonnet + input_type: jsonnet + output_path: apps/ + - input_paths: + - ${_base_directory}/component/main.jsonnet + input_type: jsonnet + output_path: exoscale-metrics-collector/ diff --git a/component/component/app.jsonnet b/component/component/app.jsonnet new file mode 100644 index 0000000..8299569 --- /dev/null +++ b/component/component/app.jsonnet @@ -0,0 +1,11 @@ +local kap = import 'lib/kapitan.libjsonnet'; +local inv = kap.inventory(); +local params = inv.parameters.exoscale_metrics_collector; +local paramsACR = inv.parameters.appuio_cloud_reporting; +local argocd = import 'lib/argocd.libjsonnet'; + +local app = argocd.App('exoscale-metrics-collector', paramsACR.namespace); + +{ + 'exoscale-metrics-collector': app, +} diff --git a/component/component/main.jsonnet b/component/component/main.jsonnet new file mode 100644 index 0000000..e86d729 --- /dev/null +++ b/component/component/main.jsonnet @@ -0,0 +1,113 @@ +local kap = import 'lib/kapitan.libjsonnet'; +local inv = kap.inventory(); +local params = inv.parameters.exoscale_metrics_collector; +local paramsACR = inv.parameters.appuio_cloud_reporting; +local kube = import 'lib/kube.libjsonnet'; +local com = import 'lib/commodore.libjsonnet'; +local collectorImage = '%(registry)s/%(repository)s:%(tag)s' % params.images.collector; + + +local labels = { + 'app.kubernetes.io/name': 'exoscale-metrics-collector', + 'app.kubernetes.io/managed-by': 'commodore', + 'app.kubernetes.io/part-of': 'appuio-cloud-reporting', + 'app.kubernetes.io/component': 'exoscale-metrics-collector', +}; + +local secrets = [ + if params.secrets[s] != null then + kube.Secret(s) { + metadata+: { + namespace: paramsACR.namespace, + }, + } + com.makeMergeable(params.secrets[s]) + for s in std.objectFields(params.secrets) +]; + +{ + assert params.secrets != null : 'secrets must be set.', + assert params.secrets.exoscale != null : 'secrets.exoscale must be set.', + assert params.secrets.exoscale.stringData != null : 'secrets.exoscale.stringData must be set.', + assert params.secrets.exoscale.stringData.api_key != null : 'secrets.exoscale.stringData.api_key must be set.', + assert params.secrets.exoscale.stringData.api_secret != null : 'secrets.exoscale.stringData.api_secret must be set.', + + secrets: std.filter(function(it) it != null, secrets), + + cronjob: { + kind: 'CronJob', + apiVersion: 'batch/v1', + metadata: { + name: 'exoscale-metrics-collector', + namespace: paramsACR.namespace, + labels+: labels, + }, + spec: { + concurrencyPolicy: 'Forbid', + failedJobsHistoryLimit: 5, + jobTemplate: { + spec: { + template: { + spec: { + restartPolicy: 'OnFailure', + containers: [ + { + name: 'exoscale-metrics-collector-backfill', + image: collectorImage, + args: [ + 'exoscale-metrics-collector', + ], + command: [ 'sh', '-c' ], + env: [ + { + name: 'password', + valueFrom: { + secretKeyRef: { + key: 'password', + name: 'reporting-db', + }, + }, + }, + { + name: 'username', + valueFrom: { + secretKeyRef: { + key: 'username', + name: 'reporting-db', + }, + }, + }, + { + name: 'ACR_DB_URL', + value: 'postgres://$(username):$(password)@%(host)s:%(port)s/%(name)s?%(parameters)s' % paramsACR.database, + }, + { + name: 'EXOSCALE_API_KEY', + valueFrom: { + secretKeyRef: { + key: 'api_key', + name: 'exoscale', + }, + }, + }, + { + name: 'EXOSCALE_API_SECRET', + valueFrom: { + secretKeyRef: { + key: 'api_secret', + name: 'exoscale', + }, + }, + }, + ], + resources: {}, + }, + ], + }, + }, + }, + }, + schedule: params.schedule, + successfulJobsHistoryLimit: 3, + }, + }, +} diff --git a/component/jsonnetfile.json b/component/jsonnetfile.json new file mode 100644 index 0000000..b3abc0c --- /dev/null +++ b/component/jsonnetfile.json @@ -0,0 +1,14 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/bitnami-labs/kube-libsonnet" + } + }, + "version": "v1.19.0" + } + ], + "legacyImports": true +} diff --git a/component/jsonnetfile.lock.json b/component/jsonnetfile.lock.json new file mode 100644 index 0000000..e850735 --- /dev/null +++ b/component/jsonnetfile.lock.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/bitnami-labs/kube-libsonnet.git", + "subdir": "" + } + }, + "version": "20bf70d8d24b3a50baaae8f951dcbc62a66b464f", + "sum": "e0p4pbwKnpRfUm8CXqe+WC1Hcwih1lqIm1ODPsCV96Q=" + } + ], + "legacyImports": false +} diff --git a/component/tests/defaults.yml b/component/tests/defaults.yml new file mode 100644 index 0000000..d94263c --- /dev/null +++ b/component/tests/defaults.yml @@ -0,0 +1,9 @@ +parameters: + appuio_cloud_reporting: + namespace: 'appuio-cloud-reporting' + database: + name: 'reporting' + host: 'reporting-db.appuio-reporting.svc' + parameters: 'sslmode=disable' + password: 'passw0rd' + port: 5432 diff --git a/component/tests/golden/defaults/exoscale-metrics-collector/apps/exoscale-metrics-collector.yaml b/component/tests/golden/defaults/exoscale-metrics-collector/apps/exoscale-metrics-collector.yaml new file mode 100644 index 0000000..e69de29 diff --git a/component/tests/golden/defaults/exoscale-metrics-collector/exoscale-metrics-collector/cronjob.yaml b/component/tests/golden/defaults/exoscale-metrics-collector/exoscale-metrics-collector/cronjob.yaml new file mode 100644 index 0000000..0887102 --- /dev/null +++ b/component/tests/golden/defaults/exoscale-metrics-collector/exoscale-metrics-collector/cronjob.yaml @@ -0,0 +1,52 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + labels: + app.kubernetes.io/component: exoscale-metrics-collector + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: exoscale-metrics-collector + app.kubernetes.io/part-of: appuio-cloud-reporting + name: exoscale-metrics-collector + namespace: appuio-cloud-reporting +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + template: + spec: + containers: + - args: + - exoscale-metrics-collector + command: + - sh + - -c + env: + - name: password + valueFrom: + secretKeyRef: + key: password + name: reporting-db + - name: username + valueFrom: + secretKeyRef: + key: username + name: reporting-db + - name: ACR_DB_URL + value: postgres://$(username):$(password)@reporting-db.appuio-reporting.svc:5432/reporting?sslmode=disable + - name: EXOSCALE_API_KEY + valueFrom: + secretKeyRef: + key: api_key + name: exoscale + - name: EXOSCALE_API_SECRET + valueFrom: + secretKeyRef: + key: api_secret + name: exoscale + image: ghcr.io/vshn/exoscale-metrics-collector:v0.0.2 + name: exoscale-metrics-collector-backfill + resources: {} + restartPolicy: OnFailure + schedule: 10 4,10,16 * * * + successfulJobsHistoryLimit: 3 diff --git a/component/tests/golden/defaults/exoscale-metrics-collector/exoscale-metrics-collector/secrets.yaml b/component/tests/golden/defaults/exoscale-metrics-collector/exoscale-metrics-collector/secrets.yaml new file mode 100644 index 0000000..e2f533b --- /dev/null +++ b/component/tests/golden/defaults/exoscale-metrics-collector/exoscale-metrics-collector/secrets.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +data: {} +kind: Secret +metadata: + annotations: {} + labels: + name: exoscale + name: exoscale + namespace: appuio-cloud-reporting +stringData: + api_key: t-silent-test-1234/c-green-test-1234/exoscale-metrics-collector/key + api_secret: t-silent-test-1234/c-green-test-1234/exoscale-metrics-collector/secret +type: Opaque