Skip to content

Commit

Permalink
feat(container): make chaos GBI compliant (#734)
Browse files Browse the repository at this point in the history
  • Loading branch information
luphaz committed Jul 19, 2023
1 parent d0e52a2 commit f149976
Show file tree
Hide file tree
Showing 28 changed files with 356 additions and 277 deletions.
2 changes: 0 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,6 @@ executors:
golang:
<<: *working_directory
docker:
# TODO: Publicize the chaos controller ci image to the datadog docker registry
#- image: datadog/chaos-controller-runner-circle:<< pipeline.parameters.CURRENT_CI_IMAGE >>
# This is circle ci images, provides default tool installed (like docker) to ease step definition and avoid apt-get/update things
# https://circleci.com/docs/circleci-images/#next-gen-language-images
- image: cimg/go:1.20.5
Expand Down
18 changes: 2 additions & 16 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,21 +1,12 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
*.tar.gz
bin/injector/injector*
bin/injector/bpf-disk-failure*
bin/injector*
bin/injector/ebpf
bin/manager/manager*
bin/manager*
bin/handler/handler*
bin/handler*
bin/chaosli/chaosli.tar.gz
bin/chaosli/chaosli_darwin_amd64
bin/chaosli/chaosli_darwin_amd64.sha256sum
bin/chaosli/chaosli_darwin_arm64*
bin/chaosli/chaosli*
chart/install.yaml
dogfood/bin
out
Expand Down Expand Up @@ -54,11 +45,6 @@ cover.profile.*
# output of chaosli create
disruption.yaml

# packer
packer_cache/
crash.log
*.json-e

# local certificates & config
*.crt
*.key
Expand Down
34 changes: 13 additions & 21 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,16 @@ variables:
HANDLER_IMAGE_NAME: chaos-handler

stages:
- ci-image
- build
- release-staging
- release-prod
- release-public
- notify

.install-make: &install-make
- apt-get update
- apt-get install -y --no-install-recommends build-essential git

.docker-runner: &docker-runner
image: registry.ddbuild.io/docker-push:1.7.0
tags: ["runner:docker"]
Expand All @@ -30,30 +33,15 @@ stages:
- DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.chaos-engineering.docker_hub_login --with-decryption --query "Parameter.Value" --out text)
- aws ssm get-parameter --region us-east-1 --name ci.chaos-engineering.docker_hub_pwd --with-decryption --query "Parameter.Value" --out text | docker login --username "$DOCKER_REGISTRY_LOGIN" --password-stdin docker.io

# CI image
ci-image:
<<: *docker-hub-login
stage: ci-image
when: manual
except: [tags, schedules]
script:
- docker buildx create --use
- docker buildx build --platform linux/amd64,linux/arm64 --tag registry.ddbuild.io/ci/chaos-controller:$CURRENT_CI_IMAGE ci --push

# main make build
build:make:
<<: *docker-hub-login
stage: build
when: always
variables:
GO_FILENAME: go1.20.5.linux-amd64.tar.gz
script:
- apt-get update
- apt-get -y install build-essential git
- curl -O https://dl.google.com/go/${GO_FILENAME}
- rm -rf /usr/local/go && tar -C /usr/local -xzf ${GO_FILENAME}
- *install-make
- export PATH="/usr/local/go/bin:${PATH}"
- make GOBIN=/usr/local/go/bin all docker-build-ebpf
- make GOBIN=/usr/local/go/bin install-go all docker-build-ebpf
artifacts:
paths:
- bin/manager/manager_amd64
Expand All @@ -71,9 +59,13 @@ build:make:
<<: *docker-hub-login
script:
- docker buildx create --use
- docker buildx build --platform linux/amd64,linux/arm64 -t registry.ddbuild.io/${CONTROLLER_IMAGE_NAME}:${TAG} -t registry.ddbuild.io/${CONTROLLER_IMAGE_NAME}:${CI_COMMIT_SHA} --label target=${TARGET_LABEL} -f bin/manager/Dockerfile ./bin/manager/ --push
- docker buildx build --platform linux/amd64,linux/arm64 -t registry.ddbuild.io/${INJECTOR_IMAGE_NAME}:${TAG} -t registry.ddbuild.io/${INJECTOR_IMAGE_NAME}:${CI_COMMIT_SHA} --label target=${TARGET_LABEL} -f bin/injector/Dockerfile ./bin/injector/ --push
- docker buildx build --platform linux/amd64,linux/arm64 -t registry.ddbuild.io/${HANDLER_IMAGE_NAME}:${TAG} -t registry.ddbuild.io/${HANDLER_IMAGE_NAME}:${CI_COMMIT_SHA} --label target=${TARGET_LABEL} -f bin/handler/Dockerfile ./bin/handler/ --push
- *install-make
- >
make docker-build-only-all \
CONTAINER_REGISTRY=registry.ddbuild.io \
CONTAINER_TAG=${TAG} \
CONTAINER_VERSION=${CI_COMMIT_SHA} \
CONTAINER_BUILD_EXTRA_ARGS="--platform=linux/amd64,linux/arm64 --label target=${TARGET_LABEL} --push"
dependencies:
- build:make

Expand Down
6 changes: 3 additions & 3 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ Once you have installed the above requirements, run the `make lima-all` command
- `make lima-build` to build the chaos-controller images
- `make lima-install` to render and apply the chaos-controller helm chart

Once the instance is started, you can log into it using either the `lima` or its longer form `limactl shell default` commands.
Once the instance is started, you can log into it using either the `lima` or its longer form `limactl shell <$LIMA_INSTANCE>` commands.

#### Change default lima instance

Expand Down Expand Up @@ -104,7 +104,7 @@ export STAGING_DD_SITE=https://app.datadoghq.com

To deploy changes made to the controller code or chart, run the `make lima-redeploy` command that will run the following targets:

- `make lima-build` to build the chaos-controller images
- `make lima-push-all` to build the chaos-controller images locally, and push them into the k3s cluster
- `make lima-install` to render and apply the chaos-controller helm chart
- `make lima-restart` to restart the chaos-controller manager pod

Expand Down Expand Up @@ -215,7 +215,7 @@ export DATADOG_API_KEY=$(security find-generic-password -a ${USER} -s datadog_ap

- Install `datadog-ci` by running `make install-datadog-ci`

- Run tests `make test || make e2e-test`
- Run tests `make test && make e2e-test`

- Go to Datadog you [test-services](https://app.datadoghq.com/ci/test-services?env=local&view=branches&paused=false)

Expand Down
54 changes: 35 additions & 19 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,9 +1,15 @@
.PHONY: *
.SILENT: release

NOW_ISO8601 = $(shell date -u +"%Y-%m-%dT%H:%M:%S")

GOOS = $(shell go env GOOS)
GOARCH = $(shell go env GOARCH)

# change also circleci go build version "cimb/go:" if you change the version below
# https://github.com/DataDog/chaos-controller/blob/main/.circleci/config.yml#L85
BUILDGOVERSION = 1.20.5

# GOBIN can be provided (gitlab), defined (custom user setup), or empty/guessed (default go setup)
GOBIN ?= $(shell go env GOBIN)
ifeq (,$(GOBIN))
Expand All @@ -19,18 +25,17 @@ LIMA_INSTALL_SINK = datadog
endif
endif

ifndef CONTROLLER_APP_VERSION
CONTROLLER_APP_VERSION = $(shell git rev-parse HEAD)$(shell git diff --quiet || echo '-dirty')
endif

# Lima requires to have images built on a specific namespace to be shared to the Kubernetes cluster when using containerd runtime
# https://github.com/abiosoft/colima#interacting-with-image-registry
CONTAINERD_REGISTRY_PREFIX ?= k8s.io
CONTAINER_REGISTRY ?= k8s.io
CONTAINER_TAG ?= latest
CONTAINER_VERSION ?= $(shell git rev-parse HEAD)$(shell git diff --quiet || echo '-dirty')
CONTAINER_BUILD_EXTRA_ARGS ?=

# Image URL to use all building/pushing image targets
MANAGER_IMAGE ?= ${CONTAINERD_REGISTRY_PREFIX}/chaos-controller:latest
INJECTOR_IMAGE ?= ${CONTAINERD_REGISTRY_PREFIX}/chaos-injector:latest
HANDLER_IMAGE ?= ${CONTAINERD_REGISTRY_PREFIX}/chaos-handler:latest
MANAGER_IMAGE ?= ${CONTAINER_REGISTRY}/chaos-controller
INJECTOR_IMAGE ?= ${CONTAINER_REGISTRY}/chaos-injector
HANDLER_IMAGE ?= ${CONTAINER_REGISTRY}/chaos-handler

LIMA_PROFILE ?= lima
LIMA_CONFIG ?= lima
Expand Down Expand Up @@ -59,7 +64,7 @@ HELM_INSTALLED_VERSION = $(shell (helm version --template="{{ .Version }}" || ec
GOLANGCI_LINT_VERSION = 1.52.2
GOLANGCI_LINT_INSTALLED_VERSION = $(shell (golangci-lint --version || echo "") | sed -E 's/.*version ([^ ]+).*/\1/')

CONTROLLER_GEN_VERSION = v0.11.4
CONTROLLER_GEN_VERSION = v0.12.0
CONTROLLER_GEN_INSTALLED_VERSION = $(shell (controller-gen --version || echo "") | awk '{ print $$2 }')

MOCKERY_VERSION = 2.28.2
Expand Down Expand Up @@ -87,12 +92,12 @@ endif
injector handler: BINARY_PATH=./cli/$(BINARY_NAME)
manager: BINARY_PATH=.

docker-build-injector: IMAGE_TAG=$(INJECTOR_IMAGE)
docker-build-handler: IMAGE_TAG=$(HANDLER_IMAGE)
docker-build-manager: IMAGE_TAG=$(MANAGER_IMAGE)
docker-build-injector docker-build-only-injector: CONTAINER_NAME=$(INJECTOR_IMAGE)
docker-build-handler docker-build-only-handler: CONTAINER_NAME=$(HANDLER_IMAGE)
docker-build-manager docker-build-only-manager: CONTAINER_NAME=$(MANAGER_IMAGE)

docker-build-ebpf:
docker buildx build --platform linux/$(GOARCH) --build-arg ARCH=$(GOARCH) -t ebpf-builder-$(GOARCH) -f bin/ebpf-builder/Dockerfile ./bin/ebpf-builder/
docker buildx build --platform linux/$(GOARCH) --build-arg BUILDGOVERSION=$(BUILDGOVERSION) -t ebpf-builder-$(GOARCH) -f bin/ebpf-builder/Dockerfile .
-rm -r bin/injector/ebpf/
ifeq (true,$(USE_VOLUMES))
# create a dummy container with volume to store files
Expand Down Expand Up @@ -135,9 +140,17 @@ _$(1)_amd:

$(1): _$(1) _$(1)_arm _$(1)_amd

docker-build-$(1): _docker-build-$(1) $(1)
docker buildx build --build-arg TARGETARCH=$(GOARCH) -t $$(IMAGE_TAG) -f bin/$(1)/Dockerfile ./bin/$(1)/
docker save $$(IMAGE_TAG) -o ./bin/$(1)/$(1).tar.gz
docker-build-$(1): _docker-build-$(1) $(1) docker-build-only-$(1)
docker save $$(CONTAINER_NAME):$(CONTAINER_TAG) -o ./bin/$(1)/$(1).tar.gz

docker-build-only-$(1):
docker buildx build \
--build-arg BUILDGOVERSION=$(BUILDGOVERSION) \
--build-arg BUILDSTAMP=$(NOW_ISO8601) \
-t $$(CONTAINER_NAME):$(CONTAINER_TAG) \
-t $$(CONTAINER_NAME):$(CONTAINER_VERSION) \
$(CONTAINER_BUILD_EXTRA_ARGS) \
-f bin/$(1)/Dockerfile ./bin/$(1)/

lima-push-$(1): docker-build-$(1)
limactl copy ./bin/$(1)/$(1).tar.gz $(LIMA_INSTANCE):/tmp/
Expand All @@ -159,6 +172,7 @@ $(foreach tgt,$(TARGETS),$(eval $(call TARGET_template,$(tgt))))
all: $(TARGETS)

docker-build-all: $(addprefix docker-build-,$(TARGETS))
docker-build-only-all: $(addprefix docker-build-only-,$(TARGETS))
lima-push-all: $(addprefix lima-push-,$(TARGETS))
minikube-load-all: $(addprefix minikube-load-,$(TARGETS))

Expand Down Expand Up @@ -297,13 +311,12 @@ lima-install-demo:
$(KUBECTL) -n chaos-demo rollout status deployment/demo-curl --timeout=60s
$(KUBECTL) -n chaos-demo rollout status deployment/demo-nginx --timeout=60s


## Install CRDs and controller into a lima k3s cluster
## In order to use already built images inside the containerd runtime
## we override images for all of our components to the expected namespace
lima-install: manifests
helm template \
--set=controller.version=$(CONTROLLER_APP_VERSION) \
--set=controller.version=$(CONTAINER_VERSION) \
--set=controller.metricsSink=$(LIMA_INSTALL_SINK) \
--set=controller.profilerSink=$(LIMA_INSTALL_SINK) \
--set=controller.tracerSink=$(LIMA_INSTALL_SINK) \
Expand Down Expand Up @@ -497,11 +510,14 @@ ifeq (,$(wildcard $(GOBIN)/yamlfmt))
endif

install-watchexec:
ifeq (,$(wildcard $(GOBIN)/gow))
ifeq (,$(wildcard $(GOBIN)/watchexec))
$(info installing watchexec...)
brew install watchexec
endif

install-go:
BUILDGOVERSION=$(BUILDGOVERSION) ./scripts/install-go

EXISTING_NAMESPACE = $(shell $(KUBECTL) get ns datadog-agent -oname || echo "")

lima-install-datadog-agent:
Expand Down
20 changes: 10 additions & 10 deletions bin/ebpf-builder/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@ RUN apt-get update && \
libbpf-dev \
llvm

ARG ARCH=arm64
RUN ([ "${ARCH}" = "amd64" ] && apt-get install -y libc6-dev-i386) || true
ARG TARGETARCH
RUN ([ "${TARGETARCH}" = "amd64" ] && apt-get install -y libc6-dev-i386) || true

# Install Go specific version.
ENV GOVERSION=1.19.4
RUN curl -LO https://golang.org/dl/go${GOVERSION}.linux-${ARCH}.tar.gz && \
tar -xf go${GOVERSION}.linux-${ARCH}.tar.gz && \
mv go/ /usr/local/ && \
ln -s /usr/local/go/bin/go /usr/local/bin/ && \
rm -f go${GOVERSION}.linux-${ARCH}.tar.gz
# Provide expected go version through build arg
ARG BUILDGOVERSION
# We defined TARGETARCH and BUILDGOVERSION as env so they can be seen by the make install-go target
ENV TARGETARCH=${TARGETARCH} \
BUILDGOVERSION=${BUILDGOVERSION}
WORKDIR /tmp
COPY ./scripts/install-go /tmp/install-go
RUN /tmp/install-go && rm -f /tmp/install-go

# Setup working directory.
RUN mkdir -p /app
WORKDIR /app/ebpf

# Execute build command.
Expand Down
8 changes: 7 additions & 1 deletion bin/handler/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
FROM scratch as handler

ARG TARGETARCH

COPY handler_${TARGETARCH} /usr/local/bin/handler

ENTRYPOINT ["/usr/local/bin/handler"]

LABEL baseimage.os="scratch"
LABEL baseimage.isgbi="scratch"
LABEL baseimage.name="scratch"

ARG BUILDSTAMP
LABEL baseimage.buildstamp="${BUILDSTAMP}"
46 changes: 39 additions & 7 deletions bin/injector/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,16 +1,48 @@
FROM ubuntu:22.10 as injector
FROM ubuntu:22.10 as binaries

RUN apt-get update && \
# iproute2 => tc
# coreutils => df
# iptables => iptables libs
# libelf1 => EBPF libs
apt-get -y install --no-install-recommends iproute2 coreutils iptables libelf1 tree && \
# make copy from binaries unified and possible
mkdir -p /lib64

FROM gcr.io/distroless/python3-debian11:latest

# binaries used by the chaos-injector, ran as commmands
COPY --from=binaries /usr/bin/df /usr/bin/ls /usr/bin/test /usr/bin/
COPY --from=binaries /usr/sbin/iptables /usr/sbin/
COPY --from=binaries /sbin/tc /sbin/tc

# libraries used by above mentioned binaries (mostly GLIBC related)
COPY --from=binaries /lib/ld-linux-aarch64.so.[1] /lib/
COPY --from=binaries /lib64/ld-linux-x86-64.so.[2] /lib64/
COPY --from=binaries /lib/tc /lib/tc/
COPY --from=binaries /usr/lib/tc /usr/lib/tc/
COPY --from=binaries /lib/aarch64-linux-gn[u] /lib/aarch64-linux-gnu/
COPY --from=binaries /lib/x86_64-linux-gn[u] /lib/x86_64-linux-gnu/
COPY --from=binaries /usr/lib/aarch64-linux-gn[u] /usr/lib/aarch64-linux-gnu/
COPY --from=binaries /usr/lib/x86_64-linux-gn[u] /usr/lib/x86_64-linux-gnu/

# no more sh
COPY --from=binaries /usr/bin/test /bin/sh

ARG TARGETARCH
ENV BPF_DISK_FAILURE_NAME "bpf-disk-failure-${TARGETARCH}"

RUN apt-get update && \
apt-get -y install curl git gcc iproute2 coreutils python3 iptables libelf1

COPY injector_${TARGETARCH} /usr/local/bin/chaos-injector
COPY injector_${TARGETARCH} /usr/local/bin/injector

COPY dns_disruption_resolver.py /usr/local/bin/dns_disruption_resolver.py
COPY ebpf/ /usr/local/bin/

# create a symlink to not break if anyone used explicitly injector somewhere
RUN ln -s /usr/local/bin/chaos-injector /usr/local/bin/injector

ENTRYPOINT ["/usr/local/bin/chaos-injector"]

LABEL baseimage.os="debian"
LABEL baseimage.isgbi="custom"
LABEL baseimage.name="gcr.io/distroless/python3-debian11:latest"

ARG BUILDSTAMP
LABEL baseimage.buildstamp="${BUILDSTAMP}"
18 changes: 14 additions & 4 deletions bin/manager/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
FROM golang:1.18-alpine

RUN addgroup -S appgroup && adduser -S appuser -G appgroup
USER appuser
ARG BUILDGOVERSION
FROM golang:${BUILDGOVERSION} as go
FROM gcr.io/distroless/base-debian11:nonroot

ARG TARGETARCH
COPY manager_${TARGETARCH} /usr/local/bin/manager

# COPY go runtime as it's needed by ddmark to dynamically load structs and their associated markers
COPY --from=go /usr/local/go /usr/local/go/
COPY --from=go /usr/local/go/bin /usr/local/bin/

ENTRYPOINT ["/usr/local/bin/manager"]

LABEL baseimage.os="debian"
LABEL baseimage.isgbi="custom"
LABEL baseimage.name="gcr.io/distroless/base-debian11:nonroot"

ARG BUILDSTAMP
LABEL baseimage.buildstamp="${BUILDSTAMP}"
Loading

0 comments on commit f149976

Please sign in to comment.