diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1b4afc2e --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +dev/data diff --git a/Makefile b/Makefile index 47af2027..86a77188 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,8 @@ GITREF = `git rev-list HEAD |head -n 1` CI_COMMIT_REF_SLUG = `git rev-parse --abbrev-ref HEAD` - -export DOCKER_BUILDKIT=1 -export COMPOSE_DOCKER_CLI_BUILD=1 +NAME=kind +KUBECONFIG:=~/.kube/kind-config-$(NAME) dev: @ cd dev/; docker-compose up --build @@ -14,3 +13,16 @@ build-dev-image: build-image: @ docker build -t "liftbridge/liftbridge:$(CI_COMMIT_REF_SLUG)" -t "liftbridge/liftbridge:$(GITREF)" -f Dockerfile . + +kind-up: + kind create cluster --config k8s/dev/kind.yaml --name=$(NAME) + +kind-down: + kind delete cluster --name=$(NAME) + +kind-apply: + @ KUBECONFIG=$(KUBECONFIG) kubectl apply -R -f k8s/dev/nats-operator.yaml + @ KUBECONFIG=$(KUBECONFIG) skaffold dev -p dev + +kind-export: + @echo export KUBECONFIG="$$(kind get kubeconfig-path --name="$(NAME)")" diff --git a/dev/Dockerfile.compose b/dev/Dockerfile.compose index 67c6b42b..0664a109 100644 --- a/dev/Dockerfile.compose +++ b/dev/Dockerfile.compose @@ -6,6 +6,7 @@ ENV GO111MODULE on ENV CGO_ENABLED 1 ENV GOARCH amd64 ENV GOOS linux +RUN go mod download RUN go build -mod=readonly -race -o liftbridge FROM debian:stretch-slim diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 14dbef19..ce540b71 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -31,7 +31,7 @@ services: '--port=9292', '--level=debug', '--nats-servers=nats:4222', - '--raft-bootstrap-peers=server-1,server-2', + '--raft-bootstrap-seed', '--id=server-1' ] environment: @@ -87,8 +87,6 @@ services: - overlay volumes: - ./data/data3:/data3 -volumes: - build-cache: networks: overlay: driver: bridge diff --git a/k8s/base/config.yaml b/k8s/base/config.yaml new file mode 100644 index 00000000..52adb8a1 --- /dev/null +++ b/k8s/base/config.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +data: + liftbridge.conf: | + listen: 0.0.0.0:9292 + # data.dir: /tmp/liftbridge/server-2 + # log.level: debug + + clustering { + min.insync.replicas: 1 + } +kind: ConfigMap +metadata: + labels: + app: liftbridge + component: config + environment: dev + k8s-app: liftbridge + name: config diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml new file mode 100644 index 00000000..1acd5193 --- /dev/null +++ b/k8s/base/kustomization.yaml @@ -0,0 +1,7 @@ +--- +namespace: default + +resources: + - svc.yaml + - statefulset.yaml + - config.yaml diff --git a/k8s/base/statefulset.yaml b/k8s/base/statefulset.yaml new file mode 100644 index 00000000..ebb1e98b --- /dev/null +++ b/k8s/base/statefulset.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: liftbridge +spec: + serviceName: "liftbridge-headless" + replicas: 3 + selector: + matchLabels: + app: liftbridge + template: + metadata: + labels: + app: liftbridge + spec: + containers: + - name: liftbridge + image: liftbridge + ports: + - name: grpc + containerPort: 9292 + volumeMounts: + - name: liftbridge-data + mountPath: /data + - mountPath: /etc/liftbridge.conf + name: config + subPath: liftbridge.conf + volumes: + - configMap: + name: config + name: config + volumeClaimTemplates: + - metadata: + name: liftbridge-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi diff --git a/k8s/base/svc.yaml b/k8s/base/svc.yaml new file mode 100644 index 00000000..f6314ffb --- /dev/null +++ b/k8s/base/svc.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: liftbridge-headless +spec: + selector: + app: liftbridge + clusterIP: "None" + ports: + - name: grpc + protocol: TCP + port: 9292 + targetPort: grpc +--- +apiVersion: v1 +kind: Service +metadata: + name: liftbridge +spec: + selector: + app: liftbridge + ports: + - name: grpc + protocol: TCP + port: 9292 + targetPort: grpc diff --git a/k8s/dev/Dockerfile.kind b/k8s/dev/Dockerfile.kind new file mode 100644 index 00000000..dcf535e5 --- /dev/null +++ b/k8s/dev/Dockerfile.kind @@ -0,0 +1,25 @@ +FROM golang:1.13-alpine as build-base +RUN apk update && apk upgrade && \ + apk add --no-cache bash git openssh make bzr +WORKDIR /workspace +ENV GO111MODULE on + +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY server/ server/ + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=readonly -o liftbridge + +FROM alpine:latest +RUN apk update && apk add --no-cache bash +COPY --from=build-base /workspace/liftbridge /usr/local/bin/liftbridge +COPY k8s/dev/entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/dev/Dockerfile.natsboard b/k8s/dev/Dockerfile.natsboard similarity index 81% rename from dev/Dockerfile.natsboard rename to k8s/dev/Dockerfile.natsboard index dc52c558..f7e2dfcc 100644 --- a/dev/Dockerfile.natsboard +++ b/k8s/dev/Dockerfile.natsboard @@ -3,3 +3,4 @@ FROM mhart/alpine-node:12 RUN npm install natsboard -g RUN apk add --no-cache netcat-openbsd COPY wait-for.sh /wait-for.sh +ENTRYPOINT ["/wait-for.sh"] diff --git a/k8s/dev/entrypoint.sh b/k8s/dev/entrypoint.sh new file mode 100755 index 00000000..23023c42 --- /dev/null +++ b/k8s/dev/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ "$HOSTNAME" = "liftbridge-0" ] +then + echo "Running in liftbridge-0 as a bootstrap seed...\n" + liftbridge --data-dir=/data --config /etc/liftbridge.conf --level=debug --nats-servers=nats://nats.liftbridge.svc:4222 --raft-bootstrap-seed --id="$HOSTNAME" +else + echo "Running in $HOSTNAME...\n" + liftbridge --data-dir=/data --config /etc/liftbridge.conf --level=debug --nats-servers=nats://nats.liftbridge.svc:4222 --raft-bootstrap-peers=liftbridge-0 --id="$HOSTNAME" +fi diff --git a/k8s/dev/kind.yaml b/k8s/dev/kind.yaml new file mode 100644 index 00000000..30fe0352 --- /dev/null +++ b/k8s/dev/kind.yaml @@ -0,0 +1,23 @@ +--- +kind: Cluster +apiVersion: kind.sigs.k8s.io/v1alpha3 +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 32767 + hostPort: 9292 + listenAddress: "0.0.0.0" + - containerPort: 32767 + hostPort: 9293 + listenAddress: "0.0.0.0" + - containerPort: 32767 + hostPort: 9294 + listenAddress: "0.0.0.0" + - containerPort: 32766 + hostPort: 3000 + listenAddress: "0.0.0.0" + - containerPort: 32765 + hostPort: 4222 + listenAddress: "0.0.0.0" + - role: worker + - role: worker diff --git a/k8s/dev/kustomization.yaml b/k8s/dev/kustomization.yaml new file mode 100644 index 00000000..302c3958 --- /dev/null +++ b/k8s/dev/kustomization.yaml @@ -0,0 +1,11 @@ +--- +namespace: liftbridge + +bases: + - ../base + +resources: + - namespace.yaml + - nats.yaml + - natsboard.yaml + - svc.yaml diff --git a/k8s/dev/namespace.yaml b/k8s/dev/namespace.yaml new file mode 100644 index 00000000..bc294fd6 --- /dev/null +++ b/k8s/dev/namespace.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: liftbridge + labels: + name: liftbridge + environment: dev diff --git a/k8s/dev/nats-operator.yaml b/k8s/dev/nats-operator.yaml new file mode 100644 index 00000000..a29292b0 --- /dev/null +++ b/k8s/dev/nats-operator.yaml @@ -0,0 +1,199 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nats-io + labels: + name: nats-io + environment: dev +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: nats + component: nats + k8s-app: nats + name: nats-operator + namespace: nats-io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: nats + component: nats + k8s-app: nats + name: nats-server + namespace: nats-io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: nats + component: nats + k8s-app: nats + name: nats-server + namespace: nats-io +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: nats + component: nats + k8s-app: nats + name: nats-operator + namespace: nats-io +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "watch", "get", "update", "list"] + - apiGroups: ["nats.io"] + resources: ["natsclusters", "natsserviceroles"] + verbs: ['*'] + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "watch", "get", "patch", "update", "delete", "list"] + - apiGroups: [""] + resources: ["services"] + verbs: ["create", "watch", "get", "patch", "update", "delete", "list"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "watch", "get", "update", "delete", "list"] + - apiGroups: [""] + resources: ["pods/exec", "pods/log", "serviceaccounts/token", "events"] + verbs: ['*'] + - apiGroups: [""] + resources: ["namespaces", "serviceaccounts"] + verbs: ["list", "get", "watch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["create", "watch", "get", "update", "delete", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: nats + component: nats + k8s-app: nats + name: nats-server-binding + namespace: nats-io +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nats-server +subjects: + - kind: ServiceAccount + name: nats-server + namespace: nats-io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: nats + component: nats + k8s-app: nats + name: nats-operator-binding + namespace: nats-io +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nats-operator +subjects: + - kind: ServiceAccount + name: nats-operator + namespace: nats-io +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + annotations: + sidecar.istio.io/inject: "false" + labels: + app: nats + component: nats + k8s-app: nats + name: nats-operator + namespace: nats-io +spec: + replicas: 1 + selector: + matchLabels: + app: nats + component: nats + k8s-app: nats + name: nats-operator + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + labels: + app: nats + component: nats + k8s-app: nats + name: nats-operator + spec: + containers: + - args: + - nats-operator + - --feature-gates=ClusterScoped=true + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: connecteverything/nats-operator:0.6.0 + imagePullPolicy: Always + name: nats-operator + ports: + - containerPort: 8080 + name: readyz + readinessProbe: + httpGet: + path: /readyz + port: readyz + initialDelaySeconds: 15 + timeoutSeconds: 3 + serviceAccountName: nats-operator +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: natsclusters.nats.io +spec: + group: nats.io + names: + kind: NatsCluster + listKind: NatsClusterList + plural: natsclusters + singular: natscluster + shortNames: + - nats + scope: Namespaced + version: v1alpha2 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: natsserviceroles.nats.io +spec: + group: nats.io + names: + kind: NatsServiceRole + listKind: NatsServiceRoleList + plural: natsserviceroles + singular: natsservicerole + scope: Namespaced + version: v1alpha2 diff --git a/k8s/dev/nats.yaml b/k8s/dev/nats.yaml new file mode 100644 index 00000000..bb1e241a --- /dev/null +++ b/k8s/dev/nats.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: nats.io/v1alpha2 +kind: NatsCluster +metadata: + labels: + app: liftbridge + component: nats + environment: dev + k8s-app: liftbridge + name: nats +spec: + natsConfig: + writeDeadline: 5s + pod: + annotations: + sidecar.istio.io/inject: "false" + enableConfigReload: true + enableMetrics: true + metricsImage: synadia/prometheus-nats-exporter + metricsImageTag: 0.5.0 + reloaderImage: connecteverything/nats-server-config-reloader + reloaderImagePullPolicy: IfNotPresent + reloaderImageTag: 0.6.0 + size: 3 + version: 2.1.0 +--- +apiVersion: v1 +kind: Service +metadata: + name: nats-external +spec: + selector: + app: nats + nats_cluster: nats + type: NodePort + ports: + - name: nats + protocol: TCP + port: 4222 + nodePort: 32765 + targetPort: 4222 diff --git a/k8s/dev/natsboard.yaml b/k8s/dev/natsboard.yaml new file mode 100644 index 00000000..4faa8062 --- /dev/null +++ b/k8s/dev/natsboard.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: natsboard + labels: + app: natsboard +spec: + replicas: 1 + selector: + matchLabels: + app: natsboard + template: + metadata: + labels: + app: natsboard + spec: + containers: + - name: natsboard + image: natsboard + command: ['/usr/bin/natsboard'] + env: + - name: "WAIT_FOR_HOST" + value: nats-mgmt.default.svc + - name: "WAIT_FOR_PORT" + value: "8222" + - name: "NATS_MON_URL" + value: "http://nats-mgmt.default.svc:8222" + ports: + - name: http + containerPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: natsboard-external +spec: + selector: + app: natsboard + type: NodePort + ports: + - name: http + protocol: TCP + port: 3000 + nodePort: 32766 + targetPort: http diff --git a/k8s/dev/svc.yaml b/k8s/dev/svc.yaml new file mode 100644 index 00000000..f8916ff0 --- /dev/null +++ b/k8s/dev/svc.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: liftbridge-external +spec: + selector: + app: liftbridge + type: NodePort + ports: + - name: grpc + protocol: TCP + port: 9292 + nodePort: 32767 + targetPort: grpc diff --git a/dev/wait-for.sh b/k8s/dev/wait-for.sh similarity index 100% rename from dev/wait-for.sh rename to k8s/dev/wait-for.sh diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 00000000..12f2cf0b --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: skaffold/v1beta17 +kind: Config +build: + artifacts: + - image: liftbridge + context: . + docker: + dockerfile: k8s/dev/Dockerfile.kind + - image: natsboard + context: k8s/dev/ + docker: + dockerfile: Dockerfile.natsboard + tagPolicy: + sha256: {} + local: + push: false + useBuildkit: true +deploy: + kustomize: + path: ./k8s/ + +profiles: + - name: dev + build: + local: + push: false + deploy: + kubeContext: kubernetes-admin@kind + kustomize: + path: ./k8s/dev