add kafka HA + notebook

.devcontainer/k3d.yaml

@@ -1,39 +1,60 @@
-apiVersion: k3d.io/v1alpha2
+apiVersion: k3d.io/v1alpha5
 kind: Simple
 servers: 1
+agents: 3
 network: k3d
 kubeAPI:
-  hostIP: "0.0.0.0"
+  hostIP: 0.0.0.0
   hostPort: "6443"
 volumes:
   - volume: /prometheus:/prometheus
     nodeFilters:
-      - server[0]
+      - server:0
+      - agent:*
   - volume: /grafana:/grafana
     nodeFilters:
-      - server[0]
+      - server:0
+      - agent:*
 ports:
   - port: 30000:30000
     nodeFilters:
-      - server[0]
+      - server:0
+      - agent:*
   - port: 30080:30080
     nodeFilters:
-      - server[0]
+      - server:0
+      - agent:*
   - port: 31080:31080
     nodeFilters:
-      - server[0]
+      - server:0
+      - agent:*
   - port: 32000:32000
     nodeFilters:
-      - server[0]
+      - server:0
+      - agent:*
 
 options:
   k3d:
-    wait: true
+    wait: false
     timeout: "60s"
   k3s:
-    extraServerArgs:
-      - --tls-san=127.0.0.1
-    extraAgentArgs: []
+    extraArgs:
+      - arg: --tls-san=127.0.0.1
+        nodeFilters:
+          - server:0
+      - arg: --no-deploy=traefik
+        nodeFilters:
+          - server:0
+    nodeLabels:
+      - label: topology.kubernetes.io/zone=zone-a
+        nodeFilters:
+          - agent:0
+      - label: topology.kubernetes.io/zone=zone-b
+        nodeFilters:
+          - agent:1
+      - label: topology.kubernetes.io/zone=zone-c
+        nodeFilters:
+          - agent:2
   kubeconfig:
     updateDefaultKubeconfig: true
     switchCurrentContext: true

cli/.kic/commands/build/make

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# validate directories
+if [ ! -f "$REPO_BASE/.devcontainer/k3d.yaml" ]
+then
+        echo "File not found - \"$REPO_BASE\"/.devcontainer/k3d.yaml"
+        exit 1
+fi
+
+kic cluster delete
+
+echo ""
+echo "Creating cluster ..."
+
+k3d cluster create \
+    --registry-use k3d-registry.localhost:5500 \
+    --config "$REPO_BASE/.devcontainer/k3d.yaml"

cli/.kic/commands/make/cluster

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# validate directories
+if [ ! -f "$REPO_BASE/.devcontainer/k3d.yaml" ]
+then
+        echo "File not found - \"$REPO_BASE\"/.devcontainer/k3d.yaml"
+        exit 1
+fi
+
+kic cluster delete
+
+echo ""
+echo "Creating cluster ..."
+
+k3d cluster create \
+    --registry-create k3d-registry.localhost:0.0.0.0:5500 \
+    --config "$REPO_BASE/.devcontainer/k3d.yaml" \
+    --network k3d \
+    --wait false

cli/.kic/root.yaml

@@ -7,6 +7,14 @@ short: Kubernetes in Codespaces CLI
 long: "Kubernetes in Codespaces CLI\n\n  A dynamic CLI for automating many Kubernetes development tasks"
 
 commands:
+  - name: make
+    short: make cluster
+
+    scriptCommands:
+    - name: cluster
+      short: Build the IMDb app
+      path: make/cluster
+
   - name: build
     short: Build local apps
 
@@ -18,3 +26,5 @@ commands:
     - name: webv
       short: Build WebValidate
       path: build/webv
+
+

deploy/bootstrap/kafka/1-service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka-svc
+  labels:
+    app: kafka-app
+spec:
+  clusterIP: None
+  ports:
+    - name: '9092'
+      port: 9092
+      protocol: TCP
+      targetPort: 9092
+  selector:
+    app: kafka-app

deploy/bootstrap/kafka/2-sts.yaml

@@ -0,0 +1,57 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: kafka
+  labels:
+    app: kafka-app
+spec:
+  serviceName: kafka-svc
+  replicas: 3
+  selector:
+    matchLabels:
+      app: kafka-app
+  template:
+    metadata:
+      labels:
+        app: kafka-app
+    spec:
+      topologySpreadConstraints:
+        - maxSkew: 2
+          topologyKey: kubernetes.io/zone
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              app: kafka-app
+      containers:
+        - name: kafka-container
+          image: doughgle/kafka-kraft
+          ports:
+            - containerPort: 9092
+            - containerPort: 9093
+          env:
+            - name: REPLICAS
+              value: '3'
+            - name: SERVICE
+              value: kafka-svc
+            - name: NAMESPACE
+              value: default
+            - name: SHARE_DIR
+              value: /mnt/kafka
+            - name: CLUSTER_ID
+              value: oh-sxaDRTcyAr6pFRbXyzA
+            - name: DEFAULT_REPLICATION_FACTOR
+              value: '3'
+            - name: DEFAULT_MIN_INSYNC_REPLICAS
+              value: '2'
+          volumeMounts:
+            - name: data
+              mountPath: /mnt/kafka
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes:
+          - "ReadWriteOnce"
+        resources:
+          requests:
+            storage: "1Gi"

deploy/bootstrap/kafka/3-pdb.yaml

@@ -0,0 +1,9 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: kafka-pdb
+spec:
+  minAvailable: 2
+  selector:
+    matchLabels:
+      app: kafka-app

run.ipynb

@@ -0,0 +1,124 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "vscode": {
+     "languageId": "shellscript"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[36mINFO\u001b[0m[0000] Using config file /workspaces/ksailnet/.devcontainer/k3d.yaml \n",
+      "\u001b[36mINFO\u001b[0m[0000] Prep: Network                                \n",
+      "\u001b[36mINFO\u001b[0m[0000] Re-using existing network 'k3d' (9289d49b248788855f6c19c2bc796a53d9fe498df6ec3adfc78d7987a9a5af5d) \n",
+      "\u001b[36mINFO\u001b[0m[0000] Created volume 'k3d-k3s-default-images'      \n",
+      "\u001b[36mINFO\u001b[0m[0001] Creating node 'k3d-k3s-default-server-0'     \n",
+      "\u001b[36mINFO\u001b[0m[0001] Creating LoadBalancer 'k3d-k3s-default-serverlb' \n",
+      "\u001b[36mINFO\u001b[0m[0001] Starting cluster 'k3s-default'               \n",
+      "\u001b[36mINFO\u001b[0m[0001] Starting servers...                          \n",
+      "\u001b[36mINFO\u001b[0m[0001] Starting Node 'k3d-k3s-default-server-0'     \n",
+      "\u001b[36mINFO\u001b[0m[0006] Starting agents...                           \n",
+      "\u001b[36mINFO\u001b[0m[0006] Starting helpers...                          \n",
+      "\u001b[36mINFO\u001b[0m[0006] Starting Node 'k3d-k3s-default-serverlb'     \n",
+      "\u001b[36mINFO\u001b[0m[0007] (Optional) Trying to get IP of the docker host and inject it into the cluster as 'host.k3d.internal' for easy access \n",
+      "\u001b[36mINFO\u001b[0m[0011] Successfully added host record to /etc/hosts in 2/2 nodes and to the CoreDNS ConfigMap \n",
+      "\u001b[36mINFO\u001b[0m[0011] Cluster 'k3s-default' created successfully!  \n",
+      "\u001b[36mINFO\u001b[0m[0011] --kubeconfig-update-default=false --> sets --kubeconfig-switch-context=false \n",
+      "\u001b[36mINFO\u001b[0m[0011] You can now use it like this:                \n",
+      "kubectl config use-context k3d-k3s-default\n",
+      "kubectl cluster-info\n"
+     ]
+    }
+   ],
+   "source": [
+    "!#/usr/local/bin/k3d cluster create -c /workspaces/ksailnet/.devcontainer/k3d.yaml"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "vscode": {
+     "languageId": "shellscript"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "305.16s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n",
+      "310.41s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error while executing topic command : Topic 'test-topic' already exists.\n",
+      "[2023-08-25 07:52:13,323] ERROR org.apache.kafka.common.errors.TopicExistsException: Topic 'test-topic' already exists.\n",
+      " (kafka.admin.TopicCommand$)\n",
+      "command terminated with exit code 1\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "317.22s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      ">"
+     ]
+    }
+   ],
+   "source": [
+    "! alias kubectl=\"/usr/local/bin/kubectl\"\n",
+    "! kubectl exec -it kafka-client -- /opt/bitnami/kafka/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server kafka-0.kafka-svc.default.svc.cluster.local:9092 --replication-factor 1 --partitions 1\n",
+    "\n",
+    "! kubectl exec -it kafka-client -- /opt/bitnami/kafka/bin/kafka-console-producer.sh --topic test-topic --bootstrap-server kafka-0.kafka-svc.default.svc.cluster.local:9092\n",
+    "\n",
+    "! kubectl exec -it kafka-client -- /opt/bitnami/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server kafka-0.kafka-svc.default.svc.cluster.local:9092\n",
+    "\n",
+    "! kubectl exec -it kafka-client -- /opt/bitnami/kafka/bin/kafka-topics.sh --describe --topic test-topic --bootstrap-server kafka-0.kafka-svc.default.svc.cluster.local:9092\n",
+    "\n",
+    "! kubectl exec -it kafka-client -- /opt/bitnami/kafka/bin/kafka-topics.sh --alter --topic test-topic --bootstrap-server kafka-0.kafka-svc.default.svc.cluster.local:9092 --partitions 3\n",
+    "\n",
+    "! kubectl exec -it kafka-client -- /opt/bitnami/kafka/bin/kafka-topics.sh --describe --topic test-topic --bootstrap-server kafka-0.kafka-svc.default.svc.cluster.local:9092\n",
+    "\n",
+    "! kubectl exec -it kafka-client -- /opt/bitnami/kafka/bin/kafka-topics.sh --delete --topic test-topic --bootstrap-server kafka-0.kafka-svc.default.svc.cluster.local:9092\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.2"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}