Use karapace.io schema registry

infrastructure/airy.conf.all

@@ -6,7 +6,7 @@ global:
 apps:
   kafka:
     brokers: "airy-cp-kafka-headless:9092"
-    schemaRegistryUrl: "http://airy-cp-schema-registry:8081"
+    schemaRegistryUrl: "http://schema-registry:8081"
     commitInterval: 30000
   # Configuration for the Redis database
   redis:

infrastructure/helm-chart/charts/apps/charts/airy-config/values.yaml

@@ -1,13 +1,12 @@
 kafka:
   brokers: "kafka-headless:9092"
-  schemaRegistryUrl: "http://core-schema-registry:8081"
+  schemaRegistryUrl: "http://schema-registry:8081"
   commitInterval: 1000
 redis:
   hostname: "redis-cluster"
   port: 6379
 postgresql:
   endpoint: "postgres:5432"
-  dbName: "admin"
   username: "postgresadmin"  
   password: "changeme"
 sources:

infrastructure/helm-chart/charts/kafka/charts/schema-registry/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 appVersion: "1.0"
-description: A Helm chart for Confluent Schema Registry on Kubernetes
+description: A Helm chart for the karapace.io Schema Registry on Kubernetes
 name: schema-registry
-version: 0.1.0
+version: 0.1.0

infrastructure/helm-chart/charts/kafka/charts/schema-registry/templates/deployment.yaml

@@ -5,27 +5,25 @@ apiVersion: apps/v1beta2
 {{- end }}
 kind: Deployment
 metadata:
-  name: {{ template "cp-schema-registry.fullname" . }}
+  name: schema-registry
   labels:
-    app: {{ template "cp-schema-registry.name" . }}
-    chart: {{ template "cp-schema-registry.chart" . }}
+    app: schema-registry
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
 spec:
   replicas: {{ .Values.replicaCount }}
   selector:
     matchLabels:
-      app: {{ template "cp-schema-registry.name" . }}
+      app: schema-registry
       release: {{ .Release.Name }}
   template:
     metadata:
       labels:
-        app: {{ template "cp-schema-registry.name" . }}
+        app: schema-registry
         release: {{ .Release.Name }}
     spec:
       containers:
-        - name: {{ template "cp-schema-registry.name" . }}-server
-          image: "{{ .Values.image }}:{{ .Values.imageTag }}"
+        - name: schema-registry-server
+          image: "{{ .Values.global.containerRegistry}}/{{ .Values.image }}:{{ .Values.global.appImageTag }}"
           imagePullPolicy: "{{ .Values.imagePullPolicy }}"
           ports:
             - name: schema-registry
@@ -34,43 +32,9 @@ spec:
           resources:
 {{ toYaml .Values.resources | indent 12 }}
           env:
-          - name: SCHEMA_REGISTRY_HOST_NAME
+          - name: KARAPACE_ADVERTISED_HOSTNAME
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
-          - name: SCHEMA_REGISTRY_LISTENERS
-            value: http://0.0.0.0:{{ .Values.servicePort }}
-          - name: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS
-            value: {{ template "cp-schema-registry.kafka.bootstrapServers" . }}
-          - name: SCHEMA_REGISTRY_KAFKASTORE_GROUP_ID
-            value: {{ template "cp-schema-registry.groupId" . }}
-          - name: SCHEMA_REGISTRY_MASTER_ELIGIBILITY
-            value: "true"
-          - name: SCHEMA_REGISTRY_HEAP_OPTS
-            value: "{{ .Values.heapOptions }}"
-          {{ range $configName, $configValue := .Values.configurationOverrides }}
-          - name: SCHEMA_REGISTRY_{{ $configName | replace "." "_" | upper }}
-            value: {{ $configValue | quote }}
-          {{ end }}
-          {{- range $key, $value := .Values.customEnv }}
-          - name: {{ $key | quote }}
-            value: {{ $value | quote }}
-          {{- end }}
-          {{- if .Values.schemaRegistryOpts }}
-          - name: SCHEMA_REGISTRY_OPTS
-            value: "{{ .Values.schemaRegistryOpts }}"
-          {{- end }}
-      {{- if .Values.imagePullSecrets }}
-      imagePullSecrets:
-{{ toYaml .Values.imagePullSecrets | indent 8 }}
-      {{- end }}
-      volumes:
-      {{- if .Values.nodeSelector }}
-      nodeSelector:
-{{ toYaml .Values.nodeSelector | indent 8 }}
-      {{- end }}
-      {{- if .Values.tolerations }}
-      tolerations:
-{{ toYaml .Values.tolerations | indent 8 }}
-      {{- end }}
-
+          - name: KARAPACE_BOOTSTRAP_URI
+            value: {{ .Values.kafka.bootstrapServers }}

infrastructure/helm-chart/charts/kafka/charts/schema-registry/templates/service.yaml

@@ -1,16 +1,14 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ template "cp-schema-registry.fullname" . }}
+  name: schema-registry
   labels:
-    app: {{ template "cp-schema-registry.name" . }}
-    chart: {{ template "cp-schema-registry.chart" . }}
+    app: schema-registry
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
 spec:
   ports:
     - name: schema-registry
       port: {{ .Values.servicePort }}
   selector:
-    app: {{ template "cp-schema-registry.name" . }}
+    app: schema-registry
     release: {{ .Release.Name }}

infrastructure/helm-chart/charts/kafka/charts/schema-registry/values.yaml

@@ -1,15 +1,6 @@
 replicaCount: 0
-image: confluentinc/cp-schema-registry
-imageTag: 5.5.0
-imagePullPolicy: IfNotPresent
-imagePullSecrets:
-configurationOverrides: {}
-customEnv: {}
+image: infrastructure/schema-registry
+imagePullPolicy: Always
 servicePort: 8081
-heapOptions: "-Xms512M -Xmx512M"
 kafka:
-  bootstrapServers: ""
-resources: {}
-podAnnotations: {}
-nodeSelector: {}
-tolerations: []
+  bootstrapServers: kafka-headless:9092

infrastructure/images/schema-registry/.gitignore

@@ -0,0 +1 @@
+karapace*egg

infrastructure/images/schema-registry/Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.9-slim-buster
+
+ARG SCHEMA_REGISTRY_VERSION
+ENV SCHEMA_REGISTRY_VERSION=$SCHEMA_REGISTRY_VERSION
+
+LABEL maintainer "https://github.com/airyhq"
+
+LABEL name="schema-registry" version=${SCHEMA_REGISTRY_VERSION}
+
+RUN apt-get update && apt-get install -y make gcc git
+
+ADD requirements.txt .
+
+RUN pip3 install -r requirements.txt
+
+ADD ${SCHEMA_REGISTRY_VERSION}.egg .
+
+RUN easy_install ${SCHEMA_REGISTRY_VERSION}.egg
+
+ADD config.json .
+
+CMD karapace config.json

infrastructure/images/schema-registry/Makefile

@@ -0,0 +1,10 @@
+build:
+	docker build -t schema-registry . --build-arg SCHEMA_REGISTRY_VERSION=${SCHEMA_REGISTRY_VERSION}
+
+release-beta: build
+	docker tag schema-registry ghcr.io/airyhq/infrastructure/schema-registry:beta
+	docker push ghcr.io/airyhq/infrastructure/schema-registry:beta
+
+release: build
+	docker tag schema-registry ghcr.io/airyhq/infrastructure/schema-registry:release
+	docker push ghcr.io/airyhq/infrastructure/schema-registry:release

infrastructure/images/schema-registry/config.json

@@ -0,0 +1,16 @@
+{
+  "karapace_registry": true,
+  "client_id": "sr-1",
+  "compatibility": "FULL",
+  "group_id": "schema-registry",
+  "log_level": "INFO",
+  "port": 8081,
+  "host": "0.0.0.0",
+  "master_eligibility": true,
+  "replication_factor": 1,
+  "security_protocol": "PLAINTEXT",
+  "ssl_cafile": null,
+  "ssl_certfile": null,
+  "ssl_keyfile": null,
+  "topic_name": "_schemas"
+}

infrastructure/images/schema-registry/requirements.txt

@@ -0,0 +1,13 @@
+git+https://github.com/aiven/avro.git@skip-namespace-validation#subdirectory=lang/py3/
+git+git://github.com/aiven/kafka-python.git@39a372538c6bdfbfcb7852c36bed5de61e34bad8
+yapf==0.30.0
+aiohttp-socks==0.3.4
+pylint==2.4.4
+pytest==5.4.1
+requests==2.23.0
+pre-commit>=2.2.0
+aiokafka==0.6.0
+jsonschema==3.2.0
+pycodestyle==2.6.0
+isort==4.3.10
+lz4==3.0.2

infrastructure/scripts/conf.sh

@@ -31,16 +31,16 @@ fi
 
 helm upgrade core ~/airy-core/helm-chart/ --set global.appImageTag=${AIRY_VERSION} --version 0.5.0 --timeout 1000s > /dev/null 2>&1
 
-kubectl scale deployment core-schema-registry --replicas=1
+kubectl scale deployment schema-registry --replicas=1
 
 wait-for-running-pod startup-helper
-wait-for-service startup-helper core-schema-registry 8081 15 Schema-registry
+wait-for-service startup-helper schema-registry 8081 15 "Schema registry"
 
 kubectl scale deployment -l type=api --replicas=1
 kubectl scale deployment -l type=sources-chatplugin --replicas=1
 kubectl scale deployment -l type=frontend --replicas=1
 
-wait-for-service startup-helper api-auth 80 10 Airy-auth
+wait-for-service startup-helper api-auth 80 10 api-auth
 
 kubectl scale deployment -l type=sources-twilio --replicas=1
 kubectl scale deployment -l type=sources-google --replicas=1

infrastructure/scripts/status.sh

@@ -9,7 +9,7 @@ kubectl run startup-helper --image busybox --command -- /bin/sh -c "tail -f /dev
 
 wait-for-ingress-service
 wait-for-running-pod startup-helper
-wait-for-service startup-helper api-auth 80 10 Airy-auth
+wait-for-service startup-helper api-auth 80 10 api-auth
 
 FACEBOOK_WEBHOOK_PUBLIC_URL=`kubectl get configmap sources-config -o jsonpath='{.data.FACEBOOK_WEBHOOK_PUBLIC_URL}'`
 GOOGLE_WEBHOOK_PUBLIC_URL=`kubectl get configmap sources-config -o jsonpath='{.data.GOOGLE_WEBHOOK_PUBLIC_URL}'`

infrastructure/scripts/trigger/start.sh

@@ -16,15 +16,15 @@ kubectl scale statefulset redis-cluster --replicas=1
 wait-for-service startup-helper redis-cluster 6379 10 Redis
 kubectl scale deployment postgres --replicas=1
 wait-for-service startup-helper postgres 5432 10 Postgres
-kubectl scale deployment core-schema-registry --replicas=1
-wait-for-service startup-helper core-schema-registry 8081 15 Schema-registry
+kubectl scale deployment schema-registry --replicas=1
+wait-for-service startup-helper schema-registry 8081 15 "Schema registry"
 
 echo "Starting up Airy Core Platform appplications"
 kubectl scale deployment -l type=api --replicas=1
 kubectl scale deployment -l type=sources-chatplugin --replicas=1
 kubectl scale deployment -l type=frontend --replicas=1
 
-wait-for-service startup-helper api-auth 80 10 Airy-auth
+wait-for-service startup-helper api-auth 80 10 api-auth
 
 kubectl scale deployment -l type=sources-twilio --replicas=1
 kubectl scale deployment -l type=sources-google --replicas=1

infrastructure/scripts/trigger/stop.sh

@@ -11,7 +11,7 @@ kubectl scale deployment -l type=sources-chatplugin --replicas=0
 kubectl scale deployment -l type=webhook --replicas=0
 kubectl scale deployment -l type=sources-chatplugin --replicas=0
 kubectl scale deployment -l type=api --replicas=0
-kubectl scale deployment core-schema-registry --replicas=0
+kubectl scale deployment schema-registry --replicas=0
 kubectl scale statefulset kafka --replicas=0
 kubectl scale statefulset zookeeper --replicas=0
 kubectl scale deployment postgres --replicas=0