Schema linking

base/cfk-components/confluent-rbac/connect/kafka-connect.yaml

@@ -28,3 +28,5 @@ spec:
         type: bearer
         bearer:
           secretRef: credential
+      tls:
+        enabled: true

base/cfk-components/confluent-rbac/connect/rolebindings/connect-base.yaml

@@ -8,7 +8,7 @@ spec:
   principal:
     name: kafka
     type: user
-  role: SecurityAdmin
+  role: ClusterAdmin
 ---
 apiVersion: platform.confluent.io/v1beta1
 kind: ConfluentRolebinding

base/cfk-components/confluent-rbac/rest-class.yaml

@@ -6,5 +6,5 @@ spec:
   kafkaRest:
     authentication:
       type: bearer
-      bearer:
+      basic:
         secretRef: credential

base/cfk-components/confluent-rbac/secrets/credential.yaml

@@ -1,6 +1,7 @@
 apiVersion: v1
 data:
   bearer.txt: dXNlcm5hbWU9a2Fma2EKcGFzc3dvcmQ9a2Fma2Etc2VjcmV0Cg==
+  basic.txt: dXNlcm5hbWU9a2Fma2EKcGFzc3dvcmQ9a2Fma2Etc2VjcmV0Cg==
   digest.txt: dXNlcm5hbWU9ImthZmthIgpwYXNzd29yZD0ia2Fma2Etc2VjcmV0Ig==
   plain.txt: dXNlcm5hbWU9ImthZmthIgpwYXNzd29yZD0ia2Fma2Etc2VjcmV0Ig==
   ldap.txt: dXNlcm5hbWU9Y249bWRzLGRjPXRlc3QsZGM9Y29tCnBhc3N3b3JkPURldmVsb3BlciEK

incubator/multi-dc-schema-registry-leader-follower/README.md

@@ -0,0 +1,88 @@
+# Basic Deployment
+This example deploys a basic deployment.  No RBAC/LDAP.  Just a single topic 'foobar' is added as part of the pipeline.
+
+
+| Feature          | Enabled | Note                     |
+|:-----------------|:-------:|:-------------------------|
+| Kafka/Zookeeper  |    ✅    |                          |
+| Control Center   |    ✅    |                          |
+| Connect          |    ❌    |                          |
+| Schema Registry  |    ❌    |                          |
+| KSQL             |    ❌    |                          |
+| TLS Encryption   |    ✅    | Self-signed certificates |
+| Authentication   |    ✅    |                          |
+| Authorization    |    ✅    | via LDAP                 |
+
+
+### Deploy CRDs
+Deploy the CRDS using the standard way:
+```shell
+kubectl apply -k ../../base/crds
+```
+### Deploy Confluent Operator and Confluent Services
+Deploy the confluent operator and services:
+```shell
+kubectl apply -k .
+```
+
+### Forward ports locally.
+#### Source Kafka
+Minikube:source:kafka:9071 -> localhost:9071
+#### Source SchemaRegistry
+Minikube:source:schemaregistry:9081 -> localhost:9081
+#### Source Kafka
+Minikube:destination:kafka:9071 -> localhost:9072
+#### Source Kafka
+Minikube:source:schemaregistry:9081 -> localhost:9082
+
+```shell
+
+nohup kubectl port-forward schemaregistry-0 8081:8081 --namespace master \
+&& \
+nohup kubectl port-forward schemaregistry-0 8082:8081 --namespace dogs \
+&& \
+nohup kubectl port-forward schemaregistry-0 8083:8081 --namespace cats 
+
+```
+kubectl port-forward \
+$(kubectl get pods -n source -l statefulset.kubernetes.io/pod-name:kafka-0 -o name) \
+:9071 -n source \ 
+&& \
+kubectl port-forward \
+$(kubectl get pods -n source -l app.kubernetes.io/component=grafana -o name) \
+:9071 -n source \
+&& \
+kubectl port-forward \
+$(kubectl get pods -n source -l app.kubernetes.io/component=grafana -o name) \
+9071 -n source
+&& \
+kubectl port-forward \
+$(kubectl get pods -n source -l app.kubernetes.io/component=grafana -o name) \
+9071 -n source
+
+```
+
+### Query schema Registry
+
+[//]: # (TODO Port forward Production Schema Registry - 8081)
+[//]: # (TODO Port forward Production Kafka Broker - 9092)
+
+```
+curl -k -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \
+  --data '{"schema": "{\"type\": \"string\"}"}' \
+   https://localhost:8081/subjects/im-a-source-schema/versions
+```
+
+
+
+```
+curl -k -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \
+  --data '{"schema": "{\"type\": \"string\"}"}' \
+   https://localhost:8082/subjects/im-a-desgination-schema/versions
+```10.109.90.121
+
+`curl -k -X GET https://10.109.90.121:8081/subjects`
+
+`curl -k -X GET https://localhost:8082/subjects`
+
+#TODO - how will 

incubator/schema-registry-linking/environments/source/source-rest-class.yaml → incubator/multi-dc-schema-registry-leader-follower/environments/cats/cats-rest-class.yaml

@@ -1,9 +1,9 @@
 apiVersion: platform.confluent.io/v1beta1
 kind: KafkaRestClass
 metadata:
-  name: source-kafka-rest
+  name: cats-kafka-rest
 spec:
   kafkaClusterRef:
     name: kafka
   kafkaRest:
-    endpoint: http://kafka.source.svc.cluster.local:8090
+    endpoint: http://kafka.cats.svc.cluster.local:8090

incubator/multi-dc-schema-registry-leader-follower/environments/cats/control-center.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: platform.confluent.io/v1beta1
+kind: ControlCenter
+metadata:
+  name: controlcenter
+spec:
+  configOverrides:
+    server:
+      - confluent.controlcenter.internal.topics.replication=1
+      - confluent.controlcenter.command.topic.replication=1
+      - confluent.monitoring.interceptor.topic.replication=1
+      - confluent.metrics.topic.replication=1

incubator/multi-dc-schema-registry-leader-follower/environments/cats/kafka.yaml

@@ -0,0 +1,20 @@
+apiVersion: platform.confluent.io/v1beta1
+kind: Kafka
+metadata:
+  name: kafka
+spec:
+  replicas: 1
+  configOverrides:
+    server:
+      - confluent.schema.registry.url=http://schemaregistry:8081
+      - confluent.balancer.topic.replication.factor=1
+      - confluent.license.topic.replication.factor=1
+      - confluent.tier.metadata.replication.factor=1
+      - confluent.metadata.topic.replication.factor=1
+      - confluent.metrics.reporter.topic.replicas=1
+      - confluent.balancer.topic.replication.factor=1
+      - confluent.security.event.logger.exporter.kafka.topic.replicas=1
+      - default.replication.factor=1
+      - event.logger.exporter.kafka.topic.replicas=1
+      - offsets.topic.replication.factor=1
+

incubator/multi-dc-schema-registry-leader-follower/environments/cats/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: cats
+resources:
+  - namespace.yaml
+  - ../../../../base/cfk-components/confluent-no-auth-no-tls
+  - sr-linking-topic-demo.yaml
+  - schema-registry.yaml
+  - cats-rest-class.yaml
+patchesStrategicMerge:
+  - kafka.yaml
+  - zookeeper.yaml
+  - control-center.yaml

incubator/schema-registry-linking/environments/source/namespace.yaml → incubator/multi-dc-schema-registry-leader-follower/environments/cats/namespace.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: source
+  name: cats

incubator/multi-dc-schema-registry-leader-follower/environments/cats/schema-registry.yaml

@@ -0,0 +1,19 @@
+apiVersion: platform.confluent.io/v1beta1
+kind: SchemaRegistry
+metadata:
+  name: schemaregistry
+spec:
+  replicas: 1
+  image:
+    application: confluentinc/cp-schema-registry:7.1.0
+    init: confluentinc/confluent-init-container:2.3.0
+  configOverrides:
+    server:
+      - kafkastore.topic.replication.factor=1
+      - leader.eligibility=false
+      - unclean.leader.election.enable=false
+      - schema.registry.group.id=id_schemaregistry_master
+      - kafkastore.topic=_schemas
+  dependencies:
+    kafka:
+      bootstrapEndpoint: http://kafka.master.svc.cluster.local:9071

incubator/schema-registry-linking/environments/source/sr-linking-topic-demo.yaml → incubator/multi-dc-schema-registry-leader-follower/environments/cats/sr-linking-topic-demo.yaml

@@ -1,11 +1,11 @@
 apiVersion: platform.confluent.io/v1beta1
 kind: KafkaTopic
 metadata:
-  name: sr-linking-topic-demo-sources
+  name: sr-linking-topic-demo-cats
 spec:
-  replicas: 3
+  replicas: 1
   partitionCount: 4
   configs:
     cleanup.policy: "delete"
   kafkaRestClassRef:
-    name: source-kafka-rest
+    name: cats-kafka-rest

incubator/multi-dc-schema-registry-leader-follower/environments/cats/zookeeper.yaml

@@ -0,0 +1,6 @@
+apiVersion: platform.confluent.io/v1beta1
+kind: Zookeeper
+metadata:
+  name: zookeeper
+spec:
+  replicas: 1

incubator/multi-dc-schema-registry-leader-follower/environments/dogs/control-center.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: platform.confluent.io/v1beta1
+kind: ControlCenter
+metadata:
+  name: controlcenter
+spec:
+  configOverrides:
+    server:
+      - confluent.controlcenter.internal.topics.replication=1
+      - confluent.controlcenter.command.topic.replication=1
+      - confluent.monitoring.interceptor.topic.replication=1
+      - confluent.metrics.topic.replication=1

incubator/schema-registry-linking/environments/destination/cluster-link-rest-class.yaml → incubator/multi-dc-schema-registry-leader-follower/environments/dogs/dogs-rest-class.yaml

@@ -1,9 +1,9 @@
 apiVersion: platform.confluent.io/v1beta1
 kind: KafkaRestClass
 metadata:
-  name: destination-kafka-rest
+  name: dogs-kafka-rest
 spec:
   kafkaClusterRef:
     name: kafka
   kafkaRest:
-    endpoint: http://kafka.destination.svc.cluster.local:8090
+    endpoint: http://kafka.dogs.svc.cluster.local:8090

incubator/multi-dc-schema-registry-leader-follower/environments/dogs/kafka.yaml

@@ -0,0 +1,20 @@
+apiVersion: platform.confluent.io/v1beta1
+kind: Kafka
+metadata:
+  name: kafka
+spec:
+  replicas: 1
+  configOverrides:
+    server:
+      - confluent.schema.registry.url=http://schemaregistry:8081
+      - confluent.balancer.topic.replication.factor=1
+      - confluent.license.topic.replication.factor=1
+      - confluent.tier.metadata.replication.factor=1
+      - confluent.metadata.topic.replication.factor=1
+      - confluent.metrics.reporter.topic.replicas=1
+      - confluent.balancer.topic.replication.factor=1
+      - confluent.security.event.logger.exporter.kafka.topic.replicas=1
+      - default.replication.factor=1
+      - event.logger.exporter.kafka.topic.replicas=1
+      - offsets.topic.replication.factor=1
+

incubator/multi-dc-schema-registry-leader-follower/environments/dogs/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: dogs
+resources:
+  - namespace.yaml
+  - ../../../../base/cfk-components/confluent-no-auth-no-tls
+  - sr-linking-topic-demo.yaml
+  - schema-registry.yaml
+  - dogs-rest-class.yaml
+patchesStrategicMerge:
+  - kafka.yaml
+  - zookeeper.yaml
+  - control-center.yaml

incubator/schema-registry-linking/environments/destination/namespace.yaml → incubator/multi-dc-schema-registry-leader-follower/environments/dogs/namespace.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: destination
+  name: dogs

incubator/multi-dc-schema-registry-leader-follower/environments/dogs/schema-registry.yaml

@@ -0,0 +1,19 @@
+apiVersion: platform.confluent.io/v1beta1
+kind: SchemaRegistry
+metadata:
+  name: schemaregistry
+spec:
+  replicas: 1
+  image:
+    application: confluentinc/cp-schema-registry:7.1.0
+    init: confluentinc/confluent-init-container:2.3.0
+  configOverrides:
+    server:
+      - kafkastore.topic.replication.factor=1
+      - leader.eligibility=false
+      - unclean.leader.election.enable=false
+      - schema.registry.group.id=id_schemaregistry_master
+      - kafkastore.topic=_schemas
+  dependencies:
+    kafka:
+      bootstrapEndpoint: http://kafka.master.svc.cluster.local:9071

incubator/schema-registry-linking/environments/destination/sr-linking-topic-demo.yaml → incubator/multi-dc-schema-registry-leader-follower/environments/dogs/sr-linking-topic-demo.yaml

@@ -1,11 +1,11 @@
 apiVersion: platform.confluent.io/v1beta1
 kind: KafkaTopic
 metadata:
-  name: sr-linking-topic-demo-destination
+  name: sr-linking-topic-demo-dogs
 spec:
-  replicas: 3
+  replicas: 1
   partitionCount: 4
   configs:
     cleanup.policy: "delete"
   kafkaRestClassRef:
-    name: destination-kafka-rest
+    name: dogs-kafka-rest

incubator/multi-dc-schema-registry-leader-follower/environments/dogs/zookeeper.yaml

@@ -0,0 +1,6 @@
+apiVersion: platform.confluent.io/v1beta1
+kind: Zookeeper
+metadata:
+  name: zookeeper
+spec:
+  replicas: 1

incubator/multi-dc-schema-registry-leader-follower/environments/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - master
+  - dogs
+  - cats
+

incubator/multi-dc-schema-registry-leader-follower/environments/master/control-center.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: platform.confluent.io/v1beta1
+kind: ControlCenter
+metadata:
+  name: controlcenter
+spec:
+  configOverrides:
+    server:
+      - confluent.controlcenter.internal.topics.replication=1
+      - confluent.controlcenter.command.topic.replication=1
+      - confluent.monitoring.interceptor.topic.replication=1
+      - confluent.metrics.topic.replication=1

incubator/multi-dc-schema-registry-leader-follower/environments/master/kafka.yaml

@@ -0,0 +1,23 @@
+apiVersion: platform.confluent.io/v1beta1
+kind: Kafka
+metadata:
+  name: kafka
+spec:
+  replicas: 1
+  configOverrides:
+    server:
+      - confluent.schema.registry.url=http://schemaregistry:8081
+      - confluent.balancer.topic.replication.factor=1
+      - confluent.license.topic.replication.factor=1
+      - confluent.tier.metadata.replication.factor=1
+      - confluent.metadata.topic.replication.factor=1
+      - confluent.metrics.reporter.topic.replicas=1
+      - confluent.balancer.topic.replication.factor=1
+      - confluent.security.event.logger.exporter.kafka.topic.replicas=1
+      - default.replication.factor=1
+      - event.logger.exporter.kafka.topic.replicas=1
+      - offsets.topic.replication.factor=1
+
+
+
+