From 13a8e0e4be705a5ce894fc7eea0034314b2d5af2 Mon Sep 17 00:00:00 2001 From: Andrea Cosentino Date: Fri, 20 Oct 2023 17:28:30 +0200 Subject: [PATCH] Added Kafka Apicurio Registry Sink Kamelet (#1697) * Added Kafka Apicurio Registry Sink Kamelet Signed-off-by: Andrea Cosentino * Added Kafka Apicurio Registry Sink Kamelet Signed-off-by: Andrea Cosentino --------- Signed-off-by: Andrea Cosentino --- docs/modules/ROOT/nav.adoc | 1 + ...rio-registry-not-secured-sink.kamelet.yaml | 115 ++++++++++++++++++ ...rio-registry-not-secured-sink.kamelet.yaml | 115 ++++++++++++++++++ 3 files changed, 231 insertions(+) create mode 100644 kamelets/kafka-apicurio-registry-not-secured-sink.kamelet.yaml create mode 100644 library/camel-kamelets/src/main/resources/kamelets/kafka-apicurio-registry-not-secured-sink.kamelet.yaml diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 227dd455c..3e344f5ef 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -123,6 +123,7 @@ * xref:json-schema-validator-action.adoc[] * xref:json-serialize-action.adoc[] * xref:jsonata-action.adoc[] +* xref:kafka-apicurio-registry-not-secured-sink.adoc[] * xref:kafka-apicurio-registry-not-secured-source.adoc[] * xref:kafka-azure-schema-registry-source.adoc[] * xref:kafka-manual-commit-action.adoc[] diff --git a/kamelets/kafka-apicurio-registry-not-secured-sink.kamelet.yaml b/kamelets/kafka-apicurio-registry-not-secured-sink.kamelet.yaml new file mode 100644 index 000000000..ebb2604d1 --- /dev/null +++ b/kamelets/kafka-apicurio-registry-not-secured-sink.kamelet.yaml @@ -0,0 +1,115 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +apiVersion: camel.apache.org/v1 +kind: Kamelet +metadata: + name: kafka-apicurio-registry-not-secured-sink + annotations: + camel.apache.org/kamelet.support.level: "Stable" + camel.apache.org/catalog.version: "4.1.0-SNAPSHOT" + camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4NCjwhLS0gR2VuZXJhdG9yOiBBZG9iZSBJbGx1c3RyYXRvciAxOS4wLjAsIFNWRyBFeHBvcnQgUGx1Zy1JbiAuIFNWRyBWZXJzaW9uOiA2LjAwIEJ1aWxkIDApICAtLT4NCjxzdmcgdmVyc2lvbj0iMS4xIiBpZD0iTGF5ZXJfMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgeD0iMHB4IiB5PSIwcHgiDQoJIHZpZXdCb3g9IjAgMCA1MDAgNTAwIiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA1MDAgNTAwOyIgeG1sOnNwYWNlPSJwcmVzZXJ2ZSI+DQo8ZyBpZD0iWE1MSURfMV8iPg0KCTxwYXRoIGlkPSJYTUxJRF85XyIgZD0iTTMxNC44LDI2OS43Yy0xNC4yLDAtMjcsNi4zLTM1LjcsMTYuMkwyNTYuOCwyNzBjMi40LTYuNSwzLjctMTMuNiwzLjctMjAuOWMwLTcuMi0xLjMtMTQuMS0zLjYtMjAuNg0KCQlsMjIuMy0xNS43YzguNyw5LjksMjEuNCwxNi4xLDM1LjYsMTYuMWMyNi4yLDAsNDcuNi0yMS4zLDQ3LjYtNDcuNnMtMjEuMy00Ny42LTQ3LjYtNDcuNnMtNDcuNiwyMS4zLTQ3LjYsNDcuNg0KCQljMCw0LjcsMC43LDkuMiwyLDEzLjVsLTIyLjMsMTUuN2MtOS4zLTExLjYtMjIuOC0xOS42LTM4LjEtMjIuMXYtMjYuOWMyMS42LTQuNSwzNy44LTIzLjcsMzcuOC00Ni42YzAtMjYuMi0yMS4zLTQ3LjYtNDcuNi00Ny42DQoJCWMtMjYuMiwwLTQ3LjYsMjEuMy00Ny42LDQ3LjZjMCwyMi42LDE1LjgsNDEuNSwzNi45LDQ2LjN2MjcuM2MtMjguOCw1LjEtNTAuOCwzMC4yLTUwLjgsNjAuNWMwLDMwLjQsMjIuMiw1NS43LDUxLjIsNjAuNXYyOC44DQoJCWMtMjEuMyw0LjctMzcuNCwyMy43LTM3LjQsNDYuNGMwLDI2LjIsMjEuMyw0Ny42LDQ3LjYsNDcuNmMyNi4yLDAsNDcuNi0yMS4zLDQ3LjYtNDcuNmMwLTIyLjctMTYtNDEuOC0zNy40LTQ2LjR2LTI4LjgNCgkJYzE1LTIuNSwyOC4yLTEwLjQsMzcuNC0yMS44bDIyLjUsMTUuOWMtMS4yLDQuMy0xLjksOC43LTEuOSwxMy40YzAsMjYuMiwyMS4zLDQ3LjYsNDcuNiw0Ny42czQ3LjYtMjEuMyw0Ny42LTQ3LjYNCgkJQzM2Mi40LDI5MSwzNDEuMSwyNjkuNywzMTQuOCwyNjkuN3ogTTMxNC44LDE1OC40YzEyLjcsMCwyMy4xLDEwLjQsMjMuMSwyMy4xYzAsMTIuNy0xMC4zLDIzLjEtMjMuMSwyMy4xcy0yMy4xLTEwLjQtMjMuMS0yMy4xDQoJCUMyOTEuOCwxNjguOCwzMDIuMSwxNTguNCwzMTQuOCwxNTguNHogTTE3NiwxMTUuMWMwLTEyLjcsMTAuMy0yMy4xLDIzLjEtMjMuMWMxMi43LDAsMjMuMSwxMC40LDIzLjEsMjMuMQ0KCQljMCwxMi43LTEwLjMsMjMuMS0yMy4xLDIzLjFDMTg2LjMsMTM4LjIsMTc2LDEyNy44LDE3NiwxMTUuMXogTTIyMi4xLDM4NC45YzAsMTIuNy0xMC4zLDIzLjEtMjMuMSwyMy4xDQoJCWMtMTIuNywwLTIzLjEtMTAuNC0yMy4xLTIzLjFjMC0xMi43LDEwLjMtMjMuMSwyMy4xLTIzLjFDMjExLjgsMzYxLjgsMjIyLjEsMzcyLjIsMjIyLjEsMzg0Ljl6IE0xOTkuMSwyODEuMw0KCQljLTE3LjcsMC0zMi4yLTE0LjQtMzIuMi0zMi4yYzAtMTcuNywxNC40LTMyLjIsMzIuMi0zMi4yYzE3LjcsMCwzMi4yLDE0LjQsMzIuMiwzMi4yQzIzMS4yLDI2Ni45LDIxNi44LDI4MS4zLDE5OS4xLDI4MS4zeg0KCQkgTTMxNC44LDM0MC4zYy0xMi43LDAtMjMuMS0xMC40LTIzLjEtMjMuMWMwLTEyLjcsMTAuMy0yMy4xLDIzLjEtMjMuMXMyMy4xLDEwLjQsMjMuMSwyMy4xQzMzNy45LDMzMCwzMjcuNSwzNDAuMywzMTQuOCwzNDAuM3oiLz4NCjwvZz4NCjwvc3ZnPg0K" + camel.apache.org/provider: "Apache Software Foundation" + camel.apache.org/kamelet.group: "Kafka" + camel.apache.org/kamelet.namespace: "Kafka" + labels: + camel.apache.org/kamelet.type: "sink" +spec: + definition: + title: "Kafka Not Secured with Apicurio Registry Sink" + description: |- + Send data to Kafka topics on an insecure broker with Apicurio Registry. + + The Kamelet is able to understand the following headers to be set: + + - `key` / `ce-key`: as message key + + - `partition-key` / `ce-partitionkey`: as message partition key + + Both the headers are optional. + + Three headers, if specified, will be deduplicated with different names, kafka.key will be duplicated into kafka.KEY, kafka.topic into kafka.TOPIC and kafka.override_topic into kafka.OVERRIDE_TOPIC + required: + - topic + - bootstrapServers + - apicurioRegistryUrl + type: object + properties: + topic: + title: Topic Names + description: Comma separated list of Kafka topic names + type: string + bootstrapServers: + title: Bootstrap Servers + description: Comma separated list of Kafka Broker URLs + type: string + valueSerializer: + title: Value Serializer + description: Serliazer class for value that implements the Serializer interface. + type: string + default: "io.apicurio.registry.serde.avro.AvroKafkaSerializer" + apicurioRegistryUrl: + title: Apicurio Registry URL + description: The Apicurio Schema Registry URL + type: string + x-descriptors: + - urn:keda:metadata:apicurioRegistryUrl + - urn:keda:required + avroDatumProvider: + title: Avro Datum Provider + description: How to write data with Avro + type: string + default: "io.apicurio.registry.serde.avro.ReflectAvroDatumProvider" + dependencies: + - "camel:core" + - "camel:kamelet" + - "camel:kafka" + - "mvn:io.apicurio:apicurio-registry-serdes-avro-serde:2.4.12.Final" + template: + from: + uri: "kamelet:source" + steps: + - choice: + when: + - simple: "${header[key]}" + steps: + - set-header: + name: kafka.KEY + simple: "${header[key]}" + - simple: "${header[ce-key]}" + steps: + - set-header: + name: kafka.KEY + simple: "${header[ce-key]}" + - choice: + when: + - simple: "${header[partition-key]}" + steps: + - set-header: + name: kafka.PARTITION_KEY + simple: "${header[partition-key]}" + - simple: "${header[ce-partitionkey]}" + steps: + - set-header: + name: kafka.PARTITION_KEY + simple: "${header[ce-partitionkey]}" + - to: + uri: "kafka:{{topic}}" + parameters: + brokers: "{{bootstrapServers}}" + valueSerializer: "{{valueSerializer}}" + additionalProperties.apicurio.registry.url: "{{apicurioRegistryUrl}}" + additionalProperties.apicurio.registry.avro-datum-provider: "{{avroDatumProvider}}" diff --git a/library/camel-kamelets/src/main/resources/kamelets/kafka-apicurio-registry-not-secured-sink.kamelet.yaml b/library/camel-kamelets/src/main/resources/kamelets/kafka-apicurio-registry-not-secured-sink.kamelet.yaml new file mode 100644 index 000000000..ebb2604d1 --- /dev/null +++ b/library/camel-kamelets/src/main/resources/kamelets/kafka-apicurio-registry-not-secured-sink.kamelet.yaml @@ -0,0 +1,115 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +apiVersion: camel.apache.org/v1 +kind: Kamelet +metadata: + name: kafka-apicurio-registry-not-secured-sink + annotations: + camel.apache.org/kamelet.support.level: "Stable" + camel.apache.org/catalog.version: "4.1.0-SNAPSHOT" + camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4NCjwhLS0gR2VuZXJhdG9yOiBBZG9iZSBJbGx1c3RyYXRvciAxOS4wLjAsIFNWRyBFeHBvcnQgUGx1Zy1JbiAuIFNWRyBWZXJzaW9uOiA2LjAwIEJ1aWxkIDApICAtLT4NCjxzdmcgdmVyc2lvbj0iMS4xIiBpZD0iTGF5ZXJfMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgeD0iMHB4IiB5PSIwcHgiDQoJIHZpZXdCb3g9IjAgMCA1MDAgNTAwIiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA1MDAgNTAwOyIgeG1sOnNwYWNlPSJwcmVzZXJ2ZSI+DQo8ZyBpZD0iWE1MSURfMV8iPg0KCTxwYXRoIGlkPSJYTUxJRF85XyIgZD0iTTMxNC44LDI2OS43Yy0xNC4yLDAtMjcsNi4zLTM1LjcsMTYuMkwyNTYuOCwyNzBjMi40LTYuNSwzLjctMTMuNiwzLjctMjAuOWMwLTcuMi0xLjMtMTQuMS0zLjYtMjAuNg0KCQlsMjIuMy0xNS43YzguNyw5LjksMjEuNCwxNi4xLDM1LjYsMTYuMWMyNi4yLDAsNDcuNi0yMS4zLDQ3LjYtNDcuNnMtMjEuMy00Ny42LTQ3LjYtNDcuNnMtNDcuNiwyMS4zLTQ3LjYsNDcuNg0KCQljMCw0LjcsMC43LDkuMiwyLDEzLjVsLTIyLjMsMTUuN2MtOS4zLTExLjYtMjIuOC0xOS42LTM4LjEtMjIuMXYtMjYuOWMyMS42LTQuNSwzNy44LTIzLjcsMzcuOC00Ni42YzAtMjYuMi0yMS4zLTQ3LjYtNDcuNi00Ny42DQoJCWMtMjYuMiwwLTQ3LjYsMjEuMy00Ny42LDQ3LjZjMCwyMi42LDE1LjgsNDEuNSwzNi45LDQ2LjN2MjcuM2MtMjguOCw1LjEtNTAuOCwzMC4yLTUwLjgsNjAuNWMwLDMwLjQsMjIuMiw1NS43LDUxLjIsNjAuNXYyOC44DQoJCWMtMjEuMyw0LjctMzcuNCwyMy43LTM3LjQsNDYuNGMwLDI2LjIsMjEuMyw0Ny42LDQ3LjYsNDcuNmMyNi4yLDAsNDcuNi0yMS4zLDQ3LjYtNDcuNmMwLTIyLjctMTYtNDEuOC0zNy40LTQ2LjR2LTI4LjgNCgkJYzE1LTIuNSwyOC4yLTEwLjQsMzcuNC0yMS44bDIyLjUsMTUuOWMtMS4yLDQuMy0xLjksOC43LTEuOSwxMy40YzAsMjYuMiwyMS4zLDQ3LjYsNDcuNiw0Ny42czQ3LjYtMjEuMyw0Ny42LTQ3LjYNCgkJQzM2Mi40LDI5MSwzNDEuMSwyNjkuNywzMTQuOCwyNjkuN3ogTTMxNC44LDE1OC40YzEyLjcsMCwyMy4xLDEwLjQsMjMuMSwyMy4xYzAsMTIuNy0xMC4zLDIzLjEtMjMuMSwyMy4xcy0yMy4xLTEwLjQtMjMuMS0yMy4xDQoJCUMyOTEuOCwxNjguOCwzMDIuMSwxNTguNCwzMTQuOCwxNTguNHogTTE3NiwxMTUuMWMwLTEyLjcsMTAuMy0yMy4xLDIzLjEtMjMuMWMxMi43LDAsMjMuMSwxMC40LDIzLjEsMjMuMQ0KCQljMCwxMi43LTEwLjMsMjMuMS0yMy4xLDIzLjFDMTg2LjMsMTM4LjIsMTc2LDEyNy44LDE3NiwxMTUuMXogTTIyMi4xLDM4NC45YzAsMTIuNy0xMC4zLDIzLjEtMjMuMSwyMy4xDQoJCWMtMTIuNywwLTIzLjEtMTAuNC0yMy4xLTIzLjFjMC0xMi43LDEwLjMtMjMuMSwyMy4xLTIzLjFDMjExLjgsMzYxLjgsMjIyLjEsMzcyLjIsMjIyLjEsMzg0Ljl6IE0xOTkuMSwyODEuMw0KCQljLTE3LjcsMC0zMi4yLTE0LjQtMzIuMi0zMi4yYzAtMTcuNywxNC40LTMyLjIsMzIuMi0zMi4yYzE3LjcsMCwzMi4yLDE0LjQsMzIuMiwzMi4yQzIzMS4yLDI2Ni45LDIxNi44LDI4MS4zLDE5OS4xLDI4MS4zeg0KCQkgTTMxNC44LDM0MC4zYy0xMi43LDAtMjMuMS0xMC40LTIzLjEtMjMuMWMwLTEyLjcsMTAuMy0yMy4xLDIzLjEtMjMuMXMyMy4xLDEwLjQsMjMuMSwyMy4xQzMzNy45LDMzMCwzMjcuNSwzNDAuMywzMTQuOCwzNDAuM3oiLz4NCjwvZz4NCjwvc3ZnPg0K" + camel.apache.org/provider: "Apache Software Foundation" + camel.apache.org/kamelet.group: "Kafka" + camel.apache.org/kamelet.namespace: "Kafka" + labels: + camel.apache.org/kamelet.type: "sink" +spec: + definition: + title: "Kafka Not Secured with Apicurio Registry Sink" + description: |- + Send data to Kafka topics on an insecure broker with Apicurio Registry. + + The Kamelet is able to understand the following headers to be set: + + - `key` / `ce-key`: as message key + + - `partition-key` / `ce-partitionkey`: as message partition key + + Both the headers are optional. + + Three headers, if specified, will be deduplicated with different names, kafka.key will be duplicated into kafka.KEY, kafka.topic into kafka.TOPIC and kafka.override_topic into kafka.OVERRIDE_TOPIC + required: + - topic + - bootstrapServers + - apicurioRegistryUrl + type: object + properties: + topic: + title: Topic Names + description: Comma separated list of Kafka topic names + type: string + bootstrapServers: + title: Bootstrap Servers + description: Comma separated list of Kafka Broker URLs + type: string + valueSerializer: + title: Value Serializer + description: Serliazer class for value that implements the Serializer interface. + type: string + default: "io.apicurio.registry.serde.avro.AvroKafkaSerializer" + apicurioRegistryUrl: + title: Apicurio Registry URL + description: The Apicurio Schema Registry URL + type: string + x-descriptors: + - urn:keda:metadata:apicurioRegistryUrl + - urn:keda:required + avroDatumProvider: + title: Avro Datum Provider + description: How to write data with Avro + type: string + default: "io.apicurio.registry.serde.avro.ReflectAvroDatumProvider" + dependencies: + - "camel:core" + - "camel:kamelet" + - "camel:kafka" + - "mvn:io.apicurio:apicurio-registry-serdes-avro-serde:2.4.12.Final" + template: + from: + uri: "kamelet:source" + steps: + - choice: + when: + - simple: "${header[key]}" + steps: + - set-header: + name: kafka.KEY + simple: "${header[key]}" + - simple: "${header[ce-key]}" + steps: + - set-header: + name: kafka.KEY + simple: "${header[ce-key]}" + - choice: + when: + - simple: "${header[partition-key]}" + steps: + - set-header: + name: kafka.PARTITION_KEY + simple: "${header[partition-key]}" + - simple: "${header[ce-partitionkey]}" + steps: + - set-header: + name: kafka.PARTITION_KEY + simple: "${header[ce-partitionkey]}" + - to: + uri: "kafka:{{topic}}" + parameters: + brokers: "{{bootstrapServers}}" + valueSerializer: "{{valueSerializer}}" + additionalProperties.apicurio.registry.url: "{{apicurioRegistryUrl}}" + additionalProperties.apicurio.registry.avro-datum-provider: "{{avroDatumProvider}}"