Update non-working examples in Kamelet Catalog docs

Fix #866

* There are some generated kamelet bindings are invalid, the auto
  generation cannot resolve all the required steps, so there is
  a need to create it manually
* Add a comment marker in kamelet binding examples for the auto
  generation to skip its creation
* The doc generation source the kamelet binding example file when it
  doesn't auto generate the example
* the camel-kamelets directory is hardcoded in kamelets.js as it is
  assumed this generation mechanism already assumes this directory

README.md

@@ -163,6 +163,12 @@ Kamelets submitted with tests that verify their correctness **MUST** be labeled
 
 **NOTE**: there's no way at the moment to inject credentials for external systems into the CI in order to write more advanced tests, but we can expect we'll find an usable strategy in the long run
 
+### Kamelet Binding Examples
+
+Binding examples are highly encouraged to be added under `templates/bindings/camel-k` directory for Kamelet Binding and `templates/bindings/core` for the YAML routes.
+
+When the Kamelet Catalog documentation is generated, the examples in each Kamelet documentation page are automatically generated, but the generator code is not wise enough and it may generate a Kamelet Binding that doesn't work, requiring additional steps. In this case, the binding example should be added to the above mentioned directories, and add the comment marker at the first line `"# example_for_kamelet_doc"` only in the Camel K Kamelet Binding example (in `templates/bindings/camel-k`). When the documentation mechanism runs, it will source this binding example into the kamelet documentation page as example.
+
 ## Releasing
 
 This project is released as standard Apache Camel module.

docs/modules/ROOT/examples/js/kamelets.js

@@ -16,6 +16,8 @@
  */
 
 const util = require('camel-website-util')
+const fs = require('fs')
+const yaml = require('js-yaml');
 
 const QUOTED_CHARS = /[$`"\\]/g
 
@@ -26,34 +28,48 @@ const QUOTE_REPLACEMENTS = {
   '\\': '\\\\',
 }
 
+// marker added to the first line of kamelet binding files in templates/bindings/camel-k
+// generator will not generate a kamelet binding example and will source this kamelet binding file into the generated doc
+const EXAMPLE_KAMELET_DOC_MARKER = "example_for_kamelet_doc"
+
+// regex to replace the sink type
+const regex = new RegExp(`(  sink:\\n\\s*ref:\\n)(\\s*kind:)(.*)(\\n\\s*apiVersion:)(.*)(\\n\\s*name:)(.*)`, 'g')
+
 const svgb64Prefix = 'data:image/svg+xml;base64,'
 
 module.exports = {
   binding: (binding, apiVersion, kind, metadata_, spec_, refKind, refApiVersion, refName) => {
       const name = metadata_.name
       const metadata = {name: `${name}-binding`}
-      const kamelet = {
-        ref: {
-          kind,
+
+      genExample = shouldGenerateKameletBindingExample(metadata.name)
+      if (genExample) {
+        const kamelet = {
+          ref: {
+            kind,
+            apiVersion,
+            name,
+          },
+          properties: kameletPropertyList(spec_.definition)
+        }
+        const platform = {
+          ref: {
+            kind: refKind,
+            apiVersion: refApiVersion,
+            name: refName,
+          },
+        }
+        const base = {
           apiVersion,
-          name,
-        },
-        properties: kameletPropertyList(spec_.definition)
-      }
-      const platform = {
-        ref: {
-          kind: refKind,
-          apiVersion: refApiVersion,
-          name: refName,
-        },
-      }
-      const base = {
-        apiVersion,
-        kind: 'KameletBinding',
-        metadata,
+          kind: 'KameletBinding',
+          metadata,
+        }
+        const fn = kameletBindings[binding] || (() => `unrecognized binding ${binding}`)
+        return fn(base, kamelet, platform)
+      } else {
+        content = readKameletBindingExample(metadata.name, refApiVersion, refKind, refName)
+        return content
       }
-      const fn = kameletBindings[binding] || (() => `unrecognized binding ${binding}`)
-      return fn(base, kamelet, platform)
   },
 
   bindingCommand: (binding, name, definition, topic) => {
@@ -128,6 +144,45 @@ function kameletPropertyList (definition) {
   )
 }
 
+// verify if the existing kamelet binding example should be automatically generated
+// by checking if there is a comment marker in the first line
+function shouldGenerateKameletBindingExample(file) {
+  f = "../camel-kamelets/templates/bindings/camel-k/" + file + ".yaml"
+  try {
+    bufContent = fs.readFileSync(f)
+    content = bufContent.toString()
+    line = content.split(/\r?\n/)[0]
+    return line.indexOf(EXAMPLE_KAMELET_DOC_MARKER) < 0
+  } catch (err) {
+    console.log("Error reading kamelet binding example file " + file + ": " +  err)
+    return true
+  }
+}
+
+// source the kamelet binding example from the example file
+// skip the first line and replace the sink kind when the kind is a knative channel
+function readKameletBindingExample(file, apiVersion, kind, name) {
+  f = "../camel-kamelets/templates/bindings/camel-k/" + file + ".yaml"
+  try {
+    bufContent = fs.readFileSync(f)
+    content = bufContent.toString()
+    lines = content.split(/\r?\n/)
+    klbContent = ""
+    // skip the first line, as it contains the comment marker
+    for (i = 1; i < lines.length; i++) {
+      klbContent += lines[i] + "\n"
+    }
+    // uses a knative channel sink
+    klbContent = klbContent.replace(regex, "$1$2 " + kind + "$4 " + apiVersion + "$6 " + name);
+    yamlDoc = yaml.load(klbContent);
+    return yamlDoc
+  } catch (err) {
+    console.log("Error reading kamelet binding example file " + file + ": " +  err)
+    return err
+  }
+}
+
+
 const kameletBindings = {
   action: (base, kamelet, platform) => Object.assign(base, {
       spec: {

templates/bindings/camel-k/avro-deserialize-action-binding.yaml

@@ -1,3 +1,4 @@
+# example_for_kamelet_doc
 apiVersion: camel.apache.org/v1alpha1
 kind: KameletBinding
 metadata:
@@ -9,14 +10,28 @@ spec:
       apiVersion: camel.apache.org/v1alpha1
       name: timer-source
     properties:
-      message: "Hello"
+      message: '{"first":"Ada","last":"Lovelace"}'
   steps:
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: json-deserialize-action
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: avro-serialize-action
+    properties:
+      schema: "{\"type\": \"record\", \"namespace\": \"com.example\", \"name\": \"FullName\", \"fields\": [{\"name\": \"first\", \"type\": \"string\"},{\"name\": \"last\", \"type\": \"string\"}]}"
   - ref:
       kind: Kamelet
       apiVersion: camel.apache.org/v1alpha1
       name: avro-deserialize-action
     properties:
       schema: "{\"type\": \"record\", \"namespace\": \"com.example\", \"name\": \"FullName\", \"fields\": [{\"name\": \"first\", \"type\": \"string\"},{\"name\": \"last\", \"type\": \"string\"}]}"
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: json-serialize-action
   sink:
     ref:
       kind: KafkaTopic

templates/bindings/camel-k/avro-serialize-action-binding.yaml

@@ -1,3 +1,4 @@
+# example_for_kamelet_doc
 apiVersion: camel.apache.org/v1alpha1
 kind: KameletBinding
 metadata:
@@ -9,8 +10,12 @@ spec:
       apiVersion: camel.apache.org/v1alpha1
       name: timer-source
     properties:
-      message: "Hello"
+      message: '{"first":"Ada","last":"Lovelace"}'
   steps:
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: json-deserialize-action
   - ref:
       kind: Kamelet
       apiVersion: camel.apache.org/v1alpha1

templates/bindings/camel-k/caffeine-action-binding.yaml

@@ -1,3 +1,4 @@
+# example_for_kamelet_doc
 apiVersion: camel.apache.org/v1alpha1
 kind: KameletBinding
 metadata:
@@ -9,12 +10,69 @@ spec:
       apiVersion: camel.apache.org/v1alpha1
       name: timer-source
     properties:
-      message: "Hello"
+      message: '{"foo":"bar"}'
+      period: 10000
   steps:
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: json-deserialize-action
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: log-sink
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: insert-header-action
+    properties:
+      name: "caffeine-key"
+      value: "my-key"
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: insert-header-action
+    properties:
+      name: "caffeine-operation"
+      value: "PUT"
   - ref:
       kind: Kamelet
       apiVersion: camel.apache.org/v1alpha1
       name: caffeine-action
+    properties:
+      cacheName: "my-cache"
+  # extract the foo field from the body, cleaning the body
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: extract-field-action
+    properties:
+      field: '{"foo"}'
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: log-sink
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: insert-header-action
+    properties:
+      name: "caffeine-key"
+      value: "my-key"
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: insert-header-action
+    properties:
+      name: "caffeine-operation"
+      value: "GET"
+  # retrieve the json payload from the cache and put into the body
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: caffeine-action
+    properties:
+      cacheName: "my-cache"
   sink:
     ref:
       kind: KafkaTopic

templates/bindings/camel-k/has-header-filter-action-binding.yaml

@@ -1,3 +1,4 @@
+# example_for_kamelet_doc
 apiVersion: camel.apache.org/v1alpha1
 kind: KameletBinding
 metadata:
@@ -11,12 +12,19 @@ spec:
     properties:
       message: "Hello"
   steps:
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: insert-header-action
+    properties:
+      name: "my-header"
+      value: "my-value"
   - ref:
       kind: Kamelet
       apiVersion: camel.apache.org/v1alpha1
       name: has-header-filter-action
     properties:
-      name: "headerName"
+      name: "my-header"
   sink:
     ref:
       kind: KafkaTopic

templates/bindings/camel-k/insert-field-action-binding.yaml

@@ -1,3 +1,4 @@
+# example_for_kamelet_doc
 apiVersion: camel.apache.org/v1alpha1
 kind: KameletBinding
 metadata:
@@ -9,8 +10,12 @@ spec:
       apiVersion: camel.apache.org/v1alpha1
       name: timer-source
     properties:
-      message: "Hello"
+      message: '{"foo":"John"}'
   steps:
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: json-deserialize-action
   - ref:
       kind: Kamelet
       apiVersion: camel.apache.org/v1alpha1

templates/bindings/camel-k/protobuf-deserialize-action-binding.yaml

@@ -1,3 +1,4 @@
+# example_for_kamelet_doc
 apiVersion: camel.apache.org/v1alpha1
 kind: KameletBinding
 metadata:
@@ -9,8 +10,18 @@ spec:
       apiVersion: camel.apache.org/v1alpha1
       name: timer-source
     properties:
-      message: "Hello"
+      message: '{"first": "John", "last":"Doe"}'
   steps:
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: json-deserialize-action
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: protobuf-serialize-action
+    properties:
+      schema: "message Person { required string first = 1; required string last = 2; }"
   - ref:
       kind: Kamelet
       apiVersion: camel.apache.org/v1alpha1

templates/bindings/camel-k/protobuf-serialize-action-binding.yaml

@@ -1,16 +1,21 @@
+# example_for_kamelet_doc
 apiVersion: camel.apache.org/v1alpha1
 kind: KameletBinding
 metadata:
-  name: protobuf-serialize-action-binding
+  name: protobuf-deserialize-action-binding
 spec:
   source:
     ref:
       kind: Kamelet
       apiVersion: camel.apache.org/v1alpha1
       name: timer-source
     properties:
-      message: "Hello"
+      message: '{"first": "John", "last":"Doe"}'
   steps:
+  - ref:
+      kind: Kamelet
+      apiVersion: camel.apache.org/v1alpha1
+      name: json-deserialize-action
   - ref:
       kind: Kamelet
       apiVersion: camel.apache.org/v1alpha1

templates/bindings/core/avro-deserialize-action-binding.yaml

@@ -3,11 +3,19 @@
       uri: "kamelet:timer-source"
       parameters:
         period: 1000
-        message: "{ \"foo\": \"John\"}"
+        message: '{"first":"Ada","last":"Lovelace"}'
     steps:
+      - to:
+          uri: "kamelet:json-deserialize-action"
+      - to:
+          uri: "kamelet:avro-serialize-action"
+          parameters:
+            schema: "{\"type\": \"record\", \"namespace\": \"com.example\", \"name\": \"FullName\", \"fields\": [{\"name\": \"first\", \"type\": \"string\"},{\"name\": \"last\", \"type\": \"string\"}]}"
       - to:
           uri: "kamelet:avro-deserialize-action"
           parameters:
             schema: "{\"type\": \"record\", \"namespace\": \"com.example\", \"name\": \"FullName\", \"fields\": [{\"name\": \"first\", \"type\": \"string\"},{\"name\": \"last\", \"type\": \"string\"}]}"
+      - to:
+          uri: "kamelet:json-serialize-action"
       - to:
           uri: "log:info"

templates/bindings/core/avro-serialize-action-binding.yaml

@@ -3,8 +3,10 @@
       uri: "kamelet:timer-source"
       parameters:
         period: 1000
-        message: "{ \"foo\": \"John\"}"
+        message: '{"first":"Ada","last":"Lovelace"}'
     steps:
+      - to:
+          uri: "kamelet:json-deserialize-action"
       - to:
           uri: "kamelet:avro-serialize-action"
           parameters: