[agents] Add support for using Camel components as sources

examples/applications/camel-source/.gitignore

@@ -0,0 +1 @@
+java/lib/*

examples/applications/camel-source/README.md

@@ -0,0 +1,37 @@
+# Using Apache Camel as a source
+
+This sample application shows how to use Apache Camel as a source.
+We are using the camel-github component to read the messages from GitHub.
+
+https://camel.apache.org/components/4.0.x/github-component.html
+
+In the application we are using the following LangStream agents:
+
+- camel-source: to read the messages from GitHub
+- compute: to manipulate the messages coming from Camel
+
+## Deploy the LangStream application
+
+In order to consume events from GitHub you have to provide a token and the references to the repository.
+
+
+```
+export CAMEL_GITHUB_OAUTH_TOKEN=xxxx
+export CAMEL_GITHUB_REPO_NAME=langstream
+export CAMEL_GITHUB_REPO_OWNER=langstream
+export CAMEL_GITHUB_BRANCH=main
+```
+
+Then you can run the application
+
+```
+./bin/langstream docker run test -app examples/applications/camel-source -s examples/secrets/secrets.yaml
+```
+
+## Chat
+
+Since the application opens a gateway, we can use the gateway API to read the messages:
+```
+./bin/langstream gateway consume test -c github
+```
+

examples/applications/camel-source/configuration.yaml

@@ -0,0 +1,29 @@
+#
+# Copyright DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+configuration:
+  defaults:
+    globals:
+      github-event: "PULLREQUESTCOMMENT"
+  dependencies:
+    - name: "Apache Camel GitHub component"
+      url: "https://repo1.maven.org/maven2/org/apache/camel/camel-github/4.1.0/camel-github-4.1.0.jar"
+      sha512sum: "f40e55ddfde21e11e6a26fbb50e4f75df37976c673dbc4aeeaa350257b0179f4a9a71f099f8ac3cbc60cc9d8ff9559984a304fb7d6785bcfb22a9943ceb8d7bd"
+      type: "java-library"
+    - name: "Apache Camel GitHub component - dependency"
+      url: "https://repo1.maven.org/maven2/org/eclipse/mylyn/github/org.eclipse.egit.github.core/2.1.5/org.eclipse.egit.github.core-2.1.5.jar"
+      sha512sum: "4fab14efed656b6705ad74ab9c05a346fe4fdc737b1b5130842aee7ebba98b895b7cb8c35d15e3da150116f445ae3b5c88daaaa5319c39144cb365247b9d0630"
+      type: "java-library"

examples/applications/camel-source/gateways.yaml

@@ -0,0 +1,21 @@
+#
+#
+# Copyright DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+gateways:
+  - id: "github"
+    type: consume
+    topic: "output-topic"

examples/applications/camel-source/pipeline.yaml

@@ -0,0 +1,36 @@
+#
+# Copyright DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+topics:
+  - name: "output-topic"
+    creation-mode: create-if-not-exists
+pipeline:
+  - name: "read-from-github"
+    type: "camel-source"
+
+    configuration:
+      component-uri: "github:${globals.github-event}/${secrets.camel-github-source.branch}"
+      component-options:
+         oauthToken: "${secrets.camel-github-source.oauthToken}"
+         repoName: "${secrets.camel-github-source.repoName}"
+         repoOwner: "${secrets.camel-github-source.repoOwner}"
+  - name: "keep only the contents of the comment"
+    type: "compute"
+    output: "output-topic"
+    configuration:
+        fields:
+           - name: "value"
+             expression: "value.body"

examples/secrets/secrets.yaml

@@ -133,11 +133,15 @@ secrets:
       secret-key: "${BEDROCK_SECRET_KEY}"
       region: "${REGION:-us-east-1}"
       completions-model: "${BEDROCK_COMPLETIONS_MODEL}"
+  - name: camel-github-source
+    id: camel-github-source
+    data:
+      oauthToken: "${CAMEL_GITHUB_OAUTH_TOKEN:-xxxx}"
+      repoName: "${CAMEL_GITHUB_REPO_NAME:-langstream}"
+      repoOwner: "${CAMEL_GITHUB_REPO_OWNER:-langstream}"
+      branch: "${CAMEL_GITHUB_BRANCH:-main}"
   - id: slack 
     data:
       token: "${SLACK_TOKEN:-}"
       url: "${SLACK_URL:-}"
       provider: "${SLACK_PROVIDER:-slack}"
-#      embeddings-model: "${OPEN_AI_EMBEDDINGS_MODEL:-text-embedding-ada-002}"
-#      chat-completions-model: "${OPEN_AI_CHAT_COMPLETIONS_MODEL:-gpt-3.5-turbo}"
-#      text-completions-model: "${OPEN_AI_TEXT_COMPLETIONS_MODEL:-gpt-3.5-turbo-instruct}"

langstream-agents/langstream-agent-camel/pom.xml

@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright DataStax, Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <groupId>ai.langstream</groupId>
+    <artifactId>langstream-agents</artifactId>
+    <version>0.3.2-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>langstream-agent-camel</artifactId>
+  <packaging>jar</packaging>
+  <name>LangStream - Apache Camel Agents</name>
+  <properties>
+    <maven.compiler.source>17</maven.compiler.source>
+    <maven.compiler.target>17</maven.compiler.target>
+    <camel.version>4.1.0</camel.version>
+  </properties>
+    <dependencyManagement>
+        <dependencies>
+            <!-- Add Camel BOM -->
+            <dependency>
+                <groupId>org.apache.camel</groupId>
+                <artifactId>camel-bom</artifactId>
+                <version>${camel.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.camel</groupId>
+      <artifactId>camel-core-engine</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.camel</groupId>
+      <artifactId>camel-core-languages</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.camel</groupId>
+      <artifactId>camel-timer</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.camel</groupId>
+      <artifactId>camel-log</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>langstream-api</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>junit-jupiter</artifactId>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.nifi</groupId>
+        <artifactId>nifi-nar-maven-plugin</artifactId>
+        <extensions>true</extensions>
+        <configuration>
+          <classifier>nar</classifier>
+        </configuration>
+        <executions>
+          <execution>
+            <id>default-nar</id>
+            <phase>package</phase>
+            <goals>
+              <goal>nar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>