SFTP / SCP connectors missing authentication options to use a private key

it-tests/Makefile

@@ -33,6 +33,8 @@ tests:
 	./timer-aws-sns-it-test.sh $(camel-version)
 	cd aws/aws-kinesis-firehose/sink/ && \
 	./timer-aws-kinesis-firehose-it-test.sh $(camel-version)
+	cd misc/sftp/source/ && \
+	./sftp-log-it-test.sh $(camel-version)
 	./scripts/results.sh
 	rm -rf tests
 

it-tests/misc/sftp/source/sftp-log-it-test.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+if [ $# -ne 1 ]; then
+    echo $0: usage: sftp-log-it-test.sh camel-version
+    exit 1
+fi
+
+camel_version=$1
+
+docker pull emberstack/sftp
+docker run -p 24:22 --name sftp -d emberstack/sftp 
+sftpid=`docker ps -aqf "name=sftp"`
+
+echo $sftpid
+
+jbang run -Dcamel.jbang.version=$camel_version camel@apache/camel run --local-kamelet-dir=../../../../kamelets/ sftp-log.yaml &
+
+sleep 10
+
+docker exec -it $sftpid bash -c "echo 'Ciao' >> /home/demo/sftp/demos/file1.txt"
+docker exec -it $sftpid bash -c "echo 'Ciao' >> /home/demo/sftp/demos/file2.txt"
+docker exec -it $sftpid bash -c "echo 'Ciao' >> /home/demo/sftp/demos/file3.txt"
+docker exec -it $sftpid bash -c "echo 'Ciao' >> /home/demo/sftp/demos/file4.txt"
+docker exec -it $sftpid bash -c "echo 'Ciao' >> /home/demo/sftp/demos/file5.txt"
+
+sleep 10
+
+variable=`jbang run -Dcamel.jbang.version=$camel_version camel@apache/camel get | tail -n +2` 
+success=`echo $variable | cut -d' ' -f11`
+fail=`echo $variable | cut -d' ' -f12`
+if [[ $success == 5 && $fail == 0 ]] 
+then 
+    mkdir -p ../../../tests/
+    echo "Test Successful" > ../../../tests/sftp-log-it-test.result;
+else
+    mkdir -p ../../../tests/
+    echo "Test failed" > ../../../tests/sftp-log-it-test.result;
+fi
+
+jbang run -Dcamel.jbang.version=$camel_version camel@apache/camel stop sftp-log
+
+docker stop sftp
+docker rm sftp
+
+cat ../../../tests/sftp-log-it-test.result

it-tests/misc/sftp/source/sftp-log.yaml

@@ -0,0 +1,32 @@
+## ---------------------------------------------------------------------------
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements.  See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License.  You may obtain a copy of the License at
+##
+##      http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+## ---------------------------------------------------------------------------
+
+- route:
+    from:
+      uri: "kamelet:sftp-source"
+      parameters:
+        username: demo
+        password: demo
+        connectionHost: localhost
+        connectionPort: 24
+        directoryName: demos/
+        recursive: true
+      steps:
+      - to: 
+          uri: "kamelet:log-sink"
+          parameters:
+            showStreams: false  

kamelets/sftp-sink.kamelet.yaml

@@ -38,8 +38,6 @@ spec:
     required:
       - connectionHost
       - connectionPort
-      - username
-      - password
       - directoryName
     type: object
     properties:
@@ -89,6 +87,31 @@ spec:
         default: false
         x-descriptors:
         - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+      privateKeyFile:
+        title: Private Key File
+        description: Set the private key file so that the SFTP endpoint can do private key verification.
+        type: string
+      privateKeyPassphrase:
+        title: Private Key Passphrase
+        description: Set the private key file passphrase so that the SFTP endpoint can do private key verification.
+        type: string
+      privateKeyUri:
+        title: Private Key URI
+        description: Set the private key file (loaded from classpath by default) so that the SFTP endpoint can do private key verification.
+        type: string
+        pattern: "^(http|https|file|classpath)://.*"
+      strictHostKeyChecking:
+        title: Strict Host Checking
+        description: Sets whether to use strict host key checking.
+        type: string
+        default: no
+      useUserKnownHostsFile:
+        title: Use User Known Hosts File
+        description: If knownHostFile has not been explicit configured then use the host file from System.getProperty(user.home)/.ssh/known_hosts.
+        type: boolean
+        default: true
+        x-descriptors:
+        - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
   dependencies:
     - "camel:ftp"
     - "camel:core"
@@ -110,9 +133,15 @@ spec:
                 name: CamelFileName
                 simple: "${header[ce-file]}"
       - to:
-          uri: "sftp:{{username}}@{{connectionHost}}:{{connectionPort}}/{{directoryName}}"
+          uri: "sftp:{{connectionHost}}:{{connectionPort}}/{{directoryName}}"
           parameters:
-            password: "{{password}}"
+            username: "{{?username}}"
+            password: "{{?password}}"
+            privateKeyFile: "{{?privateKeyFile}}"
+            privateKeyPassphrase: "{{?privateKeyPassphrase}}"
+            privateKeyUri: "{{?privateKeyUri}}"
+            strictHostKeyChecking: "{{?strictHostKeyChecking}}"
+            useUserKnownHostsFile: "{{?useUserKnownHostsFile}}"            
             passiveMode: "{{passiveMode}}"
             fileExist: "{{fileExist}}"
             binary: "{{binary}}"

kamelets/sftp-source.kamelet.yaml

@@ -34,8 +34,6 @@ spec:
     required:
       - connectionHost
       - connectionPort
-      - username
-      - password
       - directoryName
     type: object
     properties:
@@ -101,27 +99,58 @@ spec:
         default: false
         x-descriptors:
         - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+      privateKeyFile:
+        title: Private Key File
+        description: Set the private key file so that the SFTP endpoint can do private key verification.
+        type: string
+      privateKeyPassphrase:
+        title: Private Key Passphrase
+        description: Set the private key file passphrase so that the SFTP endpoint can do private key verification.
+        type: string
+      privateKeyUri:
+        title: Private Key URI
+        description: Set the private key file (loaded from classpath by default) so that the SFTP endpoint can do private key verification.
+        type: string
+        pattern: "^(http|https|file|classpath)://.*"
+      strictHostKeyChecking:
+        title: Strict Host Checking
+        description: Sets whether to use strict host key checking.
+        type: string
+        default: no
+      useUserKnownHostsFile:
+        title: Use User Known Hosts File
+        description: If knownHostFile has not been explicit configured then use the host file from System.getProperty(user.home)/.ssh/known_hosts.
+        type: boolean
+        default: true
+        x-descriptors:
+        - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
   dependencies:
     - "camel:ftp"
     - "camel:core"
     - "camel:kamelet"
   template:
     from:
-      uri: "sftp:{{username}}@{{connectionHost}}:{{connectionPort}}/{{directoryName}}"
+      uri: "sftp:{{connectionHost}}:{{connectionPort}}/{{directoryName}}"
       parameters:
-        password: "{{password}}"
+        username: "{{?username}}"
+        password: "{{?password}}"
+        privateKeyFile: "{{?privateKeyFile}}"
+        privateKeyPassphrase: "{{?privateKeyPassphrase}}"
+        privateKeyUri: "{{?privateKeyUri}}"
+        strictHostKeyChecking: "{{?strictHostKeyChecking}}"
+        useUserKnownHostsFile: "{{?useUserKnownHostsFile}}"   
         passiveMode: "{{passiveMode}}"
         recursive: "{{recursive}}"
         idempotent: "{{idempotent}}"
         ignoreFileNotFoundOrPermissionError: "{{ignoreFileNotFoundOrPermissionError}}"
         binary: "{{binary}}"
       steps:
+      - set-body:
+          simple: "${body.getBody()}"
       - set-header:
           name: file
           simple: "${header[CamelFileName]}"
       - set-header:
           name: ce-file
           simple: "${header[CamelFileName]}"
-      - convert-body-to:
-          type: "java.io.InputStream"
       - to: "kamelet:sink"

library/camel-kamelets/src/main/resources/kamelets/sftp-sink.kamelet.yaml

@@ -38,8 +38,6 @@ spec:
     required:
       - connectionHost
       - connectionPort
-      - username
-      - password
       - directoryName
     type: object
     properties:
@@ -89,6 +87,31 @@ spec:
         default: false
         x-descriptors:
         - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+      privateKeyFile:
+        title: Private Key File
+        description: Set the private key file so that the SFTP endpoint can do private key verification.
+        type: string
+      privateKeyPassphrase:
+        title: Private Key Passphrase
+        description: Set the private key file passphrase so that the SFTP endpoint can do private key verification.
+        type: string
+      privateKeyUri:
+        title: Private Key URI
+        description: Set the private key file (loaded from classpath by default) so that the SFTP endpoint can do private key verification.
+        type: string
+        pattern: "^(http|https|file|classpath)://.*"
+      strictHostKeyChecking:
+        title: Strict Host Checking
+        description: Sets whether to use strict host key checking.
+        type: string
+        default: no
+      useUserKnownHostsFile:
+        title: Use User Known Hosts File
+        description: If knownHostFile has not been explicit configured then use the host file from System.getProperty(user.home)/.ssh/known_hosts.
+        type: boolean
+        default: true
+        x-descriptors:
+        - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
   dependencies:
     - "camel:ftp"
     - "camel:core"
@@ -110,9 +133,15 @@ spec:
                 name: CamelFileName
                 simple: "${header[ce-file]}"
       - to:
-          uri: "sftp:{{username}}@{{connectionHost}}:{{connectionPort}}/{{directoryName}}"
+          uri: "sftp:{{connectionHost}}:{{connectionPort}}/{{directoryName}}"
           parameters:
-            password: "{{password}}"
+            username: "{{?username}}"
+            password: "{{?password}}"
+            privateKeyFile: "{{?privateKeyFile}}"
+            privateKeyPassphrase: "{{?privateKeyPassphrase}}"
+            privateKeyUri: "{{?privateKeyUri}}"
+            strictHostKeyChecking: "{{?strictHostKeyChecking}}"
+            useUserKnownHostsFile: "{{?useUserKnownHostsFile}}"            
             passiveMode: "{{passiveMode}}"
             fileExist: "{{fileExist}}"
             binary: "{{binary}}"

library/camel-kamelets/src/main/resources/kamelets/sftp-source.kamelet.yaml

@@ -34,8 +34,6 @@ spec:
     required:
       - connectionHost
       - connectionPort
-      - username
-      - password
       - directoryName
     type: object
     properties:
@@ -101,27 +99,58 @@ spec:
         default: false
         x-descriptors:
         - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
+      privateKeyFile:
+        title: Private Key File
+        description: Set the private key file so that the SFTP endpoint can do private key verification.
+        type: string
+      privateKeyPassphrase:
+        title: Private Key Passphrase
+        description: Set the private key file passphrase so that the SFTP endpoint can do private key verification.
+        type: string
+      privateKeyUri:
+        title: Private Key URI
+        description: Set the private key file (loaded from classpath by default) so that the SFTP endpoint can do private key verification.
+        type: string
+        pattern: "^(http|https|file|classpath)://.*"
+      strictHostKeyChecking:
+        title: Strict Host Checking
+        description: Sets whether to use strict host key checking.
+        type: string
+        default: no
+      useUserKnownHostsFile:
+        title: Use User Known Hosts File
+        description: If knownHostFile has not been explicit configured then use the host file from System.getProperty(user.home)/.ssh/known_hosts.
+        type: boolean
+        default: true
+        x-descriptors:
+        - 'urn:alm:descriptor:com.tectonic.ui:checkbox'
   dependencies:
     - "camel:ftp"
     - "camel:core"
     - "camel:kamelet"
   template:
     from:
-      uri: "sftp:{{username}}@{{connectionHost}}:{{connectionPort}}/{{directoryName}}"
+      uri: "sftp:{{connectionHost}}:{{connectionPort}}/{{directoryName}}"
       parameters:
-        password: "{{password}}"
+        username: "{{?username}}"
+        password: "{{?password}}"
+        privateKeyFile: "{{?privateKeyFile}}"
+        privateKeyPassphrase: "{{?privateKeyPassphrase}}"
+        privateKeyUri: "{{?privateKeyUri}}"
+        strictHostKeyChecking: "{{?strictHostKeyChecking}}"
+        useUserKnownHostsFile: "{{?useUserKnownHostsFile}}"   
         passiveMode: "{{passiveMode}}"
         recursive: "{{recursive}}"
         idempotent: "{{idempotent}}"
         ignoreFileNotFoundOrPermissionError: "{{ignoreFileNotFoundOrPermissionError}}"
         binary: "{{binary}}"
       steps:
+      - set-body:
+          simple: "${body.getBody()}"
       - set-header:
           name: file
           simple: "${header[CamelFileName]}"
       - set-header:
           name: ce-file
           simple: "${header[CamelFileName]}"
-      - convert-body-to:
-          type: "java.io.InputStream"
       - to: "kamelet:sink"

library/kamelets-maven-plugin/src/main/java/org/apache/camel/kamelets/maven/plugin/ValidateKameletsMojo.java

@@ -101,7 +101,7 @@ public void execute() throws MojoExecutionException, MojoFailureException {
                                         .map(ComponentModel.EndpointOptionModel::getName)
                                         .collect(Collectors.toList());
                         for (Map.Entry<String, Object> entry : p.entrySet()) {
-                            if (!entry.getKey().equals("period") && (!name.equals("kafka-ssl-source") && !name.equals("timer-source") && !name.equals("cron-source") && !name.equals("fhir-source") && !name.equals("beer-source"))) {
+                            if (!entry.getKey().equals("period") && (!name.equals("sftp-source") && !name.equals("kafka-ssl-source") && !name.equals("timer-source") && !name.equals("cron-source") && !name.equals("fhir-source") && !name.equals("beer-source"))) {
                                 if (!ceInternal.contains(entry.getKey())) {
                                     getLog().error("Kamelet Name: " + name);
                                     getLog().error("Scheme Name: " + cleanName);