chore: Fix YAKS tests

- Update to YAKS 0.14.1 - Update Camel K to 1.12.0-nightly - Fix jitpack dependencies used in Kamelets when running GitHub CI jobs for a PR (evaluate current project version e.g. 3.20.0-SNAPSHOT and replace with version from PR) - Enable integration logger in YAKS tests (better failure analysis when tests fail) - Fix AWS S3 tests to use path-style access with AWS Java SDK2 - Fix Kafka test to not use *.test.yaml files as these are reserved to be YAML test cases in YAKS - Do not run experimental tests with each CI job - Fix flaky earthquake-source test
apache · Feb 6, 2023 · be6881c · be6881c
1 parent 47686e4
commit be6881c
Show file tree

Hide file tree

Showing 24 changed files with 106 additions and 70 deletions.
diff --git a/.github/workflows/yaks-tests.yaml b/.github/workflows/yaks-tests.yaml
@@ -41,8 +41,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CAMEL_K_VERSION: 1.10.3
-  YAKS_VERSION: 0.11.0
+  CAMEL_K_VERSION: 1.12.0-nightly
+  YAKS_VERSION: 0.14.1
   YAKS_IMAGE_NAME: "docker.io/citrusframework/yaks"
   YAKS_RUN_OPTIONS: "--timeout=15m"
 
@@ -57,17 +57,18 @@ jobs:
     - name: Set JitPack coordinates for pull requests
       if: github.event_name == 'pull_request'
       env:
-        BASE_REF: ${{ github.base_ref }}
         HEAD_REF: ${{ github.head_ref }}
         HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
       run: |
         echo "Set JitPack dependency coordinates to ${HEAD_REPO/\//.}:camel-kamelets-utils:${HEAD_REF//\//'~'}-SNAPSHOT"
 
+        export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+        
         # Overwrite JitPack coordinates in the local Kamelets so the tests can use the utility classes in this PR
-        find kamelets -maxdepth 1 -name '*.kamelet.yaml' -exec sed -i "s/github:apache.camel-kamelets:camel-kamelets-utils:${BASE_REF}-SNAPSHOT/github:${HEAD_REPO/\//.}:camel-kamelets-utils:${HEAD_REF//\//'~'}-SNAPSHOT/g" {} +
+        find kamelets -maxdepth 1 -name '*.kamelet.yaml' -exec sed -i "s/github:apache.camel-kamelets:camel-kamelets-utils:${PROJECT_VERSION}/github:${HEAD_REPO/\//.}:camel-kamelets-utils:${HEAD_REF//\//'~'}-SNAPSHOT/g" {} +
     - name: Get Camel K CLI
       run: |
-        curl --fail -L --silent https://github.com/apache/camel-k/releases/download/v${CAMEL_K_VERSION}/camel-k-client-${CAMEL_K_VERSION}-linux-64bit.tar.gz -o kamel.tar.gz
+        curl --fail -L --silent https://github.com/apache/camel-k/releases/download/${CAMEL_K_VERSION}/camel-k-client-${CAMEL_K_VERSION}-linux-64bit.tar.gz -o kamel.tar.gz
         mkdir -p _kamel
         tar -zxf kamel.tar.gz --directory ./_kamel
         sudo mv ./_kamel/kamel /usr/local/bin/
@@ -100,7 +101,7 @@ jobs:
         export KAMEL_INSTALL_REGISTRY=$KIND_REGISTRY
         export KAMEL_INSTALL_REGISTRY_INSECURE=true
 
-        kamel install -w --operator-env-vars KAMEL_INSTALL_DEFAULT_KAMELETS=false
+        kamel install -w --operator-env-vars KAMEL_INSTALL_DEFAULT_KAMELETS=false --maven-repository=https://repository.apache.org/content/repositories/snapshots@id=apache-snapshots@snapshots
 
         # Install the local kamelets
         find kamelets -maxdepth 1 -name '*.kamelet.yaml' -exec kubectl apply -f {} \;
@@ -111,7 +112,6 @@ jobs:
       run: |
         echo "Running tests for Kamelets"
         yaks run test/aws-ddb-sink $YAKS_RUN_OPTIONS
-        
         yaks run test/aws-s3 $YAKS_RUN_OPTIONS
         
         yaks run test/extract-field-action $YAKS_RUN_OPTIONS
@@ -121,11 +121,6 @@ jobs:
         yaks run test/earthquake-source $YAKS_RUN_OPTIONS
         yaks run test/rest-openapi-sink $YAKS_RUN_OPTIONS
         yaks run test/kafka $YAKS_RUN_OPTIONS
-    - name: YAKS Tests experimental Kamelets
-      run: |
-        echo "Running tests for experimental Kamelets"
-        yaks run test/experimental/aws-ddb-sink-exp $YAKS_RUN_OPTIONS
-        yaks run test/experimental/aws-s3-exp $YAKS_RUN_OPTIONS
     - name: YAKS Report
       if: failure()
       run: |

diff --git a/test/aws-ddb-sink/amazonDDBClient.groovy b/test/aws-ddb-sink/amazonDDBClient.groovy
@@ -27,7 +27,7 @@ import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType
 
 DynamoDbClient amazonDDBClient = DynamoDbClient
         .builder()
-        .endpointOverride(URI.create("${YAKS_TESTCONTAINERS_LOCALSTACK_DYNAMODB_URL}"))
+        .endpointOverride(URI.create("${YAKS_TESTCONTAINERS_LOCALSTACK_DYNAMODB_LOCAL_URL}"))
         .credentialsProvider(StaticCredentialsProvider.create(
                 AwsBasicCredentials.create(
                         "${YAKS_TESTCONTAINERS_LOCALSTACK_ACCESS_KEY}",

diff --git a/test/aws-ddb-sink/yaks-config.yaml b/test/aws-ddb-sink/yaks-config.yaml
@@ -41,13 +41,18 @@ config:
       tags:
         - "not @ignored"
     settings:
+      loggers:
+        - name: INTEGRATION_STATUS
+          level: INFO
+        - name: INTEGRATION_LOGS
+          level: INFO
       dependencies:
-        - groupId: com.amazonaws
-          artifactId: aws-java-sdk-dynamodb
-          version: "@aws-java-sdk.version@"
         - groupId: org.apache.camel
           artifactId: camel-aws2-ddb
           version: "@camel.version@"
+        - groupId: software.amazon.awssdk
+          artifactId: dynamodb
+          version: "@aws-java-sdk2.version@"
         - groupId: org.apache.camel
           artifactId: camel-jackson
           version: "@camel.version@"

diff --git a/test/aws-s3/amazonS3Client.groovy b/test/aws-s3/amazonS3Client.groovy
@@ -22,7 +22,7 @@ import software.amazon.awssdk.services.s3.S3Client
 
 S3Client s3 = S3Client
         .builder()
-        .endpointOverride(URI.create("${YAKS_TESTCONTAINERS_LOCALSTACK_S3_URL}"))
+        .endpointOverride(URI.create("${YAKS_TESTCONTAINERS_LOCALSTACK_S3_LOCAL_URL}"))
         .credentialsProvider(StaticCredentialsProvider.create(
                 AwsBasicCredentials.create(
                         "${YAKS_TESTCONTAINERS_LOCALSTACK_ACCESS_KEY}",

diff --git a/test/aws-s3/yaks-config.yaml b/test/aws-s3/yaks-config.yaml
@@ -49,17 +49,17 @@ config:
         - "not @ignored"
     settings:
       loggers:
-        - name: Logger.Message_IN
-          level: DEBUG
-        - name: Logger.Message_OUT
-          level: DEBUG
+        - name: INTEGRATION_STATUS
+          level: INFO
+        - name: INTEGRATION_LOGS
+          level: INFO
       dependencies:
-        - groupId: com.amazonaws
-          artifactId: aws-java-sdk-kinesis
-          version: "@aws-java-sdk.version@"
         - groupId: org.apache.camel
           artifactId: camel-aws2-s3
           version: "@camel.version@"
+        - groupId: software.amazon.awssdk
+          artifactId: s3
+          version: "@aws-java-sdk2.version@"
         - groupId: org.apache.camel
           artifactId: camel-jackson
           version: "@camel.version@"

diff --git a/test/earthquake-source/earthquake-source.feature b/test/earthquake-source/earthquake-source.feature
@@ -18,7 +18,7 @@
 Feature: Kamelet earthquake-source
 
   Background:
-    Given HTTP server timeout is 15000 ms
+    Given HTTP server timeout is 150000 ms
     Given HTTP server "test-service"
 
   Scenario: Create Http server
@@ -32,7 +32,6 @@ Feature: Kamelet earthquake-source
     And create KameletBinding earthquake-source-uri
     Then KameletBinding earthquake-source-uri should be available
     Then Camel K integration earthquake-source-uri should be running
-    And Camel K integration earthquake-source-uri should print Routes startup
 
   Scenario: Verify binding
     Given expect HTTP request header: Content-Type="application/json;charset=UTF-8"

diff --git a/test/earthquake-source/yaks-config.yaml b/test/earthquake-source/yaks-config.yaml
@@ -24,6 +24,12 @@ config:
         value: false
       - name: YAKS_KUBERNETES_AUTO_REMOVE_RESOURCES
         value: false
+    settings:
+      loggers:
+        - name: INTEGRATION_STATUS
+          level: INFO
+        - name: INTEGRATION_LOGS
+          level: INFO
   dump:
     enabled: true
     failedOnly: true

diff --git a/test/experimental/aws-ddb-sink-exp/amazonDDBClient.groovy b/test/experimental/aws-ddb-sink-exp/amazonDDBClient.groovy
@@ -27,7 +27,7 @@ import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType
 
 DynamoDbClient amazonDDBClient = DynamoDbClient
         .builder()
-        .endpointOverride(URI.create("${YAKS_TESTCONTAINERS_LOCALSTACK_DYNAMODB_URL}"))
+        .endpointOverride(URI.create("${YAKS_TESTCONTAINERS_LOCALSTACK_DYNAMODB_LOCAL_URL}"))
         .credentialsProvider(StaticCredentialsProvider.create(
                 AwsBasicCredentials.create(
                         "${YAKS_TESTCONTAINERS_LOCALSTACK_ACCESS_KEY}",

diff --git a/test/experimental/aws-ddb-sink-exp/yaks-config.yaml b/test/experimental/aws-ddb-sink-exp/yaks-config.yaml
@@ -41,13 +41,18 @@ config:
       tags:
         - "not @ignored"
     settings:
+      loggers:
+        - name: INTEGRATION_STATUS
+          level: INFO
+        - name: INTEGRATION_LOGS
+          level: INFO
       dependencies:
-        - groupId: com.amazonaws
-          artifactId: aws-java-sdk-dynamodb
-          version: "@aws-java-sdk.version@"
         - groupId: org.apache.camel
           artifactId: camel-aws2-ddb
           version: "@camel.version@"
+        - groupId: software.amazon.awssdk
+          artifactId: dynamodb
+          version: "@aws-java-sdk2.version@"
         - groupId: org.apache.camel
           artifactId: camel-jackson
           version: "@camel.version@"

diff --git a/test/experimental/aws-s3-exp/amazonS3Client.groovy b/test/experimental/aws-s3-exp/amazonS3Client.groovy
@@ -22,7 +22,7 @@ import software.amazon.awssdk.services.s3.S3Client
 
 S3Client s3 = S3Client
         .builder()
-        .endpointOverride(URI.create("${YAKS_TESTCONTAINERS_LOCALSTACK_S3_URL}"))
+        .endpointOverride(URI.create("${YAKS_TESTCONTAINERS_LOCALSTACK_S3_LOCAL_URL}"))
         .credentialsProvider(StaticCredentialsProvider.create(
                 AwsBasicCredentials.create(
                         "${YAKS_TESTCONTAINERS_LOCALSTACK_ACCESS_KEY}",

diff --git a/test/experimental/aws-s3-exp/yaks-config.yaml b/test/experimental/aws-s3-exp/yaks-config.yaml
@@ -44,17 +44,17 @@ config:
         - "not @ignored"
     settings:
       loggers:
-        - name: Logger.Message_IN
-          level: DEBUG
-        - name: Logger.Message_OUT
-          level: DEBUG
+        - name: INTEGRATION_STATUS
+          level: INFO
+        - name: INTEGRATION_LOGS
+          level: INFO
       dependencies:
-        - groupId: com.amazonaws
-          artifactId: aws-java-sdk-kinesis
-          version: "@aws-java-sdk.version@"
         - groupId: org.apache.camel
           artifactId: camel-aws2-s3
           version: "@camel.version@"
+        - groupId: software.amazon.awssdk
+          artifactId: s3
+          version: "@aws-java-sdk2.version@"
         - groupId: org.apache.camel
           artifactId: camel-jackson
           version: "@camel.version@"

diff --git a/test/extract-field-action/yaks-config.yaml b/test/extract-field-action/yaks-config.yaml
@@ -24,6 +24,12 @@ config:
         value: false
       - name: YAKS_KUBERNETES_AUTO_REMOVE_RESOURCES
         value: false
+    settings:
+      loggers:
+        - name: INTEGRATION_STATUS
+          level: INFO
+        - name: INTEGRATION_LOGS
+          level: INFO
     resources:
       - extract-field-action-binding.yaml
   dump:

diff --git a/test/insert-field-action/yaks-config.yaml b/test/insert-field-action/yaks-config.yaml
@@ -24,6 +24,12 @@ config:
         value: false
       - name: YAKS_KUBERNETES_AUTO_REMOVE_RESOURCES
         value: false
+    settings:
+      loggers:
+        - name: INTEGRATION_STATUS
+          level: INFO
+        - name: INTEGRATION_LOGS
+          level: INFO
     resources:
       - insert-field-action-binding.yaml
   dump:

diff --git a/test/kafka/install.sh b/test/kafka/install.sh
@@ -16,13 +16,13 @@
 # limitations under the License.
 
 # Install Kafka
-kubectl create -f https://strimzi.io/install/latest?namespace=default
+kubectl create -f https://strimzi.io/install/latest?namespace=$YAKS_NAMESPACE
 
 # Apply the `Kafka` Cluster CR file
-kubectl apply -f https://strimzi.io/examples/latest/kafka/kafka-ephemeral-single.yaml
+kubectl apply -f https://strimzi.io/examples/latest/kafka/kafka-ephemeral-single.yaml -n $YAKS_NAMESPACE
 
 # wait for everything to start
-kubectl wait kafka/my-cluster --for=condition=Ready --timeout=300s
+kubectl wait kafka/my-cluster --for=condition=Ready --timeout=300s -n $YAKS_NAMESPACE
 
 # create default topic
-kubectl apply -f https://strimzi.io/examples/latest/topic/kafka-topic.yaml
+kubectl apply -f https://strimzi.io/examples/latest/topic/kafka-topic.yaml -n $YAKS_NAMESPACE
diff --git a/test/kafka/kafka-sink-test.yaml → test/kafka/kafka-sink-binding.yaml b/test/kafka/kafka-sink-test.yaml → test/kafka/kafka-sink-binding.yaml
@@ -18,7 +18,7 @@
 apiVersion: camel.apache.org/v1alpha1
 kind: KameletBinding
 metadata:
-  name: kafka-sink-test
+  name: kafka-sink-binding
 spec:
   source:
     ref:

diff --git a/test/kafka/kafka-sink.feature b/test/kafka/kafka-sink.feature
@@ -33,13 +33,13 @@ Feature: Kafka Kamelet sink
     Given Camel K resource polling configuration
       | maxAttempts          | 200   |
       | delayBetweenAttempts | 2000  |
-    When load KameletBinding kafka-sink-test.yaml
-    Then Camel K integration kafka-sink-test should be running
+    When load KameletBinding kafka-sink-binding.yaml
+    Then Camel K integration kafka-sink-binding should be running
 
   Scenario: Receive message on Kafka topic and verify sink output
     Given Kafka connection
       | url         | ${bootstrap.server.host}.${YAKS_NAMESPACE}:${bootstrap.server.port} |
     Then receive Kafka message with body: ${message}
 
   Scenario: Remove resources
-    Given delete KameletBinding kafka-sink-test
+    Given delete KameletBinding kafka-sink-binding
diff --git a/test/kafka/kafka-source-test.yaml → test/kafka/kafka-source-binding.yaml b/test/kafka/kafka-source-test.yaml → test/kafka/kafka-source-binding.yaml
@@ -18,7 +18,7 @@
 apiVersion: camel.apache.org/v1alpha1
 kind: KameletBinding
 metadata:
-  name: kafka-source-test
+  name: kafka-source-binding
 spec:
   source:
     ref:

diff --git a/test/kafka/kafka-source.feature b/test/kafka/kafka-source.feature
@@ -40,9 +40,9 @@ Feature: Kafka Kamelet source
     Given Camel K resource polling configuration
       | maxAttempts          | 200   |
       | delayBetweenAttempts | 2000  |
-    When load KameletBinding kafka-source-test.yaml
-    Then Camel K integration kafka-source-test should be running
-    And Camel K integration kafka-source-test should print Subscribing ${topic}-Thread 0 to topic ${topic}
+    When load KameletBinding kafka-source-binding.yaml
+    Then Camel K integration kafka-source-binding should be running
+    And Camel K integration kafka-source-binding should print Subscribing ${topic}-Thread 0 to topic ${topic}
     And sleep 10sec
 
   Scenario: Send message to Kafka topic and verify sink output
@@ -62,5 +62,5 @@ Feature: Kafka Kamelet source
     And send HTTP 200 OK
 
   Scenario: Remove resources
-    Given delete KameletBinding kafka-source-test
+    Given delete KameletBinding kafka-source-binding
     And delete Kubernetes service kafka-to-http-service
diff --git a/test/kafka/uninstall.sh b/test/kafka/uninstall.sh
@@ -16,7 +16,7 @@
 # limitations under the License.
 
 # delete Kafka cluster
-kubectl delete kafka/my-cluster
+kubectl delete kafka/my-cluster -n $YAKS_NAMESPACE
 
 # delete default topic
-kubectl delete kafkatopic/my-topic
+kubectl delete kafkatopic/my-topic -n $YAKS_NAMESPACE
diff --git a/test/kafka/yaks-config.yaml b/test/kafka/yaks-config.yaml
@@ -26,23 +26,26 @@ config:
         value: false
       - name: CITRUS_TYPE_CONVERTER
         value: camel
+    settings:
+      loggers:
+        - name: INTEGRATION_STATUS
+          level: INFO
+        - name: INTEGRATION_LOGS
+          level: INFO
     resources:
-      - kafka-source-test.yaml
-      - kafka-sink-test.yaml
+      - kafka-source-binding.yaml
+      - kafka-sink-binding.yaml
   dump:
     enabled: true
     failedOnly: true
     includes:
       - app=camel-k
 pre:
   - name: Install Kafka
-    if: env:CI=true
     script: install.sh
   - name: Setup Kafka roles
-    if: env:CI=true
     run: |
       yaks role --add strimzi
 post:
   - name: Uninstall Kafka
-    if: env:CI=true
     script: uninstall.sh
diff --git a/test/mail-sink/yaks-config.yaml b/test/mail-sink/yaks-config.yaml
@@ -25,6 +25,11 @@ config:
       - name: YAKS_KUBERNETES_AUTO_REMOVE_RESOURCES
         value: false
     settings:
+      loggers:
+        - name: INTEGRATION_STATUS
+          level: INFO
+        - name: INTEGRATION_LOGS
+          level: INFO
       dependencies:
         - groupId: com.consol.citrus
           artifactId: citrus-mail

diff --git a/test/rest-openapi-sink/yaks-config.yaml b/test/rest-openapi-sink/yaks-config.yaml
@@ -24,6 +24,12 @@ config:
         value: false
       - name: YAKS_KUBERNETES_AUTO_REMOVE_RESOURCES
         value: false
+    settings:
+      loggers:
+        - name: INTEGRATION_STATUS
+          level: INFO
+        - name: INTEGRATION_LOGS
+          level: INFO
     resources:
       - pet.json
       - openapi.json