Merge pull request #543 from Altinity/refactor_testflows_tests

Refactor TestFlows tests related to Lightweight

.github/workflows/testflows-sink-connector-lightweight.yml

@@ -8,6 +8,10 @@ on:
         description: "Lightweight connector docker image"
         required: true
         type: string
+      package:
+        description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'"
+        type: string
+        default: docker://clickhouse/clickhouse-server:23.3
     secrets:
       DOCKERHUB_USERNAME:
         required: false
@@ -23,6 +27,10 @@ on:
         description: "Lightweight connector docker image"
         required: true
         type: string
+      package:
+        description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'"
+        type: string
+        default: docker://clickhouse/clickhouse-server:23.3
       extra_args:
         description: "Specific Suite To Run (Default * to run everything)."
         required: false
@@ -71,7 +79,7 @@ jobs:
 
       - name: Run testflows tests
         working-directory: sink-connector-lightweight/tests/integration
-        run: python3 -u regression.py --only "/mysql to clickhouse replication/mysql to clickhouse replication auto/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path=docker://clickhouse/clickhouse-server:23.3 --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
+        run: python3 -u regression.py --only "/mysql to clickhouse replication/auto table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
 
       - name: Create tfs results report
         if: always()
@@ -98,5 +106,6 @@ jobs:
           name: testflows-sink-connector-lightweight-artefacts
           path: |
             sink-connector-lightweight/tests/integration/logs/*.log
+            sink-connector-lightweight/tests/integration/env/auto/configs/*.yml
           if-no-files-found: error
           retention-days: 60

sink-connector-lightweight/tests/integration/env/auto/clickhouse-sink-connector-lt-service.yml

@@ -0,0 +1,17 @@
+version: "2.3"
+
+services:
+    clickhouse-sink-connector-lt:
+      hostname: clickhouse-sink-connector-lt
+      image: ${SINK_CONNECTOR_IMAGE}
+      restart: "no"
+      expose:
+        - "8083"
+        - "1976"
+        - "5005"
+      extra_hosts:
+        - "host.docker.internal:host-gateway"
+      entrypoint: bash -c "tail -f /dev/null"
+      volumes:
+        - ./configs:/configs
+        - ../../logs:/logs

sink-connector-lightweight/tests/integration/env/auto/configs/config.yml

@@ -0,0 +1,49 @@
+name: my_connector
+database.hostname: mysql-master
+database.port: '3306'
+database.user: root
+database.password: root
+database.server.name: ER54
+database.include.list: test
+clickhouse.server.url: clickhouse
+clickhouse.server.user: root
+clickhouse.server.password: root
+clickhouse.server.port: '8123'
+clickhouse.server.database: test
+database.allowPublicKeyRetrieval: 'true'
+snapshot.mode: initial
+offset.flush.interval.ms: '5000'
+connector.class: io.debezium.connector.mysql.MySqlConnector
+offset.storage: io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore
+offset.storage.jdbc.offset.table.name: altinity_sink_connector.replica_source_info
+offset.storage.jdbc.url: jdbc:clickhouse://clickhouse:8123/altinity_sink_connector
+offset.storage.jdbc.user: root
+offset.storage.jdbc.password: root
+offset.storage.jdbc.offset.table.ddl: |-
+  CREATE TABLE if not exists %s
+  (
+      `id` String,
+      `offset_key` String,
+      `offset_val` String,
+      `record_insert_ts` DateTime,
+      `record_insert_seq` UInt64,
+      `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
+  )
+  ENGINE = ReplacingMergeTree(_version)
+  ORDER BY id
+  SETTINGS index_granularity = 8198
+offset.storage.jdbc.offset.table.delete: delete from %s where 1=1
+schema.history.internal: io.debezium.storage.jdbc.history.JdbcSchemaHistory
+schema.history.internal.jdbc.url: jdbc:clickhouse://clickhouse:8123/altinity_sink_connector
+schema.history.internal.jdbc.user: root
+schema.history.internal.jdbc.password: root
+schema.history.internal.jdbc.schema.history.table.ddl: |-
+  CREATE TABLE if not exists %s
+  (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id
+schema.history.internal.jdbc.schema.history.table.name: altinity_sink_connector.replicate_schema_history
+replacingmergetree.delete.column: _sign
+enable.snapshot.ddl: 'true'
+database.connectionTimeZone: UTC
+database.serverTimezone: UTC
+clickhouse.datetime.timezone: UTC
+auto.create.tables: 'true'

sink-connector-lightweight/tests/integration/env/auto/docker-compose.yml

@@ -24,18 +24,9 @@ services:
 
 
   clickhouse-sink-connector-lt:
-    hostname: clickhouse-sink-connector-lt
-    image:  ${SINK_CONNECTOR_IMAGE}
-    restart: "no"
-    expose:
-      - "8083"
-      - "1976"
-      - "5005"
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - ./data:/data
-      - ./config.yml:/config.yml
+    extends:
+      file: clickhouse-sink-connector-lt-service.yml
+      service: clickhouse-sink-connector-lt
 
   zookeeper:
     extends:
@@ -61,19 +52,6 @@ services:
       zookeeper:
         condition: service_healthy
 
-
-  bash-tools:
-    image: registry.gitlab.com/altinity-public/container-images/test/bash-tools:5.0
-    hostname: bash-tools
-    restart: "no"
-    entrypoint:
-      - /bin/sleep
-    command:
-      - infinity
-    volumes:
-      - ../../tests/manual_scripts:/manual_scripts
-      - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/share_folder:/tmp/share_folder"
-
   clickhouse1:
     extends:
       file: clickhouse-service.yml