diff --git a/.buildkite/pipeline-master.yml b/.buildkite/pipeline-master.yml index 5ad5470f0c..95e372c0b5 100644 --- a/.buildkite/pipeline-master.yml +++ b/.buildkite/pipeline-master.yml @@ -138,6 +138,21 @@ steps: run: integration-test-ndc-postgres config: docker/buildkite/docker-compose.yml + - label: ":golang: async wf integration test with kafka" + agents: + queue: "workers" + docker: "*" + commands: + - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code + - "go test -timeout 60s -run ^TestAsyncWFIntegrationSuite -tags asyncwfintegration -count 1 -v github.com/uber/cadence/host" + retry: + automatic: + limit: 1 + plugins: + - docker-compose#v3.0.0: + run: integration-test-async-wf + config: docker/buildkite/docker-compose.yml + - wait - label: ":golang: code-coverage" diff --git a/.buildkite/pipeline-pull-request.yml b/.buildkite/pipeline-pull-request.yml index 6f2fab22b4..1ff3a62111 100644 --- a/.buildkite/pipeline-pull-request.yml +++ b/.buildkite/pipeline-pull-request.yml @@ -68,6 +68,7 @@ steps: - docker-compose#v3.0.0: run: integration-test-cassandra config: docker/buildkite/docker-compose-es7.yml + - label: ":golang: integration test with cassandra with OpenSearch v2" agents: queue: "workers" @@ -84,6 +85,7 @@ steps: - docker-compose#v3.0.0: run: integration-test-cassandra config: docker/buildkite/docker-compose-opensearch2.yml + - label: ":golang: integration ndc test with cassandra" agents: queue: "workers" @@ -169,6 +171,21 @@ steps: run: integration-test-ndc-postgres config: docker/buildkite/docker-compose.yml + - label: ":golang: async wf integration test with kafka" + agents: + queue: "workers" + docker: "*" + commands: + - "make .just-build" # ensure that we are not rebuilding binaries and not regenerating code + - "go test -timeout 60s -run ^TestAsyncWFIntegrationSuite -tags asyncwfintegration -count 1 -v github.com/uber/cadence/host" + retry: + automatic: + limit: 1 + plugins: + - docker-compose#v3.0.0: + run: integration-test-async-wf + config: docker/buildkite/docker-compose.yml + - wait - label: ":golang: code-coverage" diff --git a/docker/buildkite/docker-compose-local-async-wf.yml b/docker/buildkite/docker-compose-local-async-wf.yml index 24a7ddd324..ce2fa7d333 100644 --- a/docker/buildkite/docker-compose-local-async-wf.yml +++ b/docker/buildkite/docker-compose-local-async-wf.yml @@ -67,6 +67,7 @@ services: - "7935:7935" - "7939:7939" environment: + - "ASYNC_WF_KAFKA_QUEUE_TOPIC=async-wf-topic1" - "CASSANDRA=1" - "CASSANDRA_SEEDS=cassandra" - "ES_SEEDS=elasticsearch" diff --git a/docker/buildkite/docker-compose.yml b/docker/buildkite/docker-compose.yml index 7b769415e6..2ec4bafb84 100644 --- a/docker/buildkite/docker-compose.yml +++ b/docker/buildkite/docker-compose.yml @@ -55,6 +55,9 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # for async wf tests, create a topic with 10 partitions and 1 replica + # topic name must match ASYNC_WF_KAFKA_QUEUE_TOPIC + KAFKA_CREATE_TOPICS: "async-wf-topic1:10:1" elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.22 @@ -281,6 +284,34 @@ services: aliases: - integration-test-ndc + integration-test-async-wf: + build: + context: ../../ + dockerfile: ./docker/buildkite/Dockerfile + environment: + - "ASYNC_WF_KAFKA_QUEUE_TOPIC=async-wf-topic1" + - "CASSANDRA=1" + - "CASSANDRA_SEEDS=cassandra" + - "ES_SEEDS=elasticsearch" + - "KAFKA_SEEDS=kafka" + - "KAFKA_PORT=9092" + - BUILDKITE_AGENT_ACCESS_TOKEN + - BUILDKITE_JOB_ID + - BUILDKITE_BUILD_ID + - BUILDKITE_BUILD_NUMBER + depends_on: + cassandra: + condition: service_healthy + kafka: + condition: service_started + volumes: + - ../../:/cadence + - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent + networks: + services-network: + aliases: + - integration-test-async-wf + coverage-report: build: context: ../../ diff --git a/host/testdata/integration_async_wf_with_kafka_cluster.yaml b/host/testdata/integration_async_wf_with_kafka_cluster.yaml index 7b75405aad..3b1db5da96 100644 --- a/host/testdata/integration_async_wf_with_kafka_cluster.yaml +++ b/host/testdata/integration_async_wf_with_kafka_cluster.yaml @@ -10,7 +10,7 @@ asyncwfqueues: connection: brokers: - "${KAFKA_SEEDS}:${KAFKA_PORT}" - topic: async-wf-topic1 + topic: "${ASYNC_WF_KAFKA_QUEUE_TOPIC}" messagingclientconfig: usemock: true historyconfig: