ES: do not set _type when using Bulk API for v7 client (#5104)

* ES: do not set type for v7 * add buildkite job * disable os2 security
uber · Mar 10, 2023 · 45c3cee · 45c3cee
1 parent 434bcee
commit 45c3cee
Show file tree

Hide file tree

Showing 3 changed files with 85 additions and 4 deletions.
diff --git a/.buildkite/pipeline-pull-request.yml b/.buildkite/pipeline-pull-request.yml
@@ -62,7 +62,20 @@ steps:
       - docker-compose#v3.0.0:
           run: integration-test-cassandra
           config: docker/buildkite/docker-compose-es7.yml
-
+  - label: ":golang: integration test with cassandra with OpenSearch v2"
+    agents:
+      queue: "workers"
+      docker: "*"
+    command: "make cover_integration_profile"
+    artifact_paths:
+      - ".build/coverage/*.out"
+    retry:
+      automatic:
+        limit: 1
+    plugins:
+      - docker-compose#v3.0.0:
+          run: integration-test-cassandra
+          config: docker/buildkite/docker-compose-opensearch2.yml
   - label: ":golang: integration ndc test with cassandra"
     agents:
       queue: "workers"

diff --git a/common/elasticsearch/client_v7.go b/common/elasticsearch/client_v7.go
@@ -475,14 +475,12 @@ func (v *v7BulkProcessor) Add(request *GenericBulkableAddRequest) {
 	case BulkableDeleteRequest:
 		req = elastic.NewBulkDeleteRequest().
 			Index(request.Index).
-			Type(request.Type).
 			Id(request.ID).
 			VersionType(request.VersionType).
 			Version(request.Version)
 	case BulkableIndexRequest:
 		req = elastic.NewBulkIndexRequest().
 			Index(request.Index).
-			Type(request.Type).
 			Id(request.ID).
 			VersionType(request.VersionType).
 			Version(request.Version).
@@ -493,7 +491,6 @@ func (v *v7BulkProcessor) Add(request *GenericBulkableAddRequest) {
 		req = elastic.NewBulkIndexRequest().
 			OpType("create").
 			Index(request.Index).
-			Type(request.Type).
 			Id(request.ID).
 			VersionType("internal").
 			Doc(request.Doc)

diff --git a/docker/buildkite/docker-compose-opensearch2.yml b/docker/buildkite/docker-compose-opensearch2.yml
@@ -0,0 +1,71 @@
+version: "3.5"
+
+services:
+  cassandra:
+    image: cassandra:3.11
+    networks:
+      services-network:
+        aliases:
+          - cassandra
+
+  zookeeper:
+    image: wurstmeister/zookeeper:3.4.6
+    networks:
+      services-network:
+        aliases:
+          - zookeeper
+
+  kafka:
+    image: wurstmeister/kafka:2.12-2.1.1
+    depends_on:
+      - zookeeper
+    networks:
+      services-network:
+        aliases:
+          - kafka
+    environment:
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+
+  elasticsearch:
+    image: opensearchproject/opensearch:2.5.0
+    networks:
+      services-network:
+        aliases:
+          - elasticsearch
+    environment:
+      - discovery.type=single-node
+      - plugins.security.disabled=true
+
+  integration-test-cassandra:
+    build:
+      context: ../../
+      dockerfile: ./docker/buildkite/Dockerfile
+    environment:
+      - "CASSANDRA=1"
+      - "CASSANDRA_SEEDS=cassandra"
+      - "ES_SEEDS=elasticsearch"
+      - "KAFKA_SEEDS=kafka"
+      - "TEST_TAG=esintegration"
+      - "ES_VERSION=v7"
+      - BUILDKITE_AGENT_ACCESS_TOKEN
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_BUILD_ID
+      - BUILDKITE_BUILD_NUMBER
+    depends_on:
+      - cassandra
+      - elasticsearch
+      - kafka
+    volumes:
+      - ../../:/cadence
+      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
+    networks:
+      services-network:
+        aliases:
+          - integration-test
+
+networks:
+  services-network:
+    name: services-network
+    driver: bridge