Skip to content

Elasticsearch exception [type=version_conflict_engine_exception #3315

Closed
Closed
Elasticsearch exception [type=version_conflict_engine_exception#3315
Labels
questionEnd user question and discussion.
Milestone
6.4.0
@wuxingye

Description

@wuxingye
wuxingye
opened on Aug 25, 2019

Please answer these questions before submitting your issue.

  • Why do you submit this issue?

  • Bug

Bug

  • Which version of SkyWalking, OS and JRE?

SkyWalking: 6.3

Elasticsearch:6.8.2

OS and JRE: official docker images

mode: Zookeeper cluster

detail (docker-compose.yml):

version: '3.3'

services:
  es6_01:
    restart: always
    container_name: es6_01
    image: elasticsearch:6.8.2
    hostname: es6_01
    privileged: true
    environment:
      - TZ="Asia/Shanghai"
      - node.name=es6_01
      - network.host=es6_01
      - discovery.zen.minimum_master_nodes=2
      - discovery.zen.ping.unicast.hosts=es6_01,es6_02,es6_03
      - cluster.name=es6
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - es6_data1:/usr/share/elasticsearch/data
    ports:
      - 9211:9200
      - 9311:9300

  es6_02:
    restart: always
    container_name: es6_02
    image: elasticsearch:6.8.2
    hostname: es6_02
    privileged: true
    environment:
      - TZ="Asia/Shanghai"
      - node.name=es6_02
      - network.host=es6_02
      - discovery.zen.minimum_master_nodes=2
      - discovery.zen.ping.unicast.hosts=es6_01,es6_02,es6_03
      - cluster.name=es6
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - es6_data2:/usr/share/elasticsearch/data
    ports:
      - 9212:9200
      - 9312:9300

  es6_03:
    restart: always
    container_name: es6_03
    image: elasticsearch:6.8.2
    hostname: es6_03
    privileged: true
    environment:
      - TZ="Asia/Shanghai"
      - node.name=es6_03
      - network.host=es6_03
      - discovery.zen.minimum_master_nodes=2
      - discovery.zen.ping.unicast.hosts=es6_01,es6_02,es6_03
      - cluster.name=es6
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - es6_data3:/usr/share/elasticsearch/data
    ports:
      - 9213:9200
      - 9313:9300

  oap1:
    image: apache/skywalking-oap-server:6.3.0
    container_name: oap1
    hostname: oap1
    privileged: true
    restart: always
    ports:
      - 11801:11800
      - 12801:12800
    environment:
      TZ: Asia/Shanghai
      SW_CLUSTER: zookeeper
      SW_CLUSTER_ZK_HOST_PORT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      SW_STORAGE: elasticsearch
      SW_STORAGE_ES_CLUSTER_NODES: es6_01:9200,es6_02:9200,es6_03:9200
    depends_on:
      - es6_01
      - es6_02
      - es6_03
    links:
      - es6_01
      - es6_02
      - es6_03
  oap2:
    image: apache/skywalking-oap-server:6.3.0
    container_name: oap2
    hostname: oap2
    privileged: true
    restart: always
    ports:
      - 11802:11800
      - 12802:12800
    environment:
      TZ: Asia/Shanghai
      SW_CLUSTER: zookeeper
      SW_CLUSTER_ZK_HOST_PORT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      SW_STORAGE: elasticsearch
      SW_STORAGE_ES_CLUSTER_NODES: es6_01:9200,es6_02:9200,es6_03:9200
    depends_on:
      - es6_01
      - es6_02
      - es6_03
    links:
      - es6_01
      - es6_02
      - es6_03
  oap3:
    image: apache/skywalking-oap-server:6.3.0
    container_name: oap3
    hostname: oap3
    privileged: true
    restart: always
    ports:
      - 11803:11800
      - 12803:12800
    environment:
      TZ: Asia/Shanghai
      SW_CLUSTER: zookeeper
      SW_CLUSTER_ZK_HOST_PORT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      SW_STORAGE: elasticsearch
      SW_STORAGE_ES_CLUSTER_NODES: es6_01:9200,es6_02:9200,es6_03:9200
    depends_on:
      - es6_01
      - es6_02
      - es6_03
    links:
      - es6_01
      - es6_02
      - es6_03

  ui:
    image: apache/skywalking-ui:6.3.0
    container_name: ui
    depends_on:
      - oap1
      - oap2
      - oap3
    links:
      - oap1
      - oap2
      - oap3
    restart: always
    ports:
      - 2800:8080
    environment:
      SW_OAP_ADDRESS: oap1:12800,oap2:12800,oap3:12800

networks:
  default:
    external:
      name: kakarot

volumes:
  es6_data1:
    driver_opts:
      type: none
      device: /data/elasticsearch6/data1
      o: bind
  es6_data2:
    driver_opts:
      type: none
      device: /data/elasticsearch6/data2
      o: bind
  es6_data3:
    driver_opts:
      type: none
      device: /data/elasticsearch6/data3
      o: bind
  • Which company or project?

Thomas Cook

  • What happen?
    If possible, provide a way for reproducing the error. e.g. demo application, component version.

error below:

2019-08-25 08:31:32,551 - org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker -48014893 [DataCarrier.REGISTER_L2.BulkConsumePool.0.Thread] ERROR [] - Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]]
org.elasticsearch.ElasticsearchStatusException: Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]]
	at org.elasticsearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:177) ~[elasticsearch-6.3.2.jar:6.3.2]
	at org.elasticsearch.client.RestHighLevelClient.parseEntity(RestHighLevelClient.java:653) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
	at org.elasticsearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:628) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
	at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:535) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
	at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:508) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
	at org.elasticsearch.client.RestHighLevelClient.update(RestHighLevelClient.java:366) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
	at org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient.forceUpdate(ElasticSearchClient.java:254) ~[library-client-6.3.0.jar:6.3.0]
	at org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.RegisterEsDAO.forceUpdate(RegisterEsDAO.java:56) ~[storage-elasticsearch-plugin-6.3.0.jar:6.3.0]
	at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.lambda$onWork$0(RegisterPersistentWorker.java:89) ~[server-core-6.3.0.jar:6.3.0]
	at java.util.HashMap$Values.forEach(HashMap.java:981) [?:1.8.0_181]
	at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.onWork(RegisterPersistentWorker.java:84) [server-core-6.3.0.jar:6.3.0]
	at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.access$100(RegisterPersistentWorker.java:35) [server-core-6.3.0.jar:6.3.0]
	at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker$PersistentConsumer.consume(RegisterPersistentWorker.java:141) [server-core-6.3.0.jar:6.3.0]
	at org.apache.skywalking.apm.commons.datacarrier.consumer.MultipleChannelsConsumer.consume(MultipleChannelsConsumer.java:82) [apm-datacarrier-6.3.0.jar:6.3.0]
	at org.apache.skywalking.apm.commons.datacarrier.consumer.MultipleChannelsConsumer.run(MultipleChannelsConsumer.java:53) [apm-datacarrier-6.3.0.jar:6.3.0]
	Suppressed: org.elasticsearch.client.ResponseException: method [POST], host [http://es6_01:9200], URI [/service_inventory/type/SkyWalking_0_0/_update?refresh=true&timeout=1m], status line [HTTP/1.1 409 Conflict]
{"error":{"root_cause":[{"type":"version_conflict_engine_exception","reason":"[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]","index_uuid":"PQvASg4pRtSElaKDAc5KwA","shard":"0","index":"service_inventory"}],"type":"version_conflict_engine_exception","reason":"[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]","index_uuid":"PQvASg4pRtSElaKDAc5KwA","shard":"0","index":"service_inventory"},"status":409}
		at org.elasticsearch.client.RestClient$SyncResponseListener.get(RestClient.java:705) ~[elasticsearch-rest-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235) ~[elasticsearch-rest-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestClient.performRequest(RestClient.java:198) ~[elasticsearch-rest-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:522) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:508) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestHighLevelClient.update(RestHighLevelClient.java:366) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
		at org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient.forceUpdate(ElasticSearchClient.java:254) ~[library-client-6.3.0.jar:6.3.0]
		at org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.RegisterEsDAO.forceUpdate(RegisterEsDAO.java:56) ~[storage-elasticsearch-plugin-6.3.0.jar:6.3.0]
		at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.lambda$onWork$0(RegisterPersistentWorker.java:89) ~[server-core-6.3.0.jar:6.3.0]
		at java.util.HashMap$Values.forEach(HashMap.java:981) [?:1.8.0_181]
		at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.onWork(RegisterPersistentWorker.java:84) [server-core-6.3.0.jar:6.3.0]
		at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.access$100(RegisterPersistentWorker.java:35) [server-core-6.3.0.jar:6.3.0]
		at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker$PersistentConsumer.consume(RegisterPersistentWorker.java:141) [server-core-6.3.0.jar:6.3.0]
		at org.apache.skywalking.apm.commons.datacarrier.consumer.MultipleChannelsConsumer.consume(MultipleChannelsConsumer.java:82) [apm-datacarrier-6.3.0.jar:6.3.0]
		at org.apache.skywalking.apm.commons.datacarrier.consumer.MultipleChannelsConsumer.run(MultipleChannelsConsumer.java:53) [apm-datacarrier-6.3.0.jar:6.3.0]
	Caused by: org.elasticsearch.client.ResponseException: method [POST], host [http://es6_01:9200], URI [/service_inventory/type/SkyWalking_0_0/_update?refresh=true&timeout=1m], status line [HTTP/1.1 409 Conflict]
{"error":{"root_cause":[{"type":"version_conflict_engine_exception","reason":"[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]","index_uuid":"PQvASg4pRtSElaKDAc5KwA","shard":"0","index":"service_inventory"}],"type":"version_conflict_engine_exception","reason":"[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]","index_uuid":"PQvASg4pRtSElaKDAc5KwA","shard":"0","index":"service_inventory"},"status":409}
		at org.elasticsearch.client.RestClient$1.completed(RestClient.java:377) ~[elasticsearch-rest-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestClient$1.completed(RestClient.java:366) ~[elasticsearch-rest-client-6.3.2.jar:6.3.2]
		at org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:119) ~[httpcore-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.responseCompleted(DefaultClientExchangeHandlerImpl.java:177) ~[httpasyncclient-4.1.2.jar:4.1.2]
		at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.processResponse(HttpAsyncRequestExecutor.java:436) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.inputReady(HttpAsyncRequestExecutor.java:326) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.DefaultNHttpClientConnection.consumeInput(DefaultNHttpClientConnection.java:265) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:81) ~[httpasyncclient-4.1.2.jar:4.1.2]
		at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:39) ~[httpasyncclient-4.1.2.jar:4.1.2]
		at org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:114) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_181]

Requirement or improvement

  • Please describe about your requirements or improvement suggestions.

how to set up a zookeeper based skywalking cluster?

Metadata

Metadata

Assignees

No one assigned

    Labels

    questionEnd user question and discussion.

    Type

    No type

    Projects

    No projects

    Milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions