Unable to connect to Kafka docker

[svijetha]

Hi ,

I have below docker compose file , and trying to connect the producer and send message to docker image to a topic , I am unable to see the topic or the message , please help me with the command or let me know what is the issue here

version: '2.1'

services:
  elasticsearch:
    environment:
        - ES_JAVA_OPTS=-Xms2g -Xmx2g
        - cluster.name=mdm.next
        - http.cors.enabled=true
        - http.cors.allow-origin='/.*/'
        - network.host=elasticsearch
    #command:
    #  - "-Ecluster.name=mdm.next"
    #  - "-Epath.logs=/usr/share/elasticsearch/logs"
    #  - "-Ehttp.cors.enabled=true"
    #  - "-Ehttp.cors.allow-origin='/.*/'"
    #  - "-Enetwork.host=elasticsearch"
    image: ${DOCKER_REGISTRY}/mdm-elasticsearch:${SEARCH_IMAGE_VERSION}
    ports:
      - "9200"
      - "9300"
  kibana:
    depends_on:
      - elasticsearch
    environment:
      - "ELASTICSEARCH_URL=http://elasticsearch:9200"
    image: docker.elastic.co/kibana/kibana:6.2.3
    links:
      - elasticsearch
    ports:
      - "5601"
  mdm-mongo:
    image: "mongo:3.4.0"
    command: "mongod --storageEngine wiredTiger --logappend"
    ports:
      - "27017"

  zookeeper:
      image: zookeeper:3.4
      container_name: zookeeper
      hostname: zookeeper
      ports:
       - "2181:2181"
      environment:
       ZOOKEEPER_CLIENT_PORT: 2181

  kafka:
      image : wurstmeister/kafka:1.0.1
      hostname: kafka
      container_name: kafka
      ports:
       - "9092:9092"
      external_links:
       - mdm-mongo
      environment:
       KAFKA_CREATE_TOPICS: "metadata_notification"
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
       KAFKA_ADVERTISED_HOST_NAME: 10.75.143.51
      depends_on:
       - zookeeper
  metadata-service:
    environment:
      - server.port=8081
    hostname: metadata-service
    image: "mdm-docker.artifacts.cloudtrust.rocks/metadata-stub:alameda"
    ports:
      - "8081"
  rabbitmq:
    image: "rabbitmq:3-management"
    ports:
      - "5672"
      - "15672"
  search-rest:
    command:
      - java
      - "-Djava.security.egd=file:/dev/./urandom"
      - "-jar"
      - /opt/mdm-search/search-rest.jar
      - "-Dlog4j2.configuration=file:/opt/mdm-search/log4j2.properties"
      - "--spring.profiles.active=automation"
    depends_on:
      - elasticsearch
      - metadata-service
      - mdm-mongo
      - rabbitmq
      - vault
      - kafka
    environment:
      - spring.cloud.stream.kafka.binder.brokers=kafka:9092
      - spring.cloud.stream.kafka.binder.zkNodes=kafka:2181
      - metadataservice.host=metadata-service
      - metadataservice.port=8081
      - rabbitmq.amqp.host=rabbitmq
      - rabbitmq.amqp.port=5672
      - rabbitmq.mgmt.host=rabbitmq
      - rabbitmq.mgmt.port=1567
      - spring.cloud.consul.enabled=false
      - search_ecore_registration=false
      - authfilter.ipfilter.trusted.ip.ranges=127.0.0.1,1.0.0.1-255.255.255.255
    image: ${DOCKER_REGISTRY}/mdm-search-rest:${SEARCH_IMAGE_VERSION}
    links:
      - elasticsearch
    ports:
      - "8085"
  index-service:
    image: ${DOCKER_REGISTRY}/mdm-index-service:${SEARCH_IMAGE_VERSION}
    ports:
      - 8085
    links:
      - elasticsearch
    environment:
      - spring.cloud.stream.kafka.binder.brokers=kafka:9092
      - spring.cloud.stream.kafka.binder.zkNodes=kafka:2181
      - metadataservice.host=metadata-service
      - metadataservice.port=8081
      - rabbitmq.amqp.host=rabbitmq
      - rabbitmq.amqp.port=5672
      - rabbitmq.mgmt.host=rabbitmq
      - rabbitmq.mgmt.port=1567
      - spring.cloud.consul.enabled=false
      - search_ecore_registration=false
      - authfilter.ipfilter.trusted.ip.ranges=127.0.0.1,1.0.0.1-255.255.255.255
      - tomcatconfiguration.vaultpath=secret/mdm-search
    depends_on:
      - elasticsearch
      - metadata-service
      - mdm-mongo
      - rabbitmq
      - vault
      - kafka
    command:
      - java
      - -Djava.security.egd=file:/dev/./urandom
      #- -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=7878
      - -jar
      - /opt/mdm-index/index-service.jar
      - -Dlog4j2.configuration=file:/opt/mdm-index/log4j2.properties
      - --spring.profiles.active=automation
  vault:
    environment:
      SERVICE_TAGS: "dev,vault,vault_tag"
      SKIP_SETCAP: 1
    hostname: vault
    image: mdm-docker.artifacts.cloudtrust.rocks/vault:latest
    ports:
          - "8200"

[svijetha]

Both the services and Producer Api must be able to connect , kindly help

[sscaling]

It looks like you have not configured Kafka correctly. KAFKA_ADVERTISED_LISTENERS and KAFKA_ADVERTISED_HOST_NAME are mutually exclusive. Check out the Connectivity Guide to see if that answers any of your questions.

[svijetha]

Since i was trying to connect External Producer to docker image of kafka - i was running into this issue , to resolve same . add 127.0.0.1 kafka in /etc/hosts of the machine where producer is running

[acompanies]

I am always getting a '[Consumer clientId=consumer-1, groupId=KafkaExampleProducer] Connection with /127.0.0.1 disconnected' exception.
my producer and consumer are within a containerised microservice within Docker that are connecting to my local KAFKA broker. I have exposed ports for my broker and zookeeper but cannot seem to over come this issue.

docker container run -it --network=host -p2181:2181 -p8097:8097 --name kafka image