From 980e00ec57fd0a458671abc4f04287a8a042fc47 Mon Sep 17 00:00:00 2001 From: badass-techie Date: Thu, 25 Jan 2024 17:13:27 +0300 Subject: [PATCH] Add EFK stack to docker compose --- Dockerfile | 12 +++++++ docker-compose.yml | 72 ++++++++++++++++++++++++++++++++++++++++ fluentd/conf/fluent.conf | 17 ++++++++++ 3 files changed, 101 insertions(+) create mode 100644 fluentd/conf/fluent.conf diff --git a/Dockerfile b/Dockerfile index 34fbd30..8d14e1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,3 +103,15 @@ RUN pip install -r requirements.txt # Run the python file ENTRYPOINT ["python", "app.py"] + + +# Fluentd +FROM fluent/fluentd:v1.12.0-debian-1.0 AS fluentd + +USER root + +RUN gem uninstall -I elasticsearch && gem install elasticsearch -v 7.13.3 + +RUN ["gem", "install", "fluent-plugin-elasticsearch", "--no-document", "--version", "5.0.3"] + +USER fluent diff --git a/docker-compose.yml b/docker-compose.yml index aa69650..5492071 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,12 @@ services: - "8080:8080" env_file: - .env + logging: + driver: fluentd + options: + fluentd-address: localhost:24224 + fluentd-async-connect: "true" + tag: api-gateway cart-db: image: redis:7.2.1 @@ -56,6 +62,12 @@ services: depends_on: cart-db: condition: service_healthy + logging: + driver: fluentd + options: + fluentd-address: localhost:24224 + fluentd-async-connect: "true" + tag: cart identity-db: image: postgres:16.0 @@ -87,6 +99,12 @@ services: depends_on: identity-db: condition: service_healthy + logging: + driver: fluentd + options: + fluentd-address: localhost:24224 + fluentd-async-connect: "true" + tag: identity order-db: image: postgres:16.0 @@ -120,6 +138,12 @@ services: condition: service_healthy rabbitmq: condition: service_healthy + logging: + driver: fluentd + options: + fluentd-address: localhost:24224 + fluentd-async-connect: "true" + tag: order product-db: image: mongo:5.0-focal @@ -147,6 +171,12 @@ services: condition: service_healthy rabbitmq: condition: service_healthy + logging: + driver: fluentd + options: + fluentd-address: localhost:24224 + fluentd-async-connect: "true" + tag: product payment: build: @@ -160,6 +190,12 @@ services: depends_on: rabbitmq: condition: service_healthy + logging: + driver: fluentd + options: + fluentd-address: localhost:24224 + fluentd-async-connect: "true" + tag: payment prometheus: image: prom/prometheus:v2.47.2 @@ -181,6 +217,41 @@ services: depends_on: - prometheus + elasticsearch: + image: elasticsearch:7.17.0 + environment: + - discovery.type=single-node + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + ports: + - 9200:9200 + + fluentd: + build: + context: . + dockerfile: Dockerfile.dev + target: fluentd + volumes: + - ./fluentd/conf:/fluentd/etc + ports: + - "24224:24224" + - "24224:24224/udp" + restart: unless-stopped + depends_on: + - elasticsearch + + kibana: + image: kibana:7.17.0 + ports: + - "5601:5601" + environment: + - ELASTICSEARCH_URL=http://elasticsearch:9200 + volumes: + - elasticsearch-data:/usr/share/kibana/data + restart: unless-stopped + depends_on: + - elasticsearch + volumes: rabbitmq-data: @@ -189,3 +260,4 @@ volumes: order-db: product-db: grafana-data: + elasticsearch-data: diff --git a/fluentd/conf/fluent.conf b/fluentd/conf/fluent.conf new file mode 100644 index 0000000..0dea086 --- /dev/null +++ b/fluentd/conf/fluent.conf @@ -0,0 +1,17 @@ + + @type forward + port 24224 + bind 0.0.0.0 + + + + @type elasticsearch + host elasticsearch + port 9200 + logstash_format true + logstash_prefix fluentd + logstash_dateformat %Y%m%d + include_tag_key true + tag_key @log_name + flush_interval 1s + \ No newline at end of file