docker-compose.yaml

version: "3.7"
services:
  redpanda:
    command:
      - redpanda
      - start
      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
      - --advertise-kafka-addr internal://redpanda:9092,external://localhost:19092
      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
      - --advertise-pandaproxy-addr internal://redpanda:8082,external://localhost:18082
      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
      - --rpc-addr redpanda:33145
      - --advertise-rpc-addr redpanda:33145
      - --smp 1
      - --memory 1G
      - --mode dev-container
      - --default-log-level=debug
    image: docker.redpanda.com/redpandadata/redpanda:v23.1.7
    container_name: redpanda
    volumes:
      # sudo chown -R 101:101 ./logs # set correct permissions to prevent error 13
      - ./logs/redpanda:/var/lib/redpanda/data
    ports:
      - "18081:18081"
      - "18082:18082"
      - "19092:19092"
      - "19644:9644"
  console:
    container_name: redpanda-console
    image: docker.redpanda.com/vectorized/console:v2.2.3
    entrypoint: /bin/sh
    command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
    environment:
      CONFIG_FILEPATH: /tmp/config.yml
      CONSOLE_CONFIG_FILE: |
        kafka:
          brokers: ["redpanda:9092"]
          schemaRegistry:
            enabled: true
            urls: ["http://redpanda:8081"]
        redpanda:
          adminApi:
            enabled: true
            urls: ["http://redpanda:9644"]
    ports:
      - "8080:8080"
    depends_on:
      - redpanda
  jobmanager:
    build: .
    container_name: jobmanager
    ports:
      - "8081:8081"
      - "9249:9249"
    command: jobmanager
    volumes:
      - ./jars/:/opt/flink/jars
      - ./logs/flink/jm:/opt/flink/temp
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        metrics.reporters: prom
        metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory
        metrics.reporter.prom.port: 9249
        state.backend.rocksdb.metrics.actual-delayed-write-rate: true
        state.backend.rocksdb.metrics.background-errors: true
        state.backend.rocksdb.metrics.block-cache-capacity: true
        state.backend.rocksdb.metrics.estimate-num-keys: true
        state.backend.rocksdb.metrics.estimate-live-data-size: true
        state.backend.rocksdb.metrics.estimate-pending-compaction-bytes: true
        state.backend.rocksdb.metrics.num-running-compactions: true
        state.backend.rocksdb.metrics.compaction-pending: true
        state.backend.rocksdb.metrics.is-write-stopped: true
        state.backend.rocksdb.metrics.num-running-flushes: true
        state.backend.rocksdb.metrics.mem-table-flush-pending: true
        state.backend.rocksdb.metrics.block-cache-usage: true
        state.backend.rocksdb.metrics.size-all-mem-tables: true
        state.backend.rocksdb.metrics.num-live-versions: true
        state.backend.rocksdb.metrics.block-cache-pinned-usage: true
        state.backend.rocksdb.metrics.estimate-table-readers-mem: true
        state.backend.rocksdb.metrics.num-snapshots: true
        state.backend.rocksdb.metrics.num-entries-active-mem-table: true
        state.backend.rocksdb.metrics.num-deletes-imm-mem-tables: true
  taskmanager1:
    build: .
    container_name: taskmanager1
    depends_on:
      - jobmanager
    command: taskmanager
    ports:
      - "9250:9249"
    volumes:
      - ./logs/flink/tm1:/opt/flink/temp
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 5
        metrics.reporters: prom
        metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory
        metrics.reporter.prom.port: 9249
        state.backend.rocksdb.metrics.actual-delayed-write-rate: true
        state.backend.rocksdb.metrics.background-errors: true
        state.backend.rocksdb.metrics.block-cache-capacity: true
        state.backend.rocksdb.metrics.estimate-num-keys: true
        state.backend.rocksdb.metrics.estimate-live-data-size: true
        state.backend.rocksdb.metrics.estimate-pending-compaction-bytes: true
        state.backend.rocksdb.metrics.num-running-compactions: true
        state.backend.rocksdb.metrics.compaction-pending: true
        state.backend.rocksdb.metrics.is-write-stopped: true
        state.backend.rocksdb.metrics.num-running-flushes: true
        state.backend.rocksdb.metrics.mem-table-flush-pending: true
        state.backend.rocksdb.metrics.block-cache-usage: true
        state.backend.rocksdb.metrics.size-all-mem-tables: true
        state.backend.rocksdb.metrics.num-live-versions: true
        state.backend.rocksdb.metrics.block-cache-pinned-usage: true
        state.backend.rocksdb.metrics.estimate-table-readers-mem: true
        state.backend.rocksdb.metrics.num-snapshots: true
        state.backend.rocksdb.metrics.num-entries-active-mem-table: true
        state.backend.rocksdb.metrics.num-deletes-imm-mem-tables: true
  taskmanager2:
    build: .
    container_name: taskmanager2
    depends_on:
      - jobmanager
    command: taskmanager
    ports:
      - "9251:9249"
    volumes:
      - ./logs/flink/tm2:/opt/flink/temp
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 5
        metrics.reporters: prom
        metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory
        metrics.reporter.prom.port: 9249
        state.backend.rocksdb.metrics.actual-delayed-write-rate: true
        state.backend.rocksdb.metrics.background-errors: true
        state.backend.rocksdb.metrics.block-cache-capacity: true
        state.backend.rocksdb.metrics.estimate-num-keys: true
        state.backend.rocksdb.metrics.estimate-live-data-size: true
        state.backend.rocksdb.metrics.estimate-pending-compaction-bytes: true
        state.backend.rocksdb.metrics.num-running-compactions: true
        state.backend.rocksdb.metrics.compaction-pending: true
        state.backend.rocksdb.metrics.is-write-stopped: true
        state.backend.rocksdb.metrics.num-running-flushes: true
        state.backend.rocksdb.metrics.mem-table-flush-pending: true
        state.backend.rocksdb.metrics.block-cache-usage: true
        state.backend.rocksdb.metrics.size-all-mem-tables: true
        state.backend.rocksdb.metrics.num-live-versions: true
        state.backend.rocksdb.metrics.block-cache-pinned-usage: true
        state.backend.rocksdb.metrics.estimate-table-readers-mem: true
        state.backend.rocksdb.metrics.num-snapshots: true
        state.backend.rocksdb.metrics.num-entries-active-mem-table: true
        state.backend.rocksdb.metrics.num-deletes-imm-mem-tables: true
  postgres:
    image: postgres:latest
    container_name: postgres
    restart: always
    environment:
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=postgres
    ports:
      - '5432:5432'
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    command:
      - '--config.file=/etc/prometheus/config.yaml'
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus:/etc/prometheus
  #      - ./prom_data:/prometheus
  grafana:
    image: grafana/grafana
    container_name: grafana
    ports:
      - "3000:3000"
    restart: unless-stopped
    environment:
      - GF_SECURITY_ADMIN_USER=grafana
      - GF_SECURITY_ADMIN_PASSWORD=grafana
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning
      - ./grafana/dashboards:/var/lib/grafana/dashboards