risedev.yml

risedev:
  #########################################
  ### Configurations used by developers ###
  #########################################

  # The default configuration will start 1 compute node, 1 meta node and 1 frontend.
  default:
    # If you want to use the local s3 storage, enable the following line
    # - use: minio

    # If you want to use aws-s3, configure AK and SK in env var and enable the following lines:
    # - use: aws-s3
    #   bucket: test-bucket
    # If you want to use other s3 compatible object store, open this flag:
    #   s3-compatible: false

    # if you want to enable etcd backend, uncomment the following lines.
    # - use: etcd
    #   unsafe-no-fsync: true

    - use: meta-node
      unsafe-disable-recovery: true
    - use: compute-node
    - use: frontend

    # If you want to enable compactor, uncomment the following line, and enable either minio or aws-s3 as well.
    # - use: compactor

    # If you want to enable metrics, uncomment those two lines.
    # - use: prometheus
    # - use: grafana

    # If you want to enable tracing, uncomment the following line.
    # - use: jaeger

    # If you want to create source from Kafka, uncomment the following lines
    # Note that kafka depends on zookeeper, so zookeeper must be started beforehand.
    # - use: zookeeper
    #   persist-data: true
    # - use: kafka
    #   persist-data: true

  default-v6:
    - use: meta-node
      address: "[::1]"
      listen-address: "[::]"
    - use: compute-node
      address: "[::1]"
      listen-address: "[::]"
    - use: frontend
      address: "[::1]"
      listen-address: "[::]"

  # The minimum config to use with risectl.
  for-ctl:
    - use: minio
    - use: meta-node
      unsafe-disable-recovery: true
    - use: compute-node
    - use: frontend
    - use: compactor

  # `dev-compute-node` have the same settings as default except the the compute node will be started by user.
  dev-compute-node:
    - use: meta-node
    - use: compute-node
      user-managed: true
    - use: frontend

  dev-frontend:
    - use: meta-node
    - use: compute-node
    - use: frontend
      user-managed: true

  dev-meta:
    - use: meta-node
      user-managed: true
    - use: compute-node
    - use: frontend

  full:
    - use: minio
    - use: etcd
    - use: meta-node
    - use: compute-node
    - use: frontend
    - use: compactor
    - use: prometheus
    - use: grafana
    - use: zookeeper
      persist-data: true
    - use: kafka
      persist-data: true

  full-compaction-test:
    - use: aws-s3
      bucket: some-bucket # configure a valid bucket name
    - use: etcd
    - use: meta-node
      periodic-compaction-interval-sec: 99999
      enable-compaction-deterministic: true
    - use: compute-node
    - use: frontend
    - use: compactor
    - use: prometheus
    - use: grafana
    - use: zookeeper
      persist-data: true
    - use: kafka
      persist-data: true

  full-benchmark:
    - use: minio
    - use: etcd
    - use: meta-node
    - use: compute-node
    - use: frontend
    - use: compactor
    - use: prometheus
      remote-write: true
      remote-write-region: "ap-southeast-1"
      remote-write-url: "https://aps-workspaces.ap-southeast-1.amazonaws.com/workspaces/ws-f3841dad-6a5c-420f-8f62-8f66487f512a/api/v1/remote_write"
    - use: grafana
    - use: zookeeper
      persist-data: true
    - use: kafka
      persist-data: true

  3etcd-3meta-1cn-1fe:
    - use: etcd
      unsafe-no-fsync: true
      port: 2388
      peer-port: 2389
      exporter-port: 2379
    - use: etcd
      unsafe-no-fsync: true
      port: 12388
      peer-port: 12389
      exporter-port: 12379
    - use: etcd
      unsafe-no-fsync: true
      port: 22388
      peer-port: 22389
      exporter-port: 22379
    - use: meta-node
      unsafe-disable-recovery: true
      port: 5690
      dashboard-port: 5691
      exporter-port: 1250
    - use: meta-node
      unsafe-disable-recovery: true
      port: 15690
      dashboard-port: 15691
      exporter-port: 11250
    - use: meta-node
      unsafe-disable-recovery: true
      port: 25690
      dashboard-port: 25691
      exporter-port: 21250
    - use: compute-node
    - use: frontend

  ##########################################
  ### Configuration used for ./risedev p ###
  ##########################################

  playground:
    - use: meta-node
      unsafe-disable-recovery: true
      max-idle-secs-to-exit: 1800
    - use: compute-node
    - use: frontend

  playground-test:
    - use: meta-node
      unsafe-disable-recovery: true
      max-idle-secs-to-exit: 1800
      vacuum-interval-sec: 1
      collect-gc-watermark-spin-interval-sec: 1
      min-sst-retention-time-sec: 0
    - use: compute-node
    - use: frontend
    - use: compactor

  playground-3cn:
    - use: meta-node
      unsafe-disable-recovery: true
      max-idle-secs-to-exit: 1800
    - use: compute-node
      port: 5687
      exporter-port: 1222
    - use: compute-node
      port: 5688
      exporter-port: 1223
    - use: compute-node
      port: 5689
      exporter-port: 1224
    - use: frontend
    - use: compactor

  docker-playground:
    - use: meta-node
      unsafe-disable-recovery: true
      listen-address: "0.0.0.0"
      address: "127.0.0.1"
    - use: compute-node
      listen-address: "0.0.0.0"
      address: "127.0.0.1"
    - use: frontend
      listen-address: "0.0.0.0"
      address: "127.0.0.1"

  # The profile for the online playground
  online-docker-playground:
    - use: meta-node
      unsafe-disable-recovery: true
      max-idle-secs-to-exit: 1800
      listen-address: "0.0.0.0"
      address: "127.0.0.1"
    - use: compute-node
      listen-address: "0.0.0.0"
      address: "127.0.0.1"
    - use: frontend
      listen-address: "0.0.0.0"
      address: "127.0.0.1"

  ######################################
  ### Configurations used in Compose ###
  ######################################

  compose:
    - use: minio
      id: minio-0
      address: ${id}
      listen-address: "0.0.0.0"
      console-address: "0.0.0.0"

    - use: meta-node
      # Id must starts with `meta-node`, therefore to be picked up by other
      # components.
      id: meta-node-0

      # Advertise address can be `id`, so as to use docker's DNS. If running
      # in host network mode, we should use IP directly in this field.
      address: ${id}

      listen-address: "0.0.0.0"

    - use: compute-node
      id: compute-node-0
      listen-address: "0.0.0.0"
      address: ${id}

    - use: frontend
      id: frontend-node-0
      listen-address: "0.0.0.0"
      address: ${id}

    - use: compactor
      id: compactor-0
      listen-address: "0.0.0.0"
      address: ${id}

    - use: redpanda

    - use: prometheus
      id: prometheus-0
      listen-address: "0.0.0.0"
      address: ${id}

    - use: grafana
      listen-address: "0.0.0.0"
      address: ${id}
      id: grafana-0

    - use: etcd
      listen-address: "0.0.0.0"
      address: ${id}
      id: etcd-0

  # special config for deployment, see related PR for more information
  compose-3node-deploy:
    # - use: minio
    #   id: minio-0
    #   address: ${dns-host:rw-source-0}
    #   listen-address: "0.0.0.0"
    #   console-address: "0.0.0.0"

    - use: aws-s3
      bucket: ${terraform:s3-bucket}

    # Not enabled by default as all previous benchmarks are not done with etcd.
    # Also we currently only support node-level docker volume tear down.
    # - use: etcd
    #   listen-address: "0.0.0.0"
    #   address: ${dns-host:rw-meta-0}
    #   id: etcd-0

    - use: meta-node
      # Id must starts with `meta-node`, therefore to be picked up by other
      # components.
      id: meta-node-0

      # Advertise address can be `id`, so as to use docker's DNS. If running
      # in host network mode, we should use IP directly in this field.
      address: ${dns-host:rw-meta-0}
      listen-address: "0.0.0.0"

    - use: compute-node
      id: compute-node-0
      listen-address: "0.0.0.0"
      address: ${dns-host:rw-compute-0}
      async-stack-trace: verbose
      enable-tiered-cache: true

    - use: compute-node
      id: compute-node-1
      listen-address: "0.0.0.0"
      address: ${dns-host:rw-compute-1}
      async-stack-trace: verbose
      enable-tiered-cache: true

    - use: compute-node
      id: compute-node-2
      listen-address: "0.0.0.0"
      address: ${dns-host:rw-compute-2}
      async-stack-trace: verbose
      enable-tiered-cache: true

    - use: frontend
      id: frontend-node-0
      listen-address: "0.0.0.0"
      address: ${dns-host:rw-meta-0}

    - use: compactor
      id: compactor-0
      listen-address: "0.0.0.0"
      address: ${dns-host:rw-source-0}
      max-concurrent-task-number: 15
      compaction-worker-threads-number: 15

    - use: redpanda
      address: ${dns-host:rw-source-0}

    - use: prometheus
      id: prometheus-0
      listen-address: "0.0.0.0"
      address: ${dns-host:rw-meta-0}

    - use: grafana
      listen-address: "0.0.0.0"
      address: ${dns-host:rw-meta-0}
      id: grafana-0

  #################################
  ### Configurations used on CI ###
  #################################

  ci-1cn-1fe:
    - use: minio
    - use: etcd
      unsafe-no-fsync: true
    - use: meta-node
      unsafe-disable-recovery: true
      enable-committed-sst-sanity-check: true
    - use: compute-node
      enable-tiered-cache: true
    - use: frontend
    - use: compactor

  ci-3cn-1fe:
    - use: minio
    - use: etcd
      unsafe-no-fsync: true
    - use: meta-node
      unsafe-disable-recovery: true
      enable-committed-sst-sanity-check: true
    - use: compute-node
      port: 5687
      exporter-port: 1222
      enable-tiered-cache: true
    - use: compute-node
      port: 5688
      exporter-port: 1223
      enable-tiered-cache: true
    - use: compute-node
      port: 5689
      exporter-port: 1224
      enable-tiered-cache: true
    - use: frontend
    - use: compactor

  ci-3cn-3fe:
    - use: minio
    - use: etcd
      unsafe-no-fsync: true
    - use: meta-node
      unsafe-disable-recovery: true
      enable-committed-sst-sanity-check: true
    - use: compute-node
      port: 5687
      exporter-port: 1222
      enable-tiered-cache: true
    - use: compute-node
      port: 5688
      exporter-port: 1223
      enable-tiered-cache: true
    - use: compute-node
      port: 5689
      exporter-port: 1224
      enable-tiered-cache: true
    - use: frontend
      port: 4565
      exporter-port: 2222
      health-check-port: 6786
    - use: frontend
      port: 4566
      exporter-port: 2223
      health-check-port: 6787
    - use: frontend
      port: 4567
      exporter-port: 2224
      health-check-port: 6788
    - use: compactor

  ci-3cn-3fe-in-memory:
    - use: etcd
      unsafe-no-fsync: true
    - use: meta-node
      unsafe-disable-recovery: true
    - use: compute-node
      port: 5687
      exporter-port: 1222
      enable-in-memory-kv-state-backend: true
    - use: compute-node
      port: 5688
      exporter-port: 1223
      enable-in-memory-kv-state-backend: true
    - use: compute-node
      port: 5689
      exporter-port: 1224
      enable-in-memory-kv-state-backend: true
    - use: frontend
      port: 4565
      exporter-port: 2222
      health-check-port: 6786
    - use: frontend
      port: 4566
      exporter-port: 2223
      health-check-port: 6787
    - use: frontend
      port: 4567
      exporter-port: 2224
      health-check-port: 6788

  ci-kafka:
    - use: minio
    - use: etcd
      unsafe-no-fsync: true
    - use: meta-node
      unsafe-disable-recovery: true
    - use: compute-node
      enable-tiered-cache: true
    - use: frontend
    - use: compactor
    - use: zookeeper
      persist-data: true
    - use: kafka
      persist-data: true

  ci-redis:
    - use: minio
    - use: etcd
      unsafe-no-fsync: true
    - use: meta-node
      unsafe-disable-recovery: true
    - use: compute-node
      enable-tiered-cache: true
    - use: frontend
    - use: compactor
    - use: redis

  ci-compaction-test:
    - use: minio
    - use: etcd
      unsafe-no-fsync: true
    - use: meta-node
      unsafe-disable-recovery: true
      periodic-compaction-interval-sec: 99999
      enable-compaction-deterministic: true
    - use: compute-node
      enable-tiered-cache: true
    - use: frontend
    - use: compactor

compose:
  risingwave: "ghcr.io/risingwavelabs/risingwave:latest"
  prometheus: "prom/prometheus:latest"
  minio: "quay.io/minio/minio:latest"
  redpanda: "docker.vectorized.io/vectorized/redpanda:latest"
  grafana: "grafana/grafana-oss:latest"
  etcd: "quay.io/coreos/etcd:latest"

# The `use` field specified in the above `risedev` section will refer to the templates below.
template:
  minio:
    # Advertise address of MinIO s3 endpoint
    address: "127.0.0.1"

    # Advertise port of MinIO s3 endpoint
    port: 9301

    # Listen address of MinIO endpoint
    listen-address: ${address}

    # Console address of MinIO s3 endpoint
    console-address: "127.0.0.1"

    # Console port of MinIO s3 endpoint
    console-port: 9400

    # Root username (can be used to login to MinIO console)
    root-user: hummockadmin

    # Root password (can be used to login to MinIO console)
    root-password: hummockadmin

    # Bucket name to store hummock information
    hummock-bucket: hummock001

    # Id of this instance
    id: minio

    # Prometheus nodes used by this MinIO
    provide-prometheus: "prometheus*"

  etcd:
    # Id of this instance
    id: etcd-${port}

    # Advertise address of the single-node etcd.
    address: "127.0.0.1"

    # Listen port of the single-node etcd.
    port: 2388

    # Listen address
    listen-address: ${address}

    # Peer listen port of the single-node etcd.
    peer-port: 2389

    # Prometheus exporter listen port
    exporter-port: 2379

    # Whether to enable fsync (NEVER SET TO TRUE IN PRODUCTION ENVIRONMENT!)
    unsafe-no-fsync: false

    # Other etcd nodes
    provide-etcd: "etcd*"

  compute-node:
    # Compute-node advertise address
    address: "127.0.0.1"

    # Listen address
    listen-address: ${address}

    # Compute-node listen port
    port: 5688

    # Prometheus exporter listen port
    exporter-port: 1222

    # Id of this instance
    id: compute-node-${port}

    # Whether to enable async stack trace for this compute node, `off`, `on`, or `verbose`.
    # Considering the performance, `verbose` mode only effect under `release` profile with `debug_assertions` off.
    async-stack-trace: on

    # If `enable-tiered-cache` is true, hummock will use data directory as file cache.
    enable-tiered-cache: false

    # RPC endpoint for source connector node
    connector-source-endpoint: "127.0.0.1:60061"

    # Minio instances used by this compute node
    provide-minio: "minio*"

    # AWS s3 bucket used by this compute node
    provide-aws-s3: "aws-s3*"

    # Meta-nodes used by this compute node
    provide-meta-node: "meta-node*"

    # Jaeger used by this compute node
    provide-jaeger: "jaeger*"

    # Sanity check: should use shared storage if there're multiple compute nodes
    provide-compute-node: "compute-node*"

    # Sanity check: should start at lease one compactor if using shared object store
    provide-compactor: "compactor*"

    # If `user-managed` is true, this service will be started by user with the above config
    user-managed: false

    # Whether to enable in-memory pure KV state backend
    enable-in-memory-kv-state-backend: false

  meta-node:
    # Meta-node advertise address
    address: "127.0.0.1"

    # Meta-node listen port
    port: 5690

    # Listen address
    listen-address: ${address}

    # Dashboard listen port
    dashboard-port: 5691

    # Prometheus exporter listen port
    exporter-port: 1250

    # Id of this instance
    id: meta-node-${port}

    # If `user-managed` is true, this service will be started by user with the above config
    user-managed: false

    # Etcd backend config
    provide-etcd-backend: "etcd*"

    # Maximum allowed heartbeat interval in seconds.
    max-heartbeat-interval-secs: 600

    # Whether to disable recovery mode
    unsafe-disable-recovery: false

    # Interval of GC stale metadata and SST
    vacuum-interval-sec: 30

    # Interval of periodical compaction
    periodic-compaction-interval-sec: 60

    # Whether run in compaction deterministic test mode
    enable-compaction-deterministic: false

    # The spin interval when collecting global GC watermark in hummock
    collect-gc-watermark-spin-interval-sec: 5

    # Threshold used by worker node to filter out new SSTs when scanning object store, during full SST GC.
    # 7 days by default.
    min-sst-retention-time-sec: 604800

    # Enable sanity check when SSTs are committed. Disabled by default.
    enable-committed-sst-sanity-check: false

  prometheus:
    # Advertise address of Prometheus
    address: "127.0.0.1"

    # Listen port of Prometheus
    port: 9500

    # Listen address
    listen-address: ${address}

    # Id of this instance
    id: prometheus

    # If `remote_write` is true, this Prometheus instance will push metrics to remote instance
    remote-write: false

    # AWS region of remote write
    remote-write-region: ""

    # Remote write url of this instance
    remote-write-url: ""

    # Compute-nodes used by this Prometheus instance
    provide-compute-node: "compute-node*"

    # Meta-nodes used by this Prometheus instance
    provide-meta-node: "meta-node*"

    # Minio instances used by this Prometheus instance
    provide-minio: "minio*"

    # Compactors used by this Prometheus instance
    provide-compactor: "compactor*"

    # Etcd used by this Prometheus instance
    provide-etcd: "etcd*"

    # Redpanda used by this Prometheus instance
    provide-redpanda: "redpanda*"

    # Frontend used by this Prometheus instance
    provide-frontend: "frontend*"

  frontend:
    # Advertise address of frontend
    address: "127.0.0.1"

    # Listen port of frontend
    port: 4566

    # Listen address
    listen-address: ${address}

    # Prometheus exporter listen port
    exporter-port: 2222

    # Health check listen port
    health-check-port: 6786

    # Id of this instance
    id: frontend-${port}

    # Meta-nodes used by this frontend instance
    provide-meta-node: "meta-node*"

    # If `user-managed` is true, this service will be started by user with the above config
    user-managed: false

  compactor:
    # Compactor advertise address
    address: "127.0.0.1"

    # Compactor listen port
    port: 6660

    # Listen address
    listen-address: ${address}

    # Prometheus exporter listen port
    exporter-port: 1260

    # Id of this instance
    id: compactor-${port}

    # Minio instances used by this compute node
    provide-minio: "minio*"

    # AWS s3 bucket used by this compute node
    provide-aws-s3: "aws-s3*"

    # Meta-nodes used by this compute node
    provide-meta-node: "meta-node*"

    # If `user-managed` is true, this service will be started by user with the above config
    user-managed: false

    # Hint used by meta node
    max-concurrent-task-number: 15

  grafana:
    # Listen address of Grafana
    listen-address: ${address}

    # Advertise address of Grafana
    address: "127.0.0.1"

    # Listen port of Grafana
    port: 3001

    # Id of this instance
    id: grafana

    # Prometheus used by this Grafana instance
    provide-prometheus: "prometheus*"

  jaeger:
    # Id of this instance
    id: jaeger

    # Dashboard listen address of Jaeger
    dashboard-address: "127.0.0.1"

    # Dashboard listen port of Jaeger
    dashboard-port: 16680

    # Jaeger has a lot of ports open, and we don't want to make this config more complex.
    # So we keep the default value of jaeger instead of making it part of RiseDev config.

  # aws-s3 is a placeholder service to provide configurations
  aws-s3:
    # Id to be picked-up by services
    id: aws-s3

    # The bucket to be used for AWS S3
    bucket: test-bucket

    # access key, secret key and region should be set in aws config (either by env var or .aws/config)

    # used to support other s3 compatible object store.
    s3-compatible: false

  # Apache Kafka service
  kafka:
    # Id to be picked-up by services
    id: kafka-${port}

    # Advertise address of Kafka
    address: "127.0.0.1"

    # Listen port of Kafka
    port: 29092

    # Listen address
    listen-address: ${address}

    # ZooKeeper used by this Kafka instance
    provide-zookeeper: "zookeeper*"

    # If set to true, data will be persisted at data/{id}.
    persist-data: true

    # Kafka broker id. If there are multiple instances of Kafka, we will need to set.
    broker-id: 0

  # Apache ZooKeeper service
  zookeeper:
    # Id to be picked-up by services
    id: zookeeper-${port}

    # Advertise address of ZooKeeper
    address: "127.0.0.1"

    # Listen address
    listen-address: ${address}

    # Listen port of ZooKeeper
    port: 2181

    # If set to true, data will be persisted at data/{id}.
    persist-data: true

  # Only supported in RiseDev compose
  redpanda:
    # Id to be picked-up by services
    id: redpanda

    # Port used inside docker-compose cluster (e.g. create MV)
    internal-port: 29092

    # Port used on host (e.g. import data, connecting using kafkacat)
    outside-port: 9092

    # Connect address
    address: ${id}

    # Number of CPUs to use
    cpus: 8

    # Memory limit for Redpanda
    memory: 16G

  # redis service
  redis:
    # Id to be picked-up by services
    id: redis

    # listen port of redis
    port: 6379

    # address of redis
    address: "127.0.0.1"