connectivity.conf

ditto {
  service-name = "connectivity"

  mongodb {
    database = "connectivity"
    database = ${?MONGO_DB_DATABASE}
  }

  persistence.operations.delay-after-persistence-actor-shutdown = 5s
  persistence.operations.delay-after-persistence-actor-shutdown = ${?DELAY_AFTER_PERSISTENCE_ACTOR_SHUTDOWN}

  connectivity {
    connection {
      # A comma separated string of hostnames to which http requests will allowed. This overrides the blocked
      # hostnames i.e if a host is blocked *and* allowed, it will be allowed.
      allowed-hostnames = ""
      #allowed-hostnames = "localhost"
      allowed-hostnames = ${?CONNECTIVITY_CONNECTION_ALLOWED_HOSTNAMES}

      # A comma separated string of blocked hostnames to which no http requests will be sent out.
      # Set it to an empty string to permit outgoing HTTP connections against private IP addresses.
      blocked-hostnames = ""
      #blocked-hostnames = "localhost"
      blocked-hostnames = ${?CONNECTIVITY_CONNECTION_BLOCKED_HOSTNAMES}

      # Number of sources per connection
      max-source-number = ${?CONNECTION_SOURCE_NUMBER}
      # Number of targets per connection
      max-target-number = ${?CONNECTION_TARGET_NUMBER}

      supervisor {
        exponential-backoff {
          min = 1s
          max = 120s
          random-factor = 1.0
          corrupted-receive-timeout = 600s
        }
      }

      snapshot {
        threshold = 10
        interval = 15m
      }

      activity-check {
        # the interval of how long to keep a closed, "inactive" Connection in memory
        inactive-interval = 45m
        inactive-interval = ${?CONNECTION_ACTIVITY_CHECK_INTERVAL} # may be overridden with this environment variable

        # the interval of how long to keep a deleted Connection in memory:
        deleted-interval = 5m
        deleted-interval = ${?CONNECTION_ACTIVITY_CHECK_DELETED_INTERVAL}
      }

      # how long for connection actor to wait for response from client actors
      # by default this value is very high because connection establishment can take very long and if we timeout too
      # early the connection is not subscribed for events properly
      # this timeout needs to be smaller then ditto.gateway.http.request-timeout in gateway.conf
      client-actor-ask-timeout = 55s
      client-actor-ask-timeout = ${?CONNECTIVITY_CLIENT_ACTOR_ASK_TIMEOUT}

      # Whether to start 1 client actor on each node or to start all client actors on the same node.
      # It is not possible to start multiple client actors on multiple nodes.
      all-client-actors-on-one-node = false
      all-client-actors-on-one-node = ${?CONNECTIVITY_ALL_CLIENT_ACTORS_ON_ONE_NODE}

      acknowledgement {
        # lifetime of ack forwarder. Must be bigger than the largest possible command timeout (60s)
        forwarder-fallback-timeout = 65s

        # lifetime of collector of source acknowledgements. usually terminates before this long when all acks arrive.
        collector-fallback-lifetime = 100s

        # how long to wait on the collector before acknowledging negatively on the transport layer.
        collector-fallback-ask-timeout = 120s

        # size budget for the payload of issued acknowledgements by a connection target.
        issued-max-bytes = 100000
        issued-max-bytes = ${?CONNECTIVITY_ACKNOWLEDGEMENT_ISSUED_MAX_BYTES}
      }

      # how often to attempt acknowledgement label declaration for as long as it is not successful
      ack-label-declare-interval = 10s
      ack-label-declare-interval = ${?CONNECTIVITY_ACK_LABEL_DECLARE_INTERVAL}

      priority-update-interval = 24h
      priority-update-interval = ${?CONNECTIVITY_PRIORITY_UPDATE_INTERVAL}

      amqp10 {
        consumer {
          # Whether rate limit according to throughput and acknowledgement is enabled.
          rate-limit-enabled = true
          rate-limit-enabled = ${?AMQP10_CONSUMER_RATE_LIMIT_ENABLED}

          # How many unacknowledged messages are allowed at any time, counting NAcked with redeliver=true
          # This limit couples latency with throughput (long lagency before ack -> lower throughput)
          # Should be above throttling.limit not to decrease throughput further.
          max-in-flight = 200
          max-in-flight = ${?AMQP10_CONSUMER_MAX_IN_FLIGHT}

          # When to forget an NAcked redelivery=true message -- those may be consumed by another consumer
          # Equal to the max time interval per redelivery per message during subscriber downtime
          redelivery-expectation-timeout = 120s
          redelivery-expectation-timeout = ${?AMQP10_CONSUMER_REDELIVERY_EXPECTATION_TIMEOUT}

          throttling {
            # Interval at which the consumer is throttled. Values smaller than 1s are treated as 1s.
            interval = 1s
            interval = ${?AMQP10_CONSUMER_THROTTLING_INTERVAL}

            # The maximum number of messages the consumer is allowed to receive within the configured
            # throttling interval e.g. 100 msgs/s.
            # Values smaller than 1 are treated as 1.
            limit = 100
            limit = ${?AMQP10_CONSUMER_THROTTLING_LIMIT}
          }
        }

        publisher {
          # If a message can't be published it is put in a queue. Further messages are dropped when the queue is full.
          max-queue-size = 100
          max-queue-size = ${?AMQP10_PUBLISHER_MAX_QUEUE_SIZE}

          # Messages to publish in parallel per AMQP-Publisher (one per connectivity client)
          # jms-connection-handling-dispatcher will be used.
          parallelism = 10
          parallelism = ${?AMQP10_PUBLISHER_PARALLELISM}
        }

        // How many producers to cache per client actor (in addition to static addresses).
        // If 0 or negative, no message can be sent to any reply-to address or addresses containing placeholders that
        // do not match any target address.
        producer-cache-size = 10
        producer-cache-size = ${?AMQP10_PRODUCER_CACHE_SIZE}

        global-connect-timeout = 15s
        global-connect-timeout = ${?AMQP10_GLOBAL_CONNECT_TIMEOUT}

        global-send-timeout = 60s
        global-send-timeout = ${?AMQP10_GLOBAL_SEND_TIMEOUT}

        global-request-timeout = 5s
        global-request-timeout = ${?AMQP10_GLOBAL_REQUEST_TIMEOUT}

        global-prefetch-policy-all-count = 10
        global-prefetch-policy-all-count = ${?GLOBAL_PREFETCH_POLICY_ALL_COUNT}

        # Configuration for backoff of the producers. This will be used when the connection to a target fails with
        # an error message of the JMS client (can happen when e.g. user is not authorized on a target). As the actor
        # will instantly try to re-establish the targets, this may lead to a lot of error logs. Thats why this backoff
        # configuration is used. On the first error the actor will wait min-timeout before retrying to establish the target,
        # afterwards it will increase its waiting time step by step until max-timeout.
        backoff.timeout {
          # initial / minimum timeout when backing off
          min-timeout = 1s
          min-timeout = ${?AMQP10_BACK_OFF_MIN_TIMEOUT}
          # maximum backoff timeout
          max-timeout = 10m
          max-timeout = ${?AMQP10_BACK_OFF_MAX_TIMEOUT}
        }
      }

      amqp091 {
        publisher {
          # Lifetime of an entry in the cache 'outstandingAcks'.
          # It is an upper bound for the timeout of any command requesting acknowledgements from this publisher.
          # Ideally between the maximum timeout (60s) and the acknowledgement forwarder lifetime (100s).
          # No other publisher actor requires a cache TTL config because their clients take care of message ID tracking.
          pending-ack-ttl = 1m
        }
      }

      mqtt {
        # maximum mumber of MQTT messages to buffer in a source (presumably for at-least-once and exactly-once delivery)
        source-buffer-size = 8
        source-buffer-size = ${?CONNECTIVITY_MQTT_SOURCE_BUFFER_SIZE}

        # Indicates whether the client should reconnect to enforce a redelivery for a failed acknowledgement.
        reconnect-for-redelivery = false
        reconnect-for-redelivery = ${?CONNECTIVITY_MQTT_RECONNECT_FOR_REDELIVERY}

        #The amount of time that a reconnect will be delayed after a failed acknowledgement
        reconnect-for-redelivery-delay = 2s
        reconnect-for-redelivery-delay = ${?CONNECTIVITY_MQTT_RECONNECT_FOR_REDELIVERY_DELAY}

        # Indicates whether a separate client should be used for publishing. This could be useful when
        # reconnect-for-redelivery is set to true to avoid that the publisher has downtimes.
        separate-publisher-client = false
        separate-publisher-client = ${?CONNECTIVITY_MQTT_SEPARATE_PUBLISHER_CLIENT}
      }

      http-push {
        # How many messages to buffer in the publisher actor before dropping them. Each takes up to 100 KB heap space.
        max-queue-size = 100
        max-queue-size = ${?CONNECTIVITY_HTTP_PUSH_MAX_QUEUE_SIZE}

        request-timeout = 60s
        request-timeout = ${?CONNECTIVITY_HTTP_PUSH_REQUEST_TIMEOUT}

        # proxy config
        proxy {
          enabled = false
          enabled = ${?CONNECTIVITY_HTTP_PROXY_ENABLED}

          hostname = ${?CONNECTIVITY_HTTP_PROXY_HOST}
          port = ${?CONNECTIVITY_HTTP_PROXY_PORT}
          username = ${?CONNECTIVITY_HTTP_PROXY_USERNAME}
          password = ${?CONNECTIVITY_HTTP_PROXY_PASSWORD}
        }
      }

      kafka {
        consumer {
          # Properties defined by org.apache.kafka.clients.consumer.ConsumerConfig
          # can be defined in this configuration section.
          kafka-clients {
            enable.auto.commit = true

            # Disable automatic retry
            retries = 0

            # Max wait before 1 send attempt fails
            request.timeout.ms = 10000 # default: 40000

            # Max wait before send fails
            delivery.timeout.ms = 10000 # default: 120000

            # Max wait for downed broker before connection fails
            max.block.ms = 10000 # default: 60000

            # Default serializer / deserializer class for key that implements the org.apache.kafka.common.serialization.Serde interface.
            default.key.serde = "org.apache.kafka.common.serialization.Serdes$StringSerde" # default:	org.apache.kafka.common.serialization.Serdes$ByteArraySerde
            default.value.serde = "org.apache.kafka.common.serialization.Serdes$StringSerde" # default:	org.apache.kafka.common.serialization.Serdes$ByteArraySerde
          }
        }

        # Properties defined by org.apache.kafka.clients.producer.ProducerConfig
        # can be defined in this configuration section.
        producer {
          kafka-clients {
            # Close idle connections after the number of milliseconds specified by this config.
            # When a message should be produced after a connection was closed because of this timeout, the client
            # simply opens the connection again, so for producers there is no need to increase this value:
            connections.max.idle.ms = 540000 # default: 540000 (9min)

            # The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
            # If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
            reconnect.backoff.max.ms = 10000 # default: 1000
            # The base amount of time to wait before attempting to reconnect to a given host.
            # This avoids repeatedly connecting to a host in a tight loop.
            reconnect.backoff.ms = 500 # default: 50

            # Request acknowledgement
            acks = "1"

            # Disable automatic retry
            retries = 0

            # Max wait before 1 send attempt fails
            request.timeout.ms = 10000 # default: 40000

            # Max wait before send fails
            delivery.timeout.ms = 10000 # default: 120000

            # Max wait for downed broker before connection fails
            max.block.ms = 10000 # default: 60000
          }
        }
      }
    }

    mapping {

      # the buffer size used for the queue in the message mapping processor actor
      buffer-size = 100
      buffer-size = ${?CONNECTIVITY_SIGNAL_ENRICHMENT_BUFFER_SIZE}

      # parallelism to use for signal enriching a single message in the message mapping processor actor
      # when configured too low, throughput of signal enrichment will be low
      parallelism = 100
      parallelism = ${?CONNECTIVITY_SIGNAL_ENRICHMENT_PARALLELISM}

      # maximum parallelism for mapping incoming and outgoing messages for a connection. This setting limits the
      # poolSize that users can configure on their connections.
      max-pool-size = 5
      max-pool-size = ${?CONNECTIVITY_MESSAGE_MAPPING_MAX_POOL_SIZE}

      javascript {
        # the maximum script size in bytes of a mapping script to run
        # prevents loading big JS dependencies into the script (e.g. jQuery which has ~250kB)
        maxScriptSizeBytes = 50000 # 50kB
        # the maximum execution time of a mapping script to run
        # prevents endless loops and too complex scripts
        maxScriptExecutionTime = 500ms
        # the maximum call stack depth in the mapping script
        # prevents recursions or other too complex computation
        maxScriptStackDepth = 25
      }

      mapper-limits {
        # maximum number of mappers defined in one source
        max-source-mappers = 10
        # maximum number of messages invoked by a mapper
        # defined in source
        max-mapped-inbound-messages = 10
        # maximum number of mappers defined in one source
        max-target-mappers = 10
        # maximum number of messages invoked by a mapper
        # defined in target
        max-mapped-outbound-messages = 10
      }
    }

    signal-enrichment {
      // Beware: Despite similarities with gateway signal-enrichment providers,
      // this class is different and not compatible with them.
      provider = "org.eclipse.ditto.connectivity.service.mapping.ConnectivityCachingSignalEnrichmentProvider"
      provider = ${?CONNECTIVITY_SIGNAL_ENRICHMENT_PROVIDER}

      provider-config {
        # timeout for all facades
        ask-timeout = 10s
        ask-timeout = ${?CONNECTIVITY_SIGNAL_ENRICHMENT_ASK_TIMEOUT}

        cache {
          # how many things to cache in total on a single cluster node
          maximum-size = 20000
          maximum-size = ${?CONNECTIVITY_SIGNAL_ENRICHMENT_CACHE_MAXIMUM_SIZE}

          # maximum duration of inconsistency after e.g. a policy update
          expire-after-create = 2m
          expire-after-create = ${?CONNECTIVITY_SIGNAL_ENRICHMENT_CACHE_EXPIRE_AFTER_CREATE}
        }
      }
    }

    persistence-ping {
      # journal tag to query to find our which connectionPersistenceActors to ping (reconnect)
      journal-tag = ""
      journal-tag = ${?RECONNECT_JOURNAL_TAG}
      # initial delay for pinging / reconnecting the connections after the PersistencePingActor has been started.
      initial-delay = 0s
      initial-delay = ${?RECONNECT_INITIAL_DELAY}
      # interval for trying to reconnect all started connections.
      interval = 10m
      interval = ${?RECONNECT_INTERVAL}
      # how many events to read in one query
      read-journal-batch-size = 500
      read-journal-batch-size = ${?RECONNECT_READ_JOURNAL_BATCH_SIZE}

      # one of:
      #  TAGS  Elements will be ordered by their tags.
      #  ID    Elements will be ordered by their document ID.
      streaming-order = TAGS
      streaming-order = ${?PERSISTENCE_PING_STREAMING_ORDER}

      # used to throttle pinging of connections, so that not all connections are recovered at the same time
      rate {
        frequency = 1s
        frequency = ${?RECONNECT_RATE_FREQUENCY}
        entities = 1
        entities = ${?RECONNECT_RATE_ENTITIES}
      }
    }

    connection-ids-retrieval {
      # how many events to read in one query
      read-journal-batch-size = 500
      read-journal-batch-size = ${?CONNECTION_IDS_RETRIEVAL_READ_JOURNAL_BATCH_SIZE}
      # how many snapshots to read in one query
      read-snapshot-batch-size = 50
      read-snapshot-batch-size = ${?CONNECTION_IDS_RETRIEVAL_READ_SNAPSHOT_BATCH_SIZE}
    }

    client {
      # Initial timeout when connecting to a remote system. If the connection could not be established after this time, the
      # service will try to reconnect. If a failure happened during connecting, then the service will wait for at least
      # this time until it will try to reconnect. The max timeout is defined in connecting-max-timeout.
      connecting-min-timeout = 60s
      connecting-min-timeout = ${?CONNECTIVITY_CLIENT_CONNECTING_MIN_TIMEOUT}
      # Max timeout (until reconnecting) when connecting to a remote system.
      # Should be greater than connecting-min-timeout.
      connecting-max-timeout = 60m
      connecting-max-timeout = ${?CONNECTIVITY_CLIENT_CONNECTING_MAX_TIMEOUT}
      # How many times we will try to reconnect when connecting to a remote system.
      # max time ~= connecting-max-tries * connecting-max-timeout = 50 * 60m = 50h
      connecting-max-tries = 50
      connecting-max-tries = ${?CONNECTIVITY_CLIENT_CONNECTING_MAX_TRIES}
      # Max timeout until waiting for search commands.
      subscription-manager-timeout = 60s
      subscription-manager-timeout = ${?CONNECTIVITY_CLIENT_SUBSCRIPTION_MANAGER_TIMEOUT}
      # how long the service will wait for a successful connection when testing a new connection. If no response is
      # received after this duration, the test will be assumed a failure.
      testing-timeout = 10s
      testing-timeout = ${?CONNECTIVITY_CLIENT_TESTING_TIMEOUT}
      # Min backoff after connection failure.
      min-backoff = 5s
      min-backoff = ${?CONNECTIVITY_CLIENT_MIN_BACKOFF}
      # Max backoff after connection failure.
      max-backoff = 60m
      max-backoff = ${?CONNECTIVITY_CLIENT_MAX_BACKOFF}
      # How often to refresh the cache of other client actors if client count > 1
      client-actor-refs-notification-delay = 5m
    }

    monitoring {
      logger {
        successCapacity = 10
        successCapacity = ${?CONNECTIVITY_LOGGER_SUCCESS_CAPACITY}
        failureCapacity = 10
        failureCapacity = ${?CONNECTIVITY_LOGGER_FAILURE_CAPACITY}
        maxLogSizeBytes = 250000
        maxLogSizeBytes = ${?CONNECTIVITY_LOGGER_MAX_LOG_SIZE_BYTES}
        logDuration = 1h
        logDuration = ${?CONNECTIVITY_LOGGER_LOG_DURATION}
        loggingActiveCheckInterval = 5m
        loggingActiveCheckInterval = ${?CONNECTIVITY_LOGGER_ACTIVE_CHECK_INTERVAL}
      }
      counter {}
    }

    default-config-provider = true
    default-config-provider = ${?CONNECTIVITY_DEFAULT_CONFIG_PROVIDER}

    tunnel {
      # 0 workers mean the sshd client defaults which are: `Runtime.getRuntime().availableProcessors() + 1`
      workers = 0
      heartbeat-interval = 30s
      # by default, keep the tunnel open forever:
      idle-timeout = 0s
      socket-keepalive = true
    }
  }
}

akka {
  http {
    client {
      user-agent-header = eclipse-ditto/${ditto.version}

      # Setting this to less than Inf may generate occasional failures for not very active HTTP-PUSH connections.
      idle-timeout = Inf
      idle-timeout = ${?AKKA_HTTP_CLIENT_IDLE_TIMEOUT}
    }
    # The connection-pools used for http-push connections
    host-connection-pool {
      # This timeout configures a maximum amount of time, while the connection can be kept open.
      # Helps the process of rebalancing between service instances when reaching the target through a load-balancer
      max-connection-lifetime = Inf
      max-connection-lifetime = ${?AKKA_HTTP_HOSTPOOL_MAX_CONNECTION_LIFETIME}
    }
  }
  cluster {
    split-brain-resolver.lease-majority.role = ${ditto.service-name}
    split-brain-resolver.keep-majority.role = ${ditto.service-name}
    split-brain-resolver.keep-oldest.role = ${ditto.service-name}
    split-brain-resolver.static-quorum.role = ${ditto.service-name}

    sharding {
      role = ${ditto.service-name}

      # When this is set to 'on' the active entity actors will automatically be restarted
      # upon Shard restart. i.e. if the Shard is started on a different ShardRegion
      # due to rebalance or crash.
      remember-entities = on
      # increase default timeout because otherwise shards are beeing stopped and restarted if the ShardCoordinator
      # could not load initial entity ids from the "remember entities store" within this timeout. This is required
      # since remember-entities is set to "on".
      waiting-for-state-timeout = 10s
      # don't passivate shards by default as Ditto AbstractShardedPersistenceActor decides that on its own - default is 120s:
      passivate-idle-entity-after = "off"
    }

    roles = [
      "connectivity"
    ]
  }

  persistence {
    journal.auto-start-journals = [
      "akka-contrib-mongodb-persistence-connection-journal"
    ]
    snapshot-store.auto-start-snapshot-stores = [
      "akka-contrib-mongodb-persistence-connection-snapshots"
    ]
  }
}

include "ditto-protocol-subscriber.conf"

akka-contrib-mongodb-persistence-connection-journal {
  class = "akka.contrib.persistence.mongodb.MongoJournal"
  plugin-dispatcher = "connection-persistence-dispatcher"

  overrides {
    journal-collection = "connection_journal"
    journal-index = "connection_journal_index"

    realtime-collection = "connection_realtime"
    metadata-collection = "connection_metadata"
  }

  event-adapters {
    mongodbobject = "org.eclipse.ditto.connectivity.service.messaging.persistence.ConnectivityMongoEventAdapter"
  }

  event-adapter-bindings {
    "org.eclipse.ditto.base.model.signals.events.Event" = mongodbobject
    "org.bson.BsonValue" = mongodbobject
  }
}

akka-contrib-mongodb-persistence-connection-snapshots {
  class = "akka.contrib.persistence.mongodb.MongoSnapshots"
  plugin-dispatcher = "connection-persistence-dispatcher"
  overrides {
    snaps-collection = "connection_snaps"
    snaps-index = "connection_snaps_index"
  }
}

connection-persistence-dispatcher {
  type = Dispatcher
  executor = "fork-join-executor"
  fork-join-executor {
    parallelism-min = 4
    parallelism-factor = 3.0
    parallelism-max = 32
    parallelism-max = ${?DEFAULT_DISPATCHER_PARALLELISM_MAX}
  }
  # Throughput defines the maximum number of messages to be
  # processed per actor before the thread jumps to the next actor.
  # Set to 1 for as fair as possible.
  throughput = 2
}

rabbit-stats-bounded-mailbox {
  mailbox-type = "akka.dispatch.BoundedMailbox"
  mailbox-capacity = 10
  mailbox-push-timeout-time = 0s
}

message-mapping-processor-dispatcher {
  type = Dispatcher
  executor = "fork-join-executor"
  fork-join-executor {
    # Min number of threads to cap factor-based parallelism number to
    parallelism-min = 4
    # Parallelism (threads) ... ceil(available processors * factor)
    parallelism-factor = 3.0
    # Max number of threads to cap factor-based parallelism number to
    parallelism-max = 64
  }
  throughput = 5
}

jms-connection-handling-dispatcher {
  # one thread per actor because the actor blocks.
  type = PinnedDispatcher
  executor = "thread-pool-executor"
}

signal-enrichment-cache-dispatcher {
  type = "Dispatcher"
  executor = "thread-pool-executor"
  thread-pool-executor {
    keep-alive-time = 60s
    fixed-pool-size = off
    max-pool-size-max = 256
    max-pool-size-max = ${?CACHE_DISPATCHER_POOL_SIZE_MAX}
    max-pool-size-max = ${?SIGNAL_ENRICHMENT_CACHE_DISPATCHER_POOL_SIZE_MAX}
  }
}

http-push-connection-dispatcher {
  type = "Dispatcher"
  executor = "thread-pool-executor"
  thread-pool-executor {
    keep-alive-time = 60s
    fixed-pool-size = off
    max-pool-size-max = 256
    max-pool-size-max = ${?HTTP_PUSH_CONNECTION_DISPATCHER_POOL_SIZE_MAX}
  }
}

include "connectivity-extension"