Skip to content

Commit

Permalink
replace datadog key with statsd key in the config
Browse files Browse the repository at this point in the history
  • Loading branch information
@macalimlim
macalimlim committed Apr 13, 2020
1 parent ef95aad commit e83a531
Show file tree
Hide file tree
Showing 9 changed files with 166 additions and 104 deletions.
8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,9 @@ All Ziggurat configs should be in your `clonfig` `config.edn` under the `:ziggur
:datadog {:host "localhost"
:port [8125 :int]
:enabled [false :bool]}
:statsd {:host "localhost"
:port [8125 :int]
:enabled [false :bool]}
:sentry {:enabled [false :bool]
:dsn "dummy"
:worker-count [5 :int]
Expand Down Expand Up @@ -331,6 +334,7 @@ All Ziggurat configs should be in your `clonfig` `config.edn` under the `:ziggur
* enable.idempotence - When set to 'true', the producer will ensure that exactly one copy of each message is written in the stream. If 'false', producer retries due to broker failures, etc., may write duplicates of the retried message in the stream.

* datadog - The statsd host and port that metrics should be sent to, although the key name is datadog, it supports statsd as well to send metrics.
* statsd - Same as datadog but with a more appropriate name, the :datadog key will be deprecated in the future.
* sentry - Whenever a :failure keyword is returned from the mapper-function or an exception is raised while executing the mapper-function, an event is sent to sentry. You can skip this flow by disabling it.
* rabbit-mq-connection - The details required to make a connection to rabbitmq. We use rabbitmq for the retry mechanism.
* rabbit-mq - The queues that are part of the retry mechanism
Expand Down Expand Up @@ -372,6 +376,10 @@ and different timeout values.
:enable [true :bool]}}}}}
```

## Deprecation Notice
- Please note that the :datadog key inside the config file will be removed (sometime in the future) in favor of :statsd. Both contents are thesame though, it's just the key name was changed.
The reason for this is to avoid confusion with our users. We would still have backward compatibility for the :datadog key.

## Contribution
- For dev setup and contributions please refer to CONTRIBUTING.md

Expand Down
65 changes: 34 additions & 31 deletions resources/config.test.ci.edn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
:env [:dev :keyword]
:nrepl-server {:port [7011 :int]}
:datadog {:host "localhost"
:port [8126 :int]
:enabled [false :bool]}
:statsd {:host "localhost"
:port [8125 :int]
:enabled [false :bool]}
:sentry {:enabled [false :bool]
Expand Down Expand Up @@ -29,34 +32,34 @@
:enabled [true :bool]}
:http-server {:port [8010 :int]
:thread-count [100 :int]}
:stream-router {:default {:application-id "test"
:bootstrap-servers "localhost:9092"
:stream-threads-count [1 :int]
:origin-topic "topic"
:upgrade-from "1.1"
:channels {:channel-1 {:worker-count [10 :int]
:retry {:type [:linear :keyword]
:count [5 :int]
:enabled [true :bool]}}}
:producer {:bootstrap-servers "localhost:9092"
:acks "all"
:retries-config 5
:max-in-flight-requests-per-connection 5
:enable-idempotence false
:value-serializer "org.apache.kafka.common.serialization.StringSerializer"
:key-serializer "org.apache.kafka.common.serialization.StringSerializer"}}
:using-string-serde {:application-id "test"
:bootstrap-servers "localhost:9092"
:stream-threads-count [1 :int]
:origin-topic "another-test-topic"
:default-key-serde "org.apache.kafka.common.serialization.Serdes$StringSerde"
:default-value-serde "org.apache.kafka.common.serialization.Serdes$StringSerde"
:key-deserializer-encoding "UTF8"
:value-deserializer-encoding "UTF8"
:deserializer-encoding "UTF8"
:upgrade-from "1.1"
:channels {:channel-1 {:worker-count [10 :int]
:retry {:count [5 :int]
:enabled [true :bool]}}}}}
:tracer {:enabled [true :bool]
:custom-provider ""}}}
:stream-router {:default {:application-id "test"
:bootstrap-servers "localhost:9092"
:stream-threads-count [1 :int]
:origin-topic "topic"
:upgrade-from "1.1"
:channels {:channel-1 {:worker-count [10 :int]
:retry {:type [:linear :keyword]
:count [5 :int]
:enabled [true :bool]}}}
:producer {:bootstrap-servers "localhost:9092"
:acks "all"
:retries-config 5
:max-in-flight-requests-per-connection 5
:enable-idempotence false
:value-serializer "org.apache.kafka.common.serialization.StringSerializer"
:key-serializer "org.apache.kafka.common.serialization.StringSerializer"}}
:using-string-serde {:application-id "test"
:bootstrap-servers "localhost:9092"
:stream-threads-count [1 :int]
:origin-topic "another-test-topic"
:default-key-serde "org.apache.kafka.common.serialization.Serdes$StringSerde"
:default-value-serde "org.apache.kafka.common.serialization.Serdes$StringSerde"
:key-deserializer-encoding "UTF8"
:value-deserializer-encoding "UTF8"
:deserializer-encoding "UTF8"
:upgrade-from "1.1"
:channels {:channel-1 {:worker-count [10 :int]
:retry {:count [5 :int]
:enabled [true :bool]}}}}}
:tracer {:enabled [true :bool]
:custom-provider ""}}}
4 changes: 4 additions & 0 deletions resources/config.test.datadog.only.edn
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
;; TODO: delete this file in the future, :datadog key will be replaced by :statsd key
{:ziggurat {:datadog {:host "localhost"
:port [8126 :int]
:enabled [true :bool]}}}
65 changes: 34 additions & 31 deletions resources/config.test.edn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
:env [:dev :keyword]
:nrepl-server {:port [7011 :int]}
:datadog {:host "localhost"
:port [8126 :int]
:enabled [false :bool]}
:statsd {:host "localhost"
:port [8125 :int]
:enabled [false :bool]}
:sentry {:enabled [false :bool]
Expand Down Expand Up @@ -29,34 +32,34 @@
:enabled [true :bool]}
:http-server {:port [8010 :int]
:thread-count [100 :int]}
:stream-router {:default {:application-id "test"
:bootstrap-servers "localhost:9092"
:stream-threads-count [1 :int]
:origin-topic "topic"
:upgrade-from "1.1"
:channels {:channel-1 {:worker-count [10 :int]
:retry {:type [:linear :keyword]
:count [5 :int]
:enabled [true :bool]}}}
:producer {:bootstrap-servers "localhost:9092"
:acks "all"
:retries-config 5
:max-in-flight-requests-per-connection 5
:enable-idempotence false
:value-serializer "org.apache.kafka.common.serialization.StringSerializer"
:key-serializer "org.apache.kafka.common.serialization.StringSerializer"}}
:using-string-serde {:application-id "test"
:bootstrap-servers "localhost:9092"
:stream-threads-count [1 :int]
:origin-topic "another-test-topic"
:default-key-serde "org.apache.kafka.common.serialization.Serdes$StringSerde"
:default-value-serde "org.apache.kafka.common.serialization.Serdes$StringSerde"
:key-deserializer-encoding "UTF8"
:value-deserializer-encoding "UTF8"
:deserializer-encoding "UTF8"
:upgrade-from "1.1"
:channels {:channel-1 {:worker-count [10 :int]
:retry {:count [5 :int]
:enabled [true :bool]}}}}}
:tracer {:enabled [true :bool]
:custom-provider ""}}}
:stream-router {:default {:application-id "test"
:bootstrap-servers "localhost:9092"
:stream-threads-count [1 :int]
:origin-topic "topic"
:upgrade-from "1.1"
:channels {:channel-1 {:worker-count [10 :int]
:retry {:type [:linear :keyword]
:count [5 :int]
:enabled [true :bool]}}}
:producer {:bootstrap-servers "localhost:9092"
:acks "all"
:retries-config 5
:max-in-flight-requests-per-connection 5
:enable-idempotence false
:value-serializer "org.apache.kafka.common.serialization.StringSerializer"
:key-serializer "org.apache.kafka.common.serialization.StringSerializer"}}
:using-string-serde {:application-id "test"
:bootstrap-servers "localhost:9092"
:stream-threads-count [1 :int]
:origin-topic "another-test-topic"
:default-key-serde "org.apache.kafka.common.serialization.Serdes$StringSerde"
:default-value-serde "org.apache.kafka.common.serialization.Serdes$StringSerde"
:key-deserializer-encoding "UTF8"
:value-deserializer-encoding "UTF8"
:deserializer-encoding "UTF8"
:upgrade-from "1.1"
:channels {:channel-1 {:worker-count [10 :int]
:retry {:count [5 :int]
:enabled [true :bool]}}}}}
:tracer {:enabled [true :bool]
:custom-provider ""}}}
3 changes: 3 additions & 0 deletions resources/config.test.statsd.only.edn
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{:ziggurat {:statsd {:host "localhost"
:port [8126 :int]
:enabled [true :bool]}}}
13 changes: 8 additions & 5 deletions src/ziggurat/config.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@
[mount.core :refer [defstate]]
[ziggurat.util.java-util :as util])
(:gen-class
:name tech.gojek.ziggurat.internal.Config
:methods [^{:static true} [get [String] Object]
^{:static true} [getIn [java.lang.Iterable] Object]]))
^{:static true} [getIn [java.lang.Iterable] Object]]
:name tech.gojek.ziggurat.internal.Config))

(def config-file "config.edn")

(def default-config {:ziggurat {:nrepl-server {:port 70171}
:datadog {:port 8125
:datadog {:port 8125 ;; TODO: :datadog key will be removed in the future, will be replaced by the :statsd key
:enabled false}
:sentry {:enabled false
:worker-count 10
Expand Down Expand Up @@ -66,7 +66,7 @@

(defstate config
:start (let [config-values-from-env (config-from-env config-file)
app-name (-> config-values-from-env :ziggurat :app-name)]
app-name (-> config-values-from-env :ziggurat :app-name)]
(deep-merge (interpolate-config default-config app-name) config-values-from-env)))

(defn ziggurat-config []
Expand All @@ -75,6 +75,10 @@
(defn rabbitmq-config []
(get (ziggurat-config) :rabbit-mq))

(defn statsd-config []
(let [cfg (ziggurat-config)]
(get cfg :statsd (:datadog cfg)))) ;; TODO: remove datadog in the future

(defn get-in-config [ks]
(get-in (ziggurat-config) ks))

Expand All @@ -97,4 +101,3 @@
(defn -get [^String key]
(let [config-vals (get config (keyword key))]
(java-response config-vals)))

Loading

0 comments on commit e83a531

Please sign in to comment.