Removes proto-class dependency from config and references from readme

gojek · Jul 10, 2019 · c194218 · c194218
1 parent 90ac6c0
commit c194218
Show file tree

Hide file tree

Showing 6 changed files with 2 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -179,7 +179,6 @@ All Ziggurat configs should be in your `clonfig` `config.edn` under the `:ziggur
                                                           :stream-threads-count           [1 :int]
                                                           :origin-topic                   "kafka-topic-*"
                                                           :oldest-processed-message-in-s  [604800 :int]
-                                                          :proto-class          "proto-class"
                                                           :changelog-topic-replication-factor [3 :int]
                                                           :producer   {:bootstrap-servers                     "localhost:9092"
                                                                        :acks                                  "all"
@@ -224,9 +223,8 @@ All Ziggurat configs should be in your `clonfig` `config.edn` under the `:ziggur
         * application-id - The Kafka consumer group id. [Documentation](https://kafka.apache.org/intro#intro_consumers)
         * bootstrap-servers - The Kafka brokers that the application will read from. It accepts a comma seperated value.
         * stream-threads-count - The number of parallel threads that should read messages from Kafka. This can scale up to the number of partitions on the topic you wish to read from.
-        * origin-topic - The topic that the stream should read from. This can be a regex that enables you to read from multiple streams and handle the messages in the same way. It is to be kept in mind that the messages from different streams should be of the same proto-class.
+        * origin-topic - The topic that the stream should read from. This can be a regex that enables you to read from multiple streams and handle the messages in the same way. It is to be kept in mind that the messages from different streams will be passed to the same mapper-function.
         * oldest-processed-messages-in-s - The oldest message which will be processed by stream in second. By default the value is 604800 (1 week)
-        * proto-class - The proto-class of the message so that it can be decompiled before being passed to the mapper function
         * changelog-topic-replication-factor - the internal changelog topic replication factor. By default the value is 3
         * producer - Configuration for KafkaProducer. Currently, only following options are supported. Please see [Producer Configs](https://kafka.apache.org/documentation/#producerconfigs) for detailed explanation for each of the configuration parameters.
             * bootstrap.servers - A list of host/port pairs to use for establishing the initial connection to the Kafka cluster.

diff --git a/resources/config.test.ci.edn b/resources/config.test.ci.edn
@@ -32,7 +32,6 @@
                                                       :bootstrap-servers    "localhost:9092"
                                                       :stream-threads-count [1 :int]
                                                       :origin-topic         "topic"
-                                                      :proto-class          "flatland.protobuf.test.Example$Photo"
                                                       :upgrade-from         "1.1"
                                                       :channels             {:channel-1 {:worker-count [10 :int]
                                                                                          :retry        {:count   [5 :int]
@@ -48,7 +47,6 @@
                                                       :bootstrap-servers    "localhost:9092"
                                                       :stream-threads-count [1 :int]
                                                       :origin-topic         "topic"
-                                                      :proto-class          "flatland.protobuf.test.Example$Photo"
                                                       :channels             {:channel-1 {:worker-count [10 :int]
                                                                                          :retry        {:count   [5 :int]
                                                                                                         :enabled [true :bool]}}}}}}}
diff --git a/resources/config.test.edn b/resources/config.test.edn
@@ -32,7 +32,6 @@
                                                       :bootstrap-servers    "localhost:9092"
                                                       :stream-threads-count [1 :int]
                                                       :origin-topic         "topic"
-                                                      :proto-class          "flatland.protobuf.test.Example$Photo"
                                                       :upgrade-from         "1.1"
                                                       :channels             {:channel-1 {:worker-count [10 :int]
                                                                                          :retry        {:count   [5 :int]
@@ -48,7 +47,6 @@
                                                       :bootstrap-servers    "localhost:9092"
                                                       :stream-threads-count [1 :int]
                                                       :origin-topic         "topic"
-                                                      :proto-class          "flatland.protobuf.test.Example$Photo"
                                                       :channels             {:channel-1 {:worker-count [10 :int]
                                                                                          :retry        {:count   [5 :int]
                                                                                                         :enabled [true :bool]}}}}}}}
diff --git a/src/ziggurat/producer.clj b/src/ziggurat/producer.clj
@@ -16,7 +16,6 @@
                                :bootstrap-servers    \"localhost:9092\"\n
                                :stream-threads-count [1 :int]\n
                                :origin-topic         \"topic\"\n
-                               :proto-class          \"flatland.protobuf.test.Example$Photo\"\n
                                :channels             {:channel-1 {:worker-count [10 :int]\n  :retry {:count [5 :int]\n  :enabled [true :bool]}}}\n
                                :producer             {:bootstrap-servers \"localhost:9092\"\n
                                                       :acks \"all\"\n

diff --git a/src/ziggurat/streams.clj b/src/ziggurat/streams.clj
@@ -96,7 +96,7 @@
         additional-tags   {:topic_name topic-entity-name}]
     (.transform stream-builder (transformer-supplier metric-namespaces oldest-processed-message-in-s additional-tags) (into-array [(.name (store-supplier-builder))]))))
 
-(defn- topology [handler-fn {:keys [origin-topic proto-class oldest-processed-message-in-s]} topic-entity channels]
+(defn- topology [handler-fn {:keys [origin-topic oldest-processed-message-in-s]} topic-entity channels]
   (let [builder           (StreamsBuilder.)
         topic-entity-name (name topic-entity)
         topic-pattern     (Pattern/compile origin-topic)]

diff --git a/test/ziggurat/producer_test.clj b/test/ziggurat/producer_test.clj
@@ -16,7 +16,6 @@
                            :bootstrap-servers    "localhost:9092"
                            :stream-threads-count [1 :int]
                            :origin-topic         "topic"
-                           :proto-class          "flatland.protobuf.test.Example$Photo"
                            :channels             {:channel-1 {:worker-count [10 :int]
                                                               :retry        {:count   [5 :int]
                                                                              :enabled [true :bool]}}}}})