Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Improved internal stats by tracking processed logs and metrics per lo…

…g file.
  • Loading branch information...
commit 3260ae4b54a4ca29cb5d4ebb03920f19087a404e 1 parent 9e99ed5
@sbtourist authored
View
18 src/nimrod/core/store.clj
@@ -86,7 +86,7 @@
(alter memory assoc-in [(metric :ns) (metric :type) (metric :id) :samples] 0)))))))
(set-metric [this metric-ns metric-type metric-id metric aggregation]
- (update-rate-stats :operations-per-second (clock) (seconds 1))
+ (update-rate-stats [:operations-per-second] (clock) (seconds 1))
(let
[now (clock)
current-seq-value (get-in @memory [metric-ns metric-type metric-id :seq] 0)
@@ -140,7 +140,7 @@
(alter memory assoc-in [metric-ns metric-type metric-id :samples] new-samples)))))
(remove-metric [this metric-ns metric-type metric-id]
- (update-rate-stats :operations-per-second (clock) (seconds 1))
+ (update-rate-stats [:operations-per-second] (clock) (seconds 1))
(sql/with-connection connection-factory
(sql/transaction
(sql/delete-rows
@@ -151,11 +151,11 @@
(alter memory assoc-in [metric-ns metric-type] (dissoc metrics metric-id)))))
(read-metric [this metric-ns metric-type metric-id]
- (update-rate-stats :operations-per-second (clock) (seconds 1))
+ (update-rate-stats [:operations-per-second] (clock) (seconds 1))
(get-in @memory [metric-ns metric-type metric-id :metric]))
(list-metrics [this metric-ns metric-type]
- (update-rate-stats :operations-per-second (clock) (seconds 1))
+ (update-rate-stats [:operations-per-second] (clock) (seconds 1))
(sql/with-connection connection-factory
(sql/transaction
(sql/with-query-results
@@ -165,7 +165,7 @@
(into [] (map #(get %1 :id) r)))))))
(read-history [this metric-ns metric-type metric-id tags age from to]
- (update-rate-stats :operations-per-second (clock) (seconds 1))
+ (update-rate-stats [:operations-per-second] (clock) (seconds 1))
(let [now (clock)
actual-from (if (nil? from) (- now (or age default-age)) from)
actual-to (or to now)]
@@ -183,7 +183,7 @@
:values metrics})))))))
(remove-history [this metric-ns metric-type metric-id tags age from to]
- (update-rate-stats :operations-per-second (clock) (seconds 1))
+ (update-rate-stats [:operations-per-second] (clock) (seconds 1))
(let
[now (clock)
actual-from (or from 0)
@@ -206,7 +206,7 @@
(recur (min (+ upbound (days 1)) max-timestamp))))))))
(aggregate-history [this metric-ns metric-type metric-id tags age from to aggregators]
- (update-rate-stats :operations-per-second (clock) (seconds 1))
+ (update-rate-stats [:operations-per-second] (clock) (seconds 1))
(sql/with-connection connection-factory
(sql/transaction
(sql/do-prepared "SET DATABASE DEFAULT ISOLATION LEVEL SERIALIZABLE")
@@ -235,7 +235,7 @@
})))))
(list-types [this metric-ns]
- (update-rate-stats :operations-per-second (clock) (seconds 1))
+ (update-rate-stats [:operations-per-second] (clock) (seconds 1))
(sql/with-connection connection-factory
(sql/transaction
(sql/with-query-results
@@ -244,7 +244,7 @@
(into [] (for [type all-types] (type :type)))))))
(stats [this]
- (show-stats [:operations-per-second] (clock) (seconds 1))))
+ (show-stats [[:operations-per-second]] (clock) (seconds 1))))
(defn new-disk-store
([path options sampling]
View
30 src/nimrod/internal/stats.clj
@@ -3,20 +3,26 @@
(defonce stats-agent (new-agent {:stats {} :tmp {}}))
-(defn update-rate-stats [k t f]
+(defn update-rate-stats [ks t f]
(send stats-agent
(fn [state]
- (let [rate (get-in state [:stats k] 0)
- timestamp (get-in state [:tmp k :timestamp] 0)]
+ (let
+ [stats-keys (cons :stats ks)
+ tmp-keys (cons :tmp (cons :timestamp ks))
+ rate (get-in state stats-keys 0)
+ timestamp (get-in state tmp-keys 0)]
(if (<= (- t timestamp) f)
- (assoc-in state [:stats k] (inc rate))
+ (assoc-in state stats-keys (inc rate))
(-> state
- (assoc-in [:stats k] 1)
- (assoc-in [:tmp k :timestamp] t)))))))
+ (assoc-in stats-keys 1)
+ (assoc-in tmp-keys t)))))))
-(defn show-stats [ks t f]
- (into {}
- (for [k ks]
- (let [rate (get-in @stats-agent [:stats k] 0)
- timestamp (get-in @stats-agent [:tmp k :timestamp] 0)]
- [k (Math/round (double (* (/ rate (- t timestamp)) f)))]))))
+(defn show-stats [ksa t f]
+ (reduce #(assoc-in %1 (%2 0) (%2 1)) {}
+ (for [ks ksa]
+ (let
+ [stats-keys (cons :stats ks)
+ tmp-keys (cons :tmp (cons :timestamp ks))
+ rate (get-in @stats-agent stats-keys 0)
+ timestamp (get-in @stats-agent tmp-keys 0)]
+ [ks (Math/round (double (* (/ rate (- t timestamp)) f)))]))))
View
15 src/nimrod/log/tailer.clj
@@ -18,8 +18,12 @@
(proxy [TailerListenerAdapter] []
(handle [obj]
(let [now (clock) processed (process id obj)]
- (update-rate-stats :processed-logs-per-second now (seconds 1))
- (when processed (update-rate-stats :processed-metrics-per-second now (seconds 1)))))
+ (update-rate-stats [:processed-logs-per-second] now (seconds 1))
+ (update-rate-stats [:processed-logs-per-second-per-file (keyword id)] now (seconds 1))
+ (when processed
+ (do
+ (update-rate-stats [:processed-metrics-per-second] now (seconds 1))
+ (update-rate-stats [:processed-metrics-per-second-per-file (keyword id)] now (seconds 1))))))
(fileNotFound [] (log/error (str "Log file not found: " log)))
(fileRotated [] (log/info (str "Rotated log file: " log)))
(error [obj] (log/error (.getMessage obj) obj))
@@ -56,4 +60,9 @@
(for [tailer @tailers] [(tailer 0) ((tailer 1) :log)])))
(defn show-tail-stats []
- (show-stats [:processed-logs-per-second :processed-metrics-per-second] (clock) (seconds 1)))
+ (show-stats
+ (-> [[:processed-logs-per-second] [:processed-metrics-per-second]]
+ (into (for [id (keys @tailers)] [:processed-logs-per-second-per-file (keyword id)]))
+ (into (for [id (keys @tailers)] [:processed-metrics-per-second-per-file (keyword id)])))
+ (clock)
+ (seconds 1)))
View
20 test/nimrod/internal/stats_test.clj
@@ -6,10 +6,20 @@
(deftest stats
(testing "Equals the number of updates (3) times 1000/250"
- (update-rate-stats :test (clock) (seconds 1))
- (update-rate-stats :test (clock) (seconds 1))
- (update-rate-stats :test (clock) (seconds 1))
+ (update-rate-stats [:test] (clock) (seconds 1))
+ (update-rate-stats [:test] (clock) (seconds 1))
+ (update-rate-stats [:test] (clock) (seconds 1))
(Thread/sleep 250)
- (is (= 12 ((show-stats [:test] (clock) (seconds 1)) :test))))
+ (is (= 12 ((show-stats [[:test]] (clock) (seconds 1)) :test))))
(testing "Zero-ed"
- (is (= 0 ((show-stats [:test] (+ (clock) (seconds 10)) (seconds 1)) :test)))))
+ (is (= 0 ((show-stats [[:test]] (+ (clock) (seconds 10)) (seconds 1)) :test)))))
+
+(deftest multiple-stats
+ (testing "Multiple stats"
+ (update-rate-stats [:single] (clock) (seconds 1))
+ (update-rate-stats [:nested :one] (clock) (seconds 1))
+ (update-rate-stats [:nested :two] (clock) (seconds 1))
+ (Thread/sleep 250)
+ (is (< 0 ((show-stats [[:single] [:nested :one] [:nested :two]] (clock) (seconds 1)) :single)))
+ (is (< 0 (((show-stats [[:single] [:nested :one] [:nested :two]] (clock) (seconds 1)) :nested) :one)))
+ (is (< 0 (((show-stats [[:single] [:nested :one] [:nested :two]] (clock) (seconds 1)) :nested) :two)))))
Please sign in to comment.
Something went wrong with that request. Please try again.