Permalink
Browse files

Track full list of stats directly in defstat.

  • Loading branch information...
1 parent 84de280 commit b06c2beb64ff206b8098216730ae393bed2c28dd @technomancy technomancy committed Oct 2, 2012
Showing with 22 additions and 239 deletions.
  1. +19 −235 src/pulse/def.clj
  2. +1 −2 src/pulse/emitter.clj
  3. +1 −1 src/pulse/merger.clj
  4. +1 −1 src/pulse/receiver.clj
View
@@ -232,9 +232,13 @@
(let [recent-interval (or recent-interval 300)]
(last-sum pred-fn part-fn (fn [evt] (if (cnt-fn evt) 1 0)) recent-interval)))
+;; all stats
+(def all (atom []))
+
;; for stats which apply across clouds
(defmacro defstat-single [stat-name stat-body]
- `(def ~stat-name (merge ~stat-body {:name (name '~stat-name)})))
+ `(do (def ~stat-name (merge ~stat-body {:name (name '~stat-name)}))
+ (swap! all conj ~stat-name)))
(defn scope-stat [cloud stat-name]
(if (= cloud (conf/default-cloud))
@@ -247,11 +251,13 @@
~@(for [cloud (conf/clouds)
:let [scoped-name (scope-stat cloud stat-name)
scoped-var-name (symbol (string/replace scoped-name "." "-"))]]
- `(def ~scoped-var-name (assoc ~body-sym
- :name ~scoped-name
+ `(do
+ (def ~scoped-var-name (assoc ~body-sym
+ :name ~(str (conf/graphite-prefix) scoped-name)
:pred-fn (fn [evt#]
(and (= (:cloud evt#) ~cloud)
- ((:pred-fn ~body-sym) evt#)))))))))
+ ((:pred-fn ~body-sym) evt#)))))
+ (swap! all conj ~scoped-var-name))))))
(defn kv? [m k v]
(= (k m) v))
@@ -620,17 +626,17 @@
(k? evt :staleness)))
:staleness))
-(defstat hermes-elevated-route-lookups-per-minute
- (per-minute
- (fn [evt] (and (hermes-request? evt) (kv? evt :code "OK") (>=? evt :route 2.0)))))
+;; (defstat hermes-elevated-route-lookups-per-minute
+;; (per-minute
+;; (fn [evt] (and (hermes-request? evt) (kv? evt :code "OK") (>=? evt :route 2.0)))))
-(defstat hermes-slow-route-lookups-per-minute
- (per-minute
- (fn [evt] (and (hermes-request? evt) (kv? evt :code "OK") (>=? evt :route 10.0)))))
+;; (defstat hermes-slow-route-lookups-per-minute
+;; (per-minute
+;; (fn [evt] (and (hermes-request? evt) (kv? evt :code "OK") (>=? evt :route 10.0)))))
-(defstat hermes-catastrophic-route-lookups-per-minute
- (per-minute
- (fn [evt] (and (hermes-request? evt) (kv? evt :code "OK") (>=? evt :route 100.0)))))
+;; (defstat hermes-catastrophic-route-lookups-per-minute
+;; (per-minute
+;; (fn [evt] (and (hermes-request? evt) (kv? evt :code "OK") (>=? evt :route 100.0)))))
(defstat hermes-slow-redis-lookups-per-minute
(per-minute
@@ -1242,225 +1248,3 @@
(defstat-single pulse-events-per-second
(per-second
(fn [evt] (and (kv? evt :app "pulse") (kv? evt :deploy (conf/deploy))))))
-
-(def defaults
- [
- ; global
- events-per-second
- events-per-second-unparsed
- amqp-publishes-per-second
- amqp-receives-per-second
- amqp-timeouts-per-minute
-
- ; routing
- nginx-requests-per-second
- nginx-requests-domains-per-minute
- nginx-500-per-minute
- nginx-502-per-minute
- nginx-503-per-minute
- nginx-504-per-minute
- nginx-500-domains-per-minute
- nginx-502-domains-per-minute
- nginx-503-domains-per-minute
- nginx-504-domains-per-minute
- nginx-errors-per-minute
- nginx-errors-instances-per-minute
- varnish-requests-per-second
- varnish-500-per-minute
- varnish-502-per-minute
- varnish-503-per-minute
- varnish-504-per-minute
- varnish-purges-per-minute
- rendezvous-joins-per-minute
- rendezvous-rendezvous-per-minute
- hermes-requests-per-second
- hermes-requests-apps-per-minute
- hermes-h10-per-minute
- hermes-h11-per-minute
- hermes-h12-per-minute
- hermes-h13-per-minute
- hermes-h14-per-minute
- hermes-h15-per-minute
- hermes-h16-per-minute
- hermes-h17-per-minute
- hermes-h18-per-minute
- hermes-h19-per-minute
- hermes-h20-per-minute
- hermes-h80-per-minute
- hermes-h99-per-minute
- hermes-h10-apps-per-minute
- hermes-h11-apps-per-minute
- hermes-h12-apps-per-minute
- hermes-h13-apps-per-minute
- hermes-h14-apps-per-minute
- hermes-h15-apps-per-minute
- hermes-h16-apps-per-minute
- hermes-h17-apps-per-minute
- hermes-h18-apps-per-minute
- hermes-h19-apps-per-minute
- hermes-h20-apps-per-minute
- hermes-h80-apps-per-minute
- hermes-h99-apps-per-minute
- hermes-econns-per-minute
- hermes-econns-apps-per-minute
- hermes-errors-per-minute
- hermes-services-lockstep-updates-per-minute
- hermes-services-lockstep-connections-per-minute
- hermes-services-lockstep-disconnects-per-minute
- hermes-services-lockstep-mean-latency
- hermes-services-lockstep-mean-staleness
- hermes-procs-lockstep-updates-per-minute
- hermes-procs-lockstep-connections-per-minute
- hermes-procs-lockstep-disconnects-per-minute
- hermes-procs-lockstep-mean-latency
- hermes-procs-lockstep-mean-staleness
- hermes-domains-lockstep-updates-per-minute
- hermes-domains-lockstep-connections-per-minute
- hermes-domains-lockstep-disconnects-per-minute
- hermes-domains-lockstep-mean-latency
- hermes-domains-lockstep-mean-staleness
- hermes-domain-groups-lockstep-updates-per-minute
- hermes-domain-groups-lockstep-connections-per-minute
- hermes-domain-groups-lockstep-disconnects-per-minute
- hermes-domain-groups-lockstep-mean-latency
- hermes-domain-groups-lockstep-mean-staleness
-; hermes-elevated-route-lookups-per-minute
-; hermes-slow-route-lookups-per-minute
-; hermes-catastrophic-route-lookups-per-minute
- hermes-slow-redis-lookups-per-minute
- hermes-catastrophic-redis-lookups-per-minute
- hermes-processes-last
- hermes-ports-last
- logplex-msg-processed
- logplex-drain-delivered
-
- ; railgun
- railgun-running-count
- railgun-denied-count
- railgun-packed-count
- railgun-loaded-count
- railgun-critical-count
- railgun-accepting-count
- railgun-load-avg-15m-mean
- railgun-ps-running-total-last
- railgun-ps-running-web-last
- railgun-ps-running-worker-last
- railgun-ps-running-clock-last
- railgun-ps-running-console-last
- railgun-ps-running-rake-last
- railgun-ps-running-other-last
- railgun-runs-per-minute
- railgun-returns-per-minute
- railgun-kills-per-minute
- railgun-subscribes-per-minute
- railgun-unsubscribes-per-minute
- railgun-status-batches-per-minute
- railgun-gcs-per-minute
- railgun-kill-time-mean
- railgun-save-time-mean
- railgun-unpack-time-mean
- railgun-setup-time-mean
- railgun-launch-time-mean
- railgun-status-batch-time-mean
- railgun-gc-time-mean
- railgun-s3-requests-per-minute
- railgun-s3-errors-per-minute
- railgun-s3-time-mean
- railgun-slug-download-fails-per-minute
- railgun-s3-canary-requests-per-minute
- railgun-s3-canary-errors-per-minute
- railgun-s3-canary-time-mean
- railgun-r10-per-minute
- railgun-r11-per-minute
- railgun-r12-per-minute
- railgun-r14-per-minute
- railgun-r15-per-minute
- railgun-r16-per-minute
- railgun-r17-per-minute
- railgun-r18-per-minute
- railgun-r10-apps-per-minute
- railgun-r11-apps-per-minute
- railgun-r12-apps-per-minute
- railgun-r14-apps-per-minute
- railgun-r15-apps-per-minute
- railgun-r17-apps-per-minute
- railgun-r18-apps-per-minute
- railgun-inits-per-minute
- railgun-traps-per-minute
- railgun-exits-per-minute
- railgun-unhandled-exceptions-per-minute
- railgun-pings-per-minute
- railgun-heartbeats-per-minute
- railgun-events-per-second
- railgun-runtime-bus-publishes-per-minute
- railgun-runtime-bus-processing-per-minute
- railgun-runtime-bus-expired-per-minute
- railgun-runtime-bus-invalid-per-minute
- railgun-runtime-bus-failed-pushes-per-minute
- railgun-runtime-bus-failed-lpops-per-minute
-
- build-railgun-running-count
- build-railgun-denied-count
- build-railgun-packed-count
- build-railgun-loaded-count
- build-railgun-critical-count
- build-railgun-accepting-count
- build-railgun-load-avg-15m-mean
- build-railgun-ps-running-total-last
-
- ; psmgr
- psmgr-idles-per-minute
- psmgr-unidles-per-minute
- psmgr-run-requests-per-minute
- psmgr-kill-requests-per-minute
- psmgr-converges-per-second
- psmgr-unhandled-exceptions-per-minute
- psmgr-runtime-bus-receives-per-minute
- psmgr-runtime-bus-timeouts-per-minute
- psmgr-runtime-bus-published-per-minute
- psmgr-events-per-second
- psmgr-api-per-minute
- psmgr-api-time
- psmgr-runs-per-minute
- psmgr-cycles-per-minute
- psmgr-lost-runs-per-minute
- psmgr-foregrounds-per-minute
- psmgr-backgrounds-per-minute
-
- ; packaging
- gitproxy-connections-per-minute
- gitproxy-invalids-per-minute
- gitproxy-errors-per-minute
- gitproxy-successes-per-minute
- gitproxy-mean-metadata-time
- gitproxy-mean-provision-time
- gitproxy-mean-service-time
- slugc-compiles-per-minute
- slugc-failures-per-minute
- slugc-errors-per-minute
- slugc-successes-per-minute
- slugc-aspen-compiles-per-minute
- slugc-bamboo-compiles-per-minute
- slugc-cedar-compiles-per-minute
- slugc-mean-stow-time
- slugc-mean-release-time
- slugc-stow-errors-per-minute
- slugc-release-errors-per-minute
- slugc-mean-compile-time
-
- ; data
- shen-errors-per-minute
-
- ; internal
- pulse-events-per-second
- ])
-
-(defn non-defaults [stat]
- (for [cloud (conf/clouds)
- :when (not= cloud (conf/default-cloud))
- :let [scoped-name (scope-stat cloud (:name stat))
- stat-sym (symbol (string/replace scoped-name "." "-"))]
- :when (ns-resolve 'pulse.def stat-sym)]
- @(ns-resolve 'pulse.def stat-sym)))
-
-(def all (apply concat defaults (map non-defaults defaults)))
View
@@ -13,8 +13,7 @@
(atom '()))
(defn post [metrics-url stats]
- (let [{:keys [host]} (util/url-parse metrics-url)
- stats (map #(update-in % [:name] (partial str (conf/graphite-prefix))) stats)]
+ (let [{:keys [host]} (util/url-parse metrics-url)]
(log :fn "post" :at "start" :host host)
(try
(http/post metrics-url
View
@@ -40,7 +40,7 @@
(log :fn "main" :at "start")
(let [apply-queue (queue/init 2000)
publish-queue (queue/init 100)
- stats-map (init-stats def/all shard)]
+ stats-map (init-stats @def/all shard)]
(queue/init-watcher apply-queue "apply")
(queue/init-watcher publish-queue "publish")
(io/init-publishers publish-queue (conf/redis-url) "stats.merged"
View
@@ -36,7 +36,7 @@
(log :fn "main" :at "start")
(let [apply-queue (queue/init 10000)
publish-queue (queue/init 1000)
- stats-states (init-stats def/all)]
+ stats-states (init-stats @def/all)]
(queue/init-watcher apply-queue "apply")
(queue/init-watcher publish-queue "publish")
(io/init-publishers publish-queue (conf/redis-url) io/shard-channel

0 comments on commit b06c2be

Please sign in to comment.