From 4184491719b3be7129786e01d4aa466ff582103c Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 4 Feb 2013 12:32:32 +0700 Subject: [PATCH 1/2] Add middleware support, refactor appender decoration * Appenders are now decorated via 2 separate mechanisms: a per-appender decorator (as before), and a new per-juxt (i.e. combined) decorator. The per-juxt wrapping provides a performance-conscious hook for higher-level facilities like the new middleware feature. * Moved compile-time config wrapping from per-appender to per-juxt, improving performance. * Fixed appender wrapper ordering. --- README.md | 4 +- src/taoensso/timbre.clj | 162 ++++++++++++++++++++++++---------------- 2 files changed, 99 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 45315705..ce1debf7 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Timbre is an attempt to make **simple logging simple** and more **complex loggin * Small, uncomplicated **all-Clojure** library. * **Super-simple map-based config**: no arcane XML or properties files! * **Decent performance** (low overhead). - * Flexible **fn-centric appender model**. + * Flexible **fn-centric appender model** with **middleware**. * Sensible built-in appenders including simple **email appender**. * Tunable **flood control** and **asynchronous** logging support. * Robust **namespace filtering**. @@ -86,6 +86,8 @@ Configuring Timbre couldn't be simpler. Let's check out (some of) the defaults: :ns-whitelist [] :ns-blacklist [] + :middleware [] ; As of 1.4.0, see source code + :timestamp-pattern "yyyy-MMM-dd HH:mm:ss ZZ" :timestamp-locale nil diff --git a/src/taoensso/timbre.clj b/src/taoensso/timbre.clj index dc1520bb..2ac563df 100644 --- a/src/taoensso/timbre.clj +++ b/src/taoensso/timbre.clj @@ -41,23 +41,26 @@ ;;;; Default configuration and appenders (utils/defonce* config - "This map atom controls everything about the way Timbre operates. In - particular note the flexibility to add arbitrary appenders. - - An appender is a map with keys: - :doc, :min-level, :enabled?, :async?, :max-message-per-msecs, :fn? - - An appender's fn takes a single map argument with keys: - :level, :message, :more ; From all logging macros (`info`, etc.) - :profiling-stats ; From `profile` macro - :ap-config ; `shared-appender-config` - :prefix ; Output of `prefix-fn` - - Other keys include: :instant, :timestamp, :hostname, :ns, :error? - - See source code for examples. - See `set-config!`, `merge-config!`, `set-level!` for convenient config - editing." + "This map atom controls everything about the way Timbre operates. + + APPENDERS + An appender is a map with keys: + :doc, :min-level, :enabled?, :async?, :max-message-per-msecs, :fn + + An appender's fn takes a single map argument with keys: + :level, :message, :more ; From all logging macros (`info`, etc.) + :profiling-stats ; From `profile` macro + :ap-config ; `shared-appender-config` + :prefix ; Output of `prefix-fn` + And also: :instant, :timestamp, :hostname, :ns, :error? + + MIDDLEWARE + Middleware are fns (applied right-to-left) that transform the map argument + dispatched to appender fns. If any middleware returns nil, no dispatching + will occur (i.e. the event will be filtered). + + See source code for examples. See `set-config!`, `merge-config!`, `set-level!` + for convenient config editing." (atom {:current-level :debug ;;; Control log filtering by namespace patterns (e.g. ["my-app.*"]). @@ -65,16 +68,9 @@ :ns-whitelist [] :ns-blacklist [] - ;; TODO Generalized transformation/filtering unary fns to operate on - ;; logging requests to either either filter or transform logging - ;; messages (e.g. obscure security credentials). - ;; - ;; Could use a cacheable comp/juxt and include ns white/black list - ;; functionality? Possibly even just prepend to the regular appender - ;; juxt (assuming we keep ns filtering separate)? Note that this'd - ;; also make any additional middlware cost async-able. - ;; - ;; :middleware [] + ;; Fns (applied right-to-left) to transform/filter appender fn args. + ;; Useful for obfuscating credentials, pattern filtering, etc. + :middleware [] ;;; Control :timestamp format :timestamp-pattern "yyyy-MMM-dd HH:mm:ss ZZ" ; SimpleDateFormat pattern @@ -133,44 +129,12 @@ ;;;; Appender-fn decoration -(defn- make-timestamp-fn - "Returns a unary fn that formats instants using given pattern string and an - optional Locale." - [^String pattern ^Locale locale] - (let [format (if locale - (SimpleDateFormat. pattern locale) - (SimpleDateFormat. pattern))] - (fn [^Date instant] (.format ^SimpleDateFormat format instant)))) - -(comment ((make-timestamp-fn "yyyy-MMM-dd" nil) (Date.))) - -(def get-hostname - (utils/memoize-ttl - 60000 (fn [] (.. java.net.InetAddress getLocalHost getHostName)))) - (defn- wrap-appender-fn "Wraps compile-time appender fn with additional runtime capabilities controlled by compile-time config." [{apfn :fn :keys [async? max-message-per-msecs] :as appender}] - (-> - ;; Wrap to add compile-time stuff to runtime appender arguments - (let [{ap-config :shared-appender-config - :keys [timestamp-pattern timestamp-locale prefix-fn]} @config - - timestamp-fn (make-timestamp-fn timestamp-pattern timestamp-locale)] - - (fn [{:keys [instant] :as apfn-args}] - (let [apfn-args (merge apfn-args {:ap-config ap-config - :timestamp (timestamp-fn instant) - :hostname (get-hostname)})] - (apfn (assoc apfn-args :prefix (prefix-fn apfn-args)))))) - - ;; Wrap for asynchronicity support - ((fn [apfn] - (if-not async? - apfn - (let [agent (agent nil :error-mode :continue)] - (fn [apfn-args] (send-off agent (fn [_] (apfn apfn-args)))))))) + (->> ; Wrapping applies capabilities bottom-to-top + apfn ;; Wrap for runtime flood-safety support ((fn [apfn] @@ -199,7 +163,60 @@ (->> (keys timers-snapshot) (filter #(allow? (timers-snapshot %))))] (when (seq expired-timers) - (apply swap! flood-timers dissoc expired-timers)))))))))))) + (apply swap! flood-timers dissoc expired-timers)))))))))) + + ;; Wrap for async (agent) support + ((fn [apfn] + (if-not async? + apfn + (let [agent (agent nil :error-mode :continue)] + (fn [apfn-args] (send-off agent (fn [_] (apfn apfn-args)))))))))) + +(defn- make-timestamp-fn + "Returns a unary fn that formats instants using given pattern string and an + optional Locale." + [^String pattern ^Locale locale] + (let [format (if locale + (SimpleDateFormat. pattern locale) + (SimpleDateFormat. pattern))] + (fn [^Date instant] (.format ^SimpleDateFormat format instant)))) + +(comment ((make-timestamp-fn "yyyy-MMM-dd" nil) (Date.))) + +(def get-hostname + (utils/memoize-ttl + 60000 (fn [] (.. java.net.InetAddress getLocalHost getHostName)))) + +(defn- wrap-appender-juxt + "Wraps compile-time appender juxt with additional runtime capabilities + (incl. middleware) controller by compile-time config. Like `wrap-appender-fn` + but operates on the entire juxt at once." + [juxtfn] + (->> ; Wrapping applies capabilities bottom-to-top + juxtfn + + ;; Wrap to add middleware transforms/filters + ((fn [juxtfn] + (if-let [middleware (seq (:middleware @config))] + (let [composed-middleware + (apply comp (map (fn [mf] (fn [args] (when args (mf args)))) + middleware))] + (fn [juxtfn-args] + (when-let [juxtfn-args (composed-middleware juxtfn-args)] + (juxtfn juxtfn-args)))) + juxtfn))) + + ;; Wrap to add compile-time stuff to runtime appender arguments + ((fn [juxtfn] + (let [{ap-config :shared-appender-config + :keys [timestamp-pattern timestamp-locale prefix-fn]} @config + + timestamp-fn (make-timestamp-fn timestamp-pattern timestamp-locale)] + (fn [{:keys [instant] :as juxtfn-args}] + (let [juxtfn-args (merge juxtfn-args {:ap-config ap-config + :timestamp (timestamp-fn instant) + :hostname (get-hostname)})] + (juxtfn (assoc juxtfn-args :prefix (prefix-fn juxtfn-args)))))))))) ;;;; Caching @@ -208,7 +225,7 @@ (def appenders-juxt-cache "Per-level, combined relevant appender-fns to allow for fast runtime appender-fn dispatch: - {:level (juxt wrapped-appender-fn wrapped-appender-fn ...) or nil + {:level (wrapped-juxt wrapped-appender-fn wrapped-appender-fn ...) or nil ...}" (atom {})) @@ -233,10 +250,11 @@ (when-let [ap-ids (keys rel-aps)] (->> ap-ids (map #(wrap-appender-fn (rel-aps %))) - (apply juxt)))))))) + (apply juxt) + (wrap-appender-juxt)))))))) (reset! appenders-juxt-cache))) -;;; Namespace filter ; TODO Generalize to arbitrary configurable middleware juxt? +;;; Namespace filter (def ns-filter-cache "@ns-filter-cache => (fn relevant-ns? [ns] ...)" (atom (constantly true))) @@ -372,4 +390,16 @@ (spy (* 6 5 4 3 2 1)) (spy :debug :factorial6 (* 6 5 4 3 2 1)) (info (Exception. "noes!") "bar") - (spy (/ 4 0))) \ No newline at end of file + (spy (/ 4 0)) + + ;; Middleware + (info {:name "Robert Paulson" :password "Super secret"}) + (set-config! + [:middleware] + [(fn [{:keys [hostname message] :as args}] + (cond (= hostname "filtered-host") nil ; Filter + (map? message) + (if (contains? message :password) + (assoc args :message (assoc message :password "*****")) + args) + :else args))])) \ No newline at end of file From 4c76cd99e53fbd6c4f7c95116ef0052b8cb24eea Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Mon, 4 Feb 2013 14:30:41 +0700 Subject: [PATCH 2/2] Bump version (1.4.0) --- README.md | 4 ++-- project.clj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ce1debf7..9d75f841 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Current [semantic](http://semver.org/) version: ```clojure -[com.taoensso/timbre "1.3.1"] +[com.taoensso/timbre "1.4.0"] ``` # Timbre, a (sane) Clojure logging & profiling library @@ -27,7 +27,7 @@ Timbre is an attempt to make **simple logging simple** and more **complex loggin Depend on Timbre in your `project.clj`: ```clojure -[com.taoensso/timbre "1.3.1"] +[com.taoensso/timbre "1.4.0"] ``` and `use` the library: diff --git a/project.clj b/project.clj index a70eb662..75afdb10 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/timbre "1.3.1" +(defproject com.taoensso/timbre "1.4.0" :description "Clojure logging & profiling library" :url "https://github.com/ptaoussanis/timbre" :license {:name "Eclipse Public License"}