add additional sampler support for #45

jeremydaly · Aug 18, 2018 · fe54a08 · fe54a08
1 parent 31df7c5
commit fe54a08
Show file tree

Hide file tree

Showing 3 changed files with 86 additions and 39 deletions.
diff --git a/lib/logger.js b/lib/logger.js
@@ -6,6 +6,7 @@
  * @license MIT
  */
 
+const UTILS = require('./utils') // Require utils library
 
 // Config logger
 exports.config = (config,levels) => {
@@ -90,7 +91,8 @@ exports.config = (config,levels) => {
         level,
         time: timestamp(),
         id: req.id,
-        path: req.path,
+        route: req.route,
+        method: req.method,
         [messageKey]: msg,
         timer: timer(req._start),
         sample: req._sample ? true : undefined
@@ -131,12 +133,33 @@ exports.sampler = (app,req) => {
 
   if (app._logger.sampling) {
 
-    // default level to false
+    // Default level to false
     let level = false
 
-    // Grab the rule based on route
-    let rule = app._logger.sampling.rules && app._logger.sampling.rules[req.route]
-      || app._logger.sampling.defaults
+    // Create local reference to the rulesMap
+    let map = app._logger.sampling.rulesMap
+
+    // Parse the current route
+    let route = UTILS.parsePath(req.route)
+
+    // Default wildcard mapping
+    let wildcard = {}
+
+    // Loop the map and see if this route matches
+    route.forEach(part => {
+      // Capture wildcard mappings
+      if (map['*']) wildcard = map['*']
+      // Traverse map
+      map = map[part] ? map[part] : {}
+    }) // end for loop
+
+    // Set rule reference based on route
+    let ref = map['__'+req.method] ? map['__'+req.method] :
+      map['__ANY'] ? map['__ANY'] : wildcard['__'+req.method] ?
+        wildcard['__'+req.method] : wildcard['__ANY'] ?
+          wildcard['__ANY'] : -1
+
+    let rule = ref >= 0 ? app._logger.sampling.rules[ref] : app._logger.sampling.defaults
 
     // Get last sample time (default start, last, fixed count, period count and total count)
     let counts = app._sampleCounts[rule.default ? 'default' : req.route]
@@ -146,6 +169,9 @@ exports.sampler = (app,req) => {
 
     let now = Date.now()
 
+    // Calculate the current velocity
+    let velocity = rule.rate > 0 ? rule.period*1000/(counts.tCount/(now-app._initTime)*rule.period*1000*rule.rate) : 0
+
     // If this is a new period, reset values
     if ((now-counts.start) > rule.period*1000) {
       counts.start = now
@@ -164,30 +190,15 @@ exports.sampler = (app,req) => {
         counts.fCount++
         console.log('\n*********** FIXED ***********');
     } else if (rule.rate > 0 &&
-      counts.start+Math.floor(rule.period*1000/(counts.tCount/(now-app._initTime)*rule.period*1000*rule.rate)*counts.pCount) < now) {
+      counts.start+Math.floor(velocity*counts.pCount+velocity/2) < now) {
         level = rule.level
         counts.pCount++
         console.log('\n*********** RATE ***********');
     }
 
-
     // Increment total count
     counts.tCount++
 
-
-    // console.log(
-    //   '-----------------------------------',
-    //   '\nlastSample:',app._lastSample,
-    //   '\nsampleCounts:',app._sampleCounts,
-    //   '\nrequestCount:',app._requestCount,
-    //   '\ninitTime:',app._initTime,
-    //   '\nlogger:',JSON.stringify(app._logger,null,2),
-    //   '\nroute:', req.route,
-    //   '\nmethod:', req.method,
-    //   '\n-----------------------------------'
-    // )
-
-
     return level
 
   } // end if sampling
@@ -210,25 +221,49 @@ const parseSamplerConfig = (config,levels) => {
       target: Number.isInteger(inputs.target) ? inputs.target : 1,
       rate: !isNaN(inputs.rate) && inputs.rate <= 1 ? inputs.rate : 0.1,
       period: Number.isInteger(inputs.period) ? inputs.period : 60, // in seconds
-      method: (Array.isArray(inputs.method) ? inputs.method :
-        typeof inputs.method === 'string' ?
-          inputs.method.split(',') : ['*']).map(x => x.toString().trim()),
       level: Object.keys(levels).includes(inputs.level) ? inputs.level : 'trace'
     }
   }
 
+  // Init ruleMap
+  let rulesMap = {}
+
   // Parse and default rules
-  let rules = Array.isArray(cfg.rules) ? cfg.rules.reduce((acc,rule) => {
+  let rules = Array.isArray(cfg.rules) ? cfg.rules.map((rule,i) => {
     // Error if missing route or not a string
     if (!rule.route || typeof rule.route !== 'string')
       throw new Error('Invalid route specified in rule')
 
-    return Object.assign(acc, { [rule.route.trim()]: defaults(rule) })
+    // Parse methods into array (if not already)
+    let methods = (Array.isArray(rule.method) ? rule.method :
+      typeof rule.method === 'string' ?
+        rule.method.split(',') : ['ANY']).map(x => x.toString().trim().toUpperCase())
+
+    let map = {}
+    let recursive = map // create recursive reference
+
+    //rule.route.replace(/^\//,'').split('/').forEach(part => {
+    UTILS.parsePath(rule.route).forEach(part => {
+      Object.assign(recursive,{ [part === '' ? '/' : part]: {} })
+      recursive = recursive[part === '' ? '/' : part]
+    })
+
+    Object.assign(recursive, methods.reduce((acc,method) => {
+      return Object.assign(acc, { ['__'+method]: i })
+    },{}))
+
+    // Deep merge the maps
+    UTILS.deepMerge(rulesMap,map)
+
+    return defaults(rule)
   },{}) : {}
 
+  // console.log(JSON.stringify(rulesMap,null,2));
+
   return {
     defaults: Object.assign(defaults(cfg),{ default:true }),
-    rules
+    rules,
+    rulesMap
   }
 
 } // end parseSamplerConfig
diff --git a/lib/request.js b/lib/request.js
@@ -114,7 +114,8 @@ class REQUEST {
     }
 
     // Extract path from event (strip querystring just in case)
-    let path = this.app._event.path.trim().split('?')[0].replace(/^\/(.*?)(\/)*$/,'$1').split('/')
+    // let path2 = this.app._event.path.trim().split('?')[0].replace(/^\/(.*?)(\/)*$/,'$1').split('/')
+    let path = UTILS.parsePath(this.app._event.path)
 
     // Init the route
     this.route = null

diff --git a/lib/utils.js b/lib/utils.js
@@ -17,15 +17,15 @@ const entityMap = {
   '\'': '&#39;'
 }
 
-module.exports.escapeHtml = html => html.replace(/[&<>"']/g, s => entityMap[s])
+exports.escapeHtml = html => html.replace(/[&<>"']/g, s => entityMap[s])
 
 
 // From encodeurl by Douglas Christopher Wilson
 let ENCODE_CHARS_REGEXP = /(?:[^\x21\x25\x26-\x3B\x3D\x3F-\x5B\x5D\x5F\x61-\x7A\x7E]|%(?:[^0-9A-Fa-f]|[0-9A-Fa-f][^0-9A-Fa-f]|$))+/g
 let UNMATCHED_SURROGATE_PAIR_REGEXP = /(^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF]([^\uDC00-\uDFFF]|$)/g
 let UNMATCHED_SURROGATE_PAIR_REPLACE = '$1\uFFFD$2'
 
-module.exports.encodeUrl = url => String(url)
+exports.encodeUrl = url => String(url)
   .replace(UNMATCHED_SURROGATE_PAIR_REGEXP, UNMATCHED_SURROGATE_PAIR_REPLACE)
   .replace(ENCODE_CHARS_REGEXP, encodeURI)
 
@@ -34,11 +34,14 @@ module.exports.encodeUrl = url => String(url)
 const encodeBody = body =>
   typeof body === 'object' ? JSON.stringify(body) : (body && typeof body !== 'string' ? body.toString() : (body ? body : ''))
 
-module.exports.encodeBody = encodeBody
+exports.encodeBody = encodeBody
 
+exports.parsePath = path => {
+  return path.trim().split('?')[0].replace(/^\/(.*?)(\/)*$/,'$1').split('/')
+}
 
 
-module.exports.parseBody = body => {
+exports.parseBody = body => {
   try {
     return JSON.parse(body)
   } catch(e) {
@@ -65,7 +68,7 @@ const parseAuthValue = (type,value) => {
   }
 }
 
-module.exports.parseAuth = authStr => {
+exports.parseAuth = authStr => {
   let auth = authStr && typeof authStr === 'string' ? authStr.split(' ') : []
   return auth.length > 1 && ['Bearer','Basic','Digest','OAuth'].includes(auth[0]) ?
     parseAuthValue(auth[0], auth.slice(1).join(' ').trim()) :
@@ -74,10 +77,9 @@ module.exports.parseAuth = authStr => {
 
 
 
-
 const mimeMap = require('./mimemap.js') // MIME Map
 
-module.exports.mimeLookup = (input,custom={}) => {
+exports.mimeLookup = (input,custom={}) => {
   let type = input.trim().replace(/^\./,'')
 
   // If it contains a slash, return unmodified
@@ -103,20 +105,29 @@ const extractRoutes = (routes,table=[]) => {
   return table
 }
 
-module.exports.extractRoutes = extractRoutes
+exports.extractRoutes = extractRoutes
 
 
 // Generate an Etag for the supplied value
-module.exports.generateEtag = data =>
+exports.generateEtag = data =>
   crypto.createHash('sha256').update(encodeBody(data)).digest('hex').substr(0,32)
 
 
 // Check if valid S3 path
-module.exports.isS3 = path => /^s3:\/\/.+\/.+/i.test(path)
+exports.isS3 = path => /^s3:\/\/.+\/.+/i.test(path)
+
 
 // Parse S3 path
-module.exports.parseS3 = path => {
+exports.parseS3 = path => {
   if (!this.isS3(path)) throw new Error('Invalid S3 path')
   let s3object = path.replace(/^s3:\/\//i,'').split('/')
   return { Bucket: s3object.shift(), Key: s3object.join('/') }
 }
+
+
+// Deep Merge
+exports.deepMerge = (a,b) => {
+  Object.keys(b).forEach(key => (key in a) ?
+    this.deepMerge(a[key],b[key]) : Object.assign(a,b) )
+  return a
+}