saving request logging to mongodb

.travis.yml

@@ -1,5 +1,8 @@
 language: node_js
 
+services:
+  - mongodb
+
 before_install:
   - jdk_switcher use oraclejdk8
   - java -version

CHANGELOG.md

@@ -1,3 +1,8 @@
+# v0.2.0
+## Add
+1. Saving request logging to mongodb.
+2. Add the filed `req_id`.
+
 # v0.1.1
 ## Fix
 1. Fixed the issue of broken when not given the default parameter.

index.js

@@ -1,17 +1,19 @@
 const slogger = require('node-slogger');
 const serverIp = require('ip').address();
 const pid = process.pid;
+var req_id_count = 0;
 /**
  * @module req-log
  * @param {Object} options
  * @param {Object=} options.kafkaSchedule The instance of class KafkaProducer from the package of [queue-schedule](https://npmjs.com/package/queue-schedule). 
+ * @param {Object=} options.mongooseModel The instance of a mongoose Model to save the request log.
  * @param {Object=} options.alarm The alarm object, it should has the function of sendAll.
  */
-module.exports = function(options={}) {
-    const {kafkaSchedule=null,alarm=null} = options;
+module.exports = function({kafkaSchedule=null,mongooseModel=null,alarm=null}={}) {
     return function(req, res, next) {
         //记录请求时间
         const req_time = Date.now();
+        const req_id = req_id_count++;
         res.on('finish', function() {
             //记录响应时间
             const now = Date.now();
@@ -31,7 +33,7 @@ module.exports = function(options={}) {
             const referer = req.get('referer') || '';
             const session = req.session;
 
-            if (kafkaSchedule) {
+            if (kafkaSchedule || mongooseModel) {
                 const data = {
                     host,
                     original_url,
@@ -41,6 +43,7 @@ module.exports = function(options={}) {
                     ip,
                     duration,
                     pid,
+                    req_id,
                     content_length,
                     status_code,
                     req_time,
@@ -49,7 +52,16 @@ module.exports = function(options={}) {
                     session,
                     created_at: now
                 };
-                kafkaSchedule.addData(data);
+                if (kafkaSchedule) {
+                    kafkaSchedule.addData(data);
+                }
+                if (mongooseModel) {
+                    new mongooseModel(data).save(function(err) {
+                        if (err) {
+                            slogger.error('save request log to mongodb failed',err);
+                        }
+                    });
+                }
             }
 
             if (alarm) {

package.json

@@ -1,13 +1,13 @@
 {
   "name": "@yunnysunny/request-logging",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Print the express request log to console and save it to kafka when required, and even can send alram message when the response code greater than 500.",
   "main": "index.js",
   "scripts": {
     "test": "nyc mocha --recursive test/mocha --timeout 999999 --exit",
     "coverage": "nyc report --reporter=text-lcov | coveralls",
     "doc": "jsdoc2md index.js index.js > doc/api.md",
-    "release" : "git push && release-to-github-with-changelog"
+    "release": "git push && release-to-github-with-changelog"
   },
   "repository": {
     "type": "git",
@@ -39,6 +39,7 @@
     "express-session": "^1.15.6",
     "jsdoc-to-markdown": "^4.0.1",
     "mocha": "^5.2.0",
+    "mongoose": "^5.3.3",
     "nyc": "^13.0.1",
     "queue-schedule": "^0.5.3",
     "supertest": "^3.3.0"

readme.md

@@ -1,6 +1,6 @@
 # request-logging
 
-Print the express request log to console and save it to kafka when required, and even can send alram message when the response code greater than 500.
+Print the express request log to console and save it to kafka and mongodb when required, and even can send alram message when the response code greater than 500.
 
 [![npm version][npm-image]][npm-url]
 [![build status][travis-image]][travis-url]
@@ -50,6 +50,46 @@ app.use(bodyParser.urlencoded({
 
 See [api](https://github.com/yunnysunny/request-log/blob/HEAD/doc/api.md) document.
 
+## Fields
+
+If you want to save request logging to mongodb, this is the fields description, which you will used to create a mongoose schema:  
+
+| name           | type   | description                                                  |
+| -------------- | ------ | ------------------------------------------------------------ |
+| host           | String | The server's ip.                                             |
+| original_url   | String | The original url contains query string.                      |
+| path           | String | The request path doesn't contain query string.               |
+| user_agent     | String | The user agent.                                              |
+| method         | String | The http request method.                                     |
+| ip             | String | The client's ip.                                             |
+| duration       | Number | The millisecond the request costed.                          |
+| pid            | Number | The server's process id.                                     |
+| req_id         | Number | The inner request number, auto increased when new request come. |
+| content_length | Number | The content-length of the response headers.                  |
+| status_code    | Number | The status code of current HTTP response.                    |
+| req_time       | Number | The timestamp of begin time of current request occured.      |
+| req_data       | Object | The request data, which would form query string or form data. |
+| referer        | String | The HTTP referer header.                                     |
+| session        | Object | The session of current request.                              |
+
+We suggest you use such mongoose schema, which is compatible when the fields is changed:
+
+```javascript
+const {Schema} = require('mongoose');
+
+const requestLogSchema =  new Schema({
+    req_time: Date
+},{ 
+    timestamps: {
+        createdAt: 'created_at',
+        updatedAt : 'updated_at'
+    },
+    strict: false
+});
+module.exports = requestLogSchema;
+```
+
+
 ## License
 
 [MIT](https://github.com/yunnysunny/request-log/blob/HEAD/LICENSE)

test/express/config.example.json

@@ -4,5 +4,9 @@
     "kafkaConfig":{
         "topic":"req-log",
         "peers":"127.0.0.1:9092"
+    },
+    "mongoConfig" : {
+        "option" : {"useMongoClient": true},
+        "url" : "mongodb://localhost/test"
     }
 }

test/express/src/app-with-mongodb.js

@@ -0,0 +1,54 @@
+const express = require('express');
+const path = require('path');
+const bodyParser = require('body-parser');
+
+const routes = require('./routes/index');
+
+const {
+    slogger,
+    port,
+    requestLogModel
+} = require('./config');
+const requestLog = require('../../../index');
+
+const app = express();
+app.enable('trust proxy');
+
+// view engine setup
+app.set('port', port);
+app.use(requestLog({mongooseModel:requestLogModel}));
+
+app.use(bodyParser.json({limit: '1mb'}));
+app.use(bodyParser.urlencoded({
+    extended: false,
+    limit: '1mb'
+}));
+
+
+app.use(express.static(path.join(__dirname, 'public')));
+
+app.use('/', routes);
+
+// catch 404 and forward to error handler
+app.use(function(req, res, next) {
+    const err = new Error('Not Found:' + req.path);
+    err.status = 404;
+    next(err);
+});
+
+// error handlers
+app.use(function(err, req, res, next) {
+    const status = err.status;
+    if (status === 404) {
+        return res.status(404).send(err.message || '未知异常');
+    }
+    res.status(status || 500);
+    slogger.error('发现应用未捕获异常', err);
+    res.send({
+        msg: err.message || '未知异常',
+        code: 0xffff
+    });
+});
+
+
+module.exports = app;

test/express/src/config.js

@@ -1,5 +1,6 @@
 const slogger = require('node-slogger');
 const {KafkaProducer} = require('queue-schedule');
+const mongoose = require('mongoose');
 const configObj = require('../config.json');
 const settings = require('config-settings').init(configObj);
 
@@ -25,6 +26,7 @@ exports.kafkaSchedule = new KafkaProducer({
     delayInterval:1000,
     kafkaHost:kafkaHost
 });
-// let mongoConfig = settings.loadNecessaryObject('mongoConfig');
-// mongoose.Promise = global.Promise;
-// mongoose.connect(mongoConfig.url, mongoConfig.option); // connect to database
+let mongoConfig = settings.loadNecessaryObject('mongoConfig');
+mongoose.Promise = global.Promise;
+mongoose.connect(mongoConfig.url, mongoConfig.option); // connect to database
+exports.requestLogModel = mongoose.model('RequestLog',require('./schemas/request_log_schema'));

test/express/src/schemas/request_log_schema.js

@@ -0,0 +1,16 @@
+const {Schema} = require('mongoose');
+
+const requestLogSchema =  new Schema({
+    req_time: Date
+},{ 
+    autoIndex: false,
+    timestamps: {
+        createdAt: 'created_at',
+        updatedAt : 'update_at'
+    },
+    strict: false
+});
+
+
+
+module.exports = requestLogSchema;

test/mocha/mongodb_test.js

@@ -0,0 +1,34 @@
+const app = require('../express/src/app-with-mongodb');
+
+const request = require('supertest');
+const {expect} = require('chai');
+const rand = Math.random();
+const reqUrl = '/?rand='+rand;
+const {requestLogModel} = require('../express/src/config');
+describe('mongodb test:',function() {
+    it('sucess when request / ok',function(done) {
+        request(app)
+            .get(reqUrl)
+            .expect(200)
+            .end(function(err) {
+                if (err) {
+                    return done(err);
+                }
+                done();
+            });
+    });
+    it('the lastest url is ' + reqUrl,function(done) {
+        requestLogModel.findOne({},{original_url:1},{
+            sort:{_id:-1},lean:true
+        },function(err,item) {
+            if (err) {
+                return done(err);
+            }
+            if (!item) {
+                return done('save to mongo failed');
+            }
+            expect(item.original_url).equal(reqUrl);
+            done();
+        });
+    });
+});