diff --git a/.travis.yml b/.travis.yml index 6552c4d..9db811b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ language: node_js +services: + - mongodb + before_install: - jdk_switcher use oraclejdk8 - java -version diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b6b840..0faf6e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# v0.2.0 +## Add +1. Saving request logging to mongodb. +2. Add the filed `req_id`. + # v0.1.1 ## Fix 1. Fixed the issue of broken when not given the default parameter. diff --git a/index.js b/index.js index 8d43981..55a9408 100644 --- a/index.js +++ b/index.js @@ -1,17 +1,19 @@ const slogger = require('node-slogger'); const serverIp = require('ip').address(); const pid = process.pid; +var req_id_count = 0; /** * @module req-log * @param {Object} options * @param {Object=} options.kafkaSchedule The instance of class KafkaProducer from the package of [queue-schedule](https://npmjs.com/package/queue-schedule). + * @param {Object=} options.mongooseModel The instance of a mongoose Model to save the request log. * @param {Object=} options.alarm The alarm object, it should has the function of sendAll. */ -module.exports = function(options={}) { - const {kafkaSchedule=null,alarm=null} = options; +module.exports = function({kafkaSchedule=null,mongooseModel=null,alarm=null}={}) { return function(req, res, next) { //记录请求时间 const req_time = Date.now(); + const req_id = req_id_count++; res.on('finish', function() { //记录响应时间 const now = Date.now(); @@ -31,7 +33,7 @@ module.exports = function(options={}) { const referer = req.get('referer') || ''; const session = req.session; - if (kafkaSchedule) { + if (kafkaSchedule || mongooseModel) { const data = { host, original_url, @@ -41,6 +43,7 @@ module.exports = function(options={}) { ip, duration, pid, + req_id, content_length, status_code, req_time, @@ -49,7 +52,16 @@ module.exports = function(options={}) { session, created_at: now }; - kafkaSchedule.addData(data); + if (kafkaSchedule) { + kafkaSchedule.addData(data); + } + if (mongooseModel) { + new mongooseModel(data).save(function(err) { + if (err) { + slogger.error('save request log to mongodb failed',err); + } + }); + } } if (alarm) { diff --git a/package.json b/package.json index 2b264bd..ae94578 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,13 @@ { "name": "@yunnysunny/request-logging", - "version": "0.1.1", + "version": "0.2.0", "description": "Print the express request log to console and save it to kafka when required, and even can send alram message when the response code greater than 500.", "main": "index.js", "scripts": { "test": "nyc mocha --recursive test/mocha --timeout 999999 --exit", "coverage": "nyc report --reporter=text-lcov | coveralls", "doc": "jsdoc2md index.js index.js > doc/api.md", - "release" : "git push && release-to-github-with-changelog" + "release": "git push && release-to-github-with-changelog" }, "repository": { "type": "git", @@ -39,6 +39,7 @@ "express-session": "^1.15.6", "jsdoc-to-markdown": "^4.0.1", "mocha": "^5.2.0", + "mongoose": "^5.3.3", "nyc": "^13.0.1", "queue-schedule": "^0.5.3", "supertest": "^3.3.0" diff --git a/readme.md b/readme.md index 209339c..5a8ed3d 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # request-logging -Print the express request log to console and save it to kafka when required, and even can send alram message when the response code greater than 500. +Print the express request log to console and save it to kafka and mongodb when required, and even can send alram message when the response code greater than 500. [![npm version][npm-image]][npm-url] [![build status][travis-image]][travis-url] @@ -50,6 +50,46 @@ app.use(bodyParser.urlencoded({ See [api](https://github.com/yunnysunny/request-log/blob/HEAD/doc/api.md) document. +## Fields + +If you want to save request logging to mongodb, this is the fields description, which you will used to create a mongoose schema: + +| name | type | description | +| -------------- | ------ | ------------------------------------------------------------ | +| host | String | The server's ip. | +| original_url | String | The original url contains query string. | +| path | String | The request path doesn't contain query string. | +| user_agent | String | The user agent. | +| method | String | The http request method. | +| ip | String | The client's ip. | +| duration | Number | The millisecond the request costed. | +| pid | Number | The server's process id. | +| req_id | Number | The inner request number, auto increased when new request come. | +| content_length | Number | The content-length of the response headers. | +| status_code | Number | The status code of current HTTP response. | +| req_time | Number | The timestamp of begin time of current request occured. | +| req_data | Object | The request data, which would form query string or form data. | +| referer | String | The HTTP referer header. | +| session | Object | The session of current request. | + +We suggest you use such mongoose schema, which is compatible when the fields is changed: + +```javascript +const {Schema} = require('mongoose'); + +const requestLogSchema = new Schema({ + req_time: Date +},{ + timestamps: { + createdAt: 'created_at', + updatedAt : 'updated_at' + }, + strict: false +}); +module.exports = requestLogSchema; +``` + + ## License [MIT](https://github.com/yunnysunny/request-log/blob/HEAD/LICENSE) diff --git a/test/express/config.example.json b/test/express/config.example.json index bf0c343..d3641bb 100644 --- a/test/express/config.example.json +++ b/test/express/config.example.json @@ -4,5 +4,9 @@ "kafkaConfig":{ "topic":"req-log", "peers":"127.0.0.1:9092" + }, + "mongoConfig" : { + "option" : {"useMongoClient": true}, + "url" : "mongodb://localhost/test" } } \ No newline at end of file diff --git a/test/express/src/app-with-mongodb.js b/test/express/src/app-with-mongodb.js new file mode 100644 index 0000000..47c9fb8 --- /dev/null +++ b/test/express/src/app-with-mongodb.js @@ -0,0 +1,54 @@ +const express = require('express'); +const path = require('path'); +const bodyParser = require('body-parser'); + +const routes = require('./routes/index'); + +const { + slogger, + port, + requestLogModel +} = require('./config'); +const requestLog = require('../../../index'); + +const app = express(); +app.enable('trust proxy'); + +// view engine setup +app.set('port', port); +app.use(requestLog({mongooseModel:requestLogModel})); + +app.use(bodyParser.json({limit: '1mb'})); +app.use(bodyParser.urlencoded({ + extended: false, + limit: '1mb' +})); + + +app.use(express.static(path.join(__dirname, 'public'))); + +app.use('/', routes); + +// catch 404 and forward to error handler +app.use(function(req, res, next) { + const err = new Error('Not Found:' + req.path); + err.status = 404; + next(err); +}); + +// error handlers +app.use(function(err, req, res, next) { + const status = err.status; + if (status === 404) { + return res.status(404).send(err.message || '未知异常'); + } + res.status(status || 500); + slogger.error('发现应用未捕获异常', err); + res.send({ + msg: err.message || '未知异常', + code: 0xffff + }); +}); + + +module.exports = app; diff --git a/test/express/src/config.js b/test/express/src/config.js index 25d5e6e..f1ea63f 100644 --- a/test/express/src/config.js +++ b/test/express/src/config.js @@ -1,5 +1,6 @@ const slogger = require('node-slogger'); const {KafkaProducer} = require('queue-schedule'); +const mongoose = require('mongoose'); const configObj = require('../config.json'); const settings = require('config-settings').init(configObj); @@ -25,6 +26,7 @@ exports.kafkaSchedule = new KafkaProducer({ delayInterval:1000, kafkaHost:kafkaHost }); -// let mongoConfig = settings.loadNecessaryObject('mongoConfig'); -// mongoose.Promise = global.Promise; -// mongoose.connect(mongoConfig.url, mongoConfig.option); // connect to database +let mongoConfig = settings.loadNecessaryObject('mongoConfig'); +mongoose.Promise = global.Promise; +mongoose.connect(mongoConfig.url, mongoConfig.option); // connect to database +exports.requestLogModel = mongoose.model('RequestLog',require('./schemas/request_log_schema')); \ No newline at end of file diff --git a/test/express/src/schemas/request_log_schema.js b/test/express/src/schemas/request_log_schema.js new file mode 100644 index 0000000..f65c90e --- /dev/null +++ b/test/express/src/schemas/request_log_schema.js @@ -0,0 +1,16 @@ +const {Schema} = require('mongoose'); + +const requestLogSchema = new Schema({ + req_time: Date +},{ + autoIndex: false, + timestamps: { + createdAt: 'created_at', + updatedAt : 'update_at' + }, + strict: false +}); + + + +module.exports = requestLogSchema; \ No newline at end of file diff --git a/test/mocha/mongodb_test.js b/test/mocha/mongodb_test.js new file mode 100644 index 0000000..8d27f85 --- /dev/null +++ b/test/mocha/mongodb_test.js @@ -0,0 +1,34 @@ +const app = require('../express/src/app-with-mongodb'); + +const request = require('supertest'); +const {expect} = require('chai'); +const rand = Math.random(); +const reqUrl = '/?rand='+rand; +const {requestLogModel} = require('../express/src/config'); +describe('mongodb test:',function() { + it('sucess when request / ok',function(done) { + request(app) + .get(reqUrl) + .expect(200) + .end(function(err) { + if (err) { + return done(err); + } + done(); + }); + }); + it('the lastest url is ' + reqUrl,function(done) { + requestLogModel.findOne({},{original_url:1},{ + sort:{_id:-1},lean:true + },function(err,item) { + if (err) { + return done(err); + } + if (!item) { + return done('save to mongo failed'); + } + expect(item.original_url).equal(reqUrl); + done(); + }); + }); +}); \ No newline at end of file