Skip to content

Commit

Permalink
Add pino.multistream
Browse files Browse the repository at this point in the history
  • Loading branch information
mcollina committed Apr 7, 2021
1 parent cfd191a commit 8af6198
Show file tree
Hide file tree
Showing 7 changed files with 884 additions and 19 deletions.
98 changes: 98 additions & 0 deletions benchmarks/multistream.js
@@ -0,0 +1,98 @@
'use strict'

const bench = require('fastbench')
const bunyan = require('bunyan')
const pino = require('../')
const fs = require('fs')
const dest = fs.createWriteStream('/dev/null')

const tenStreams = [
{ stream: dest },
{ stream: dest },
{ stream: dest },
{ stream: dest },
{ stream: dest },
{ level: 'debug', stream: dest },
{ level: 'debug', stream: dest },
{ level: 'trace', stream: dest },
{ level: 'warn', stream: dest },
{ level: 'fatal', stream: dest }
]
const pinomsTen = pino({ level: 'debug' }, pino.multistream(tenStreams))

const fourStreams = [
{ stream: dest },
{ stream: dest },
{ level: 'debug', stream: dest },
{ level: 'trace', stream: dest }
]
const pinomsFour = pino({ level: 'debug' }, pino.multistream(fourStreams))

const pinomsOne = pino({ level: 'info' }, pino.multistream(dest))
const blogOne = bunyan.createLogger({
name: 'myapp',
streams: [{ stream: dest }]
})

const blogTen = bunyan.createLogger({
name: 'myapp',
streams: tenStreams
})
const blogFour = bunyan.createLogger({
name: 'myapp',
streams: fourStreams
})

const max = 10
const run = bench([
function benchBunyanTen (cb) {
for (let i = 0; i < max; i++) {
blogTen.info('hello world')
blogTen.debug('hello world')
blogTen.trace('hello world')
blogTen.warn('hello world')
blogTen.fatal('hello world')
}
setImmediate(cb)
},
function benchPinoMSTen (cb) {
for (let i = 0; i < max; i++) {
pinomsTen.info('hello world')
pinomsTen.debug('hello world')
pinomsTen.trace('hello world')
pinomsTen.warn('hello world')
pinomsTen.fatal('hello world')
}
setImmediate(cb)
},
function benchBunyanFour (cb) {
for (let i = 0; i < max; i++) {
blogFour.info('hello world')
blogFour.debug('hello world')
blogFour.trace('hello world')
}
setImmediate(cb)
},
function benchPinoMSFour (cb) {
for (let i = 0; i < max; i++) {
pinomsFour.info('hello world')
pinomsFour.debug('hello world')
pinomsFour.trace('hello world')
}
setImmediate(cb)
},
function benchBunyanOne (cb) {
for (let i = 0; i < max; i++) {
blogOne.info('hello world')
}
setImmediate(cb)
},
function benchPinoMSOne (cb) {
for (let i = 0; i < max; i++) {
pinomsOne.info('hello world')
}
setImmediate(cb)
}
], 10000)

run()
68 changes: 68 additions & 0 deletions docs/api.md
Expand Up @@ -24,6 +24,7 @@
* [Statics](#statics)
* [pino.destination()](#pino-destination)
* [pino.final()](#pino-final)
* [pino.multistream()](#pino-multistream)
* [pino.stdSerializers](#pino-stdserializers)
* [pino.stdTimeFunctions](#pino-stdtimefunctions)
* [pino.symbols](#pino-symbols)
Expand Down Expand Up @@ -938,6 +939,73 @@ finalLogger.info('exiting...')
* See [Asynchronous logging ⇗](/docs/asynchronous.md)
* See [Log loss prevention ⇗](/docs/asynchronous.md#log-loss-prevention)

<a id="pino-multistream"></a>

### `pino.multistream(options) => Stream`

Create a stream composed by multiple destination streams:

```js
var fs = require('fs')
var pino = require('pino')
var streams = [
{stream: fs.createWriteStream('/tmp/info.stream.out')},
{level: 'debug', stream: fs.createWriteStream('/tmp/debug.stream.out')},
{level: 'fatal', stream: fs.createWriteStream('/tmp/fatal.stream.out')}
]

var log = pino({
level: 'debug' // this MUST be set at the lowest level of the
// destinations
}, pino.multistream(streams))

log.debug('this will be written to /tmp/debug.stream.out')
log.info('this will be written to /tmp/debug.stream.out and /tmp/info.stream.out')
log.fatal('this will be written to /tmp/debug.stream.out, /tmp/info.stream.out and /tmp/fatal.stream.out')
```

In order for `multistream` to work, the log level _____must__ be set to the lowest level used in the streams array.

#### Options

* `levels`: Pass custom log level definitions to the instance as an object.

+ `dedupe`: Set this to `true` to send logs only to the stream with the higher level. Default: `false`

`dedupe` flag can be useful for example when using pino-multi-stream to redirect `error` logs to `process.stderr` and others to `process.stdout`:

```js
var pino = require('pino')
var multistream = require('pino-multi-stream').multistream
var streams = [
{stream: process.stdout},
{level: 'error', stream: process.stderr},
]

var opts = {
levels: {
silent: Infinity,
fatal: 60,
error: 50,
warn: 50,
info: 30,
debug: 20,
trace: 10
},
dedupe: true,
}

var log = pino({
level: 'debug' // this MUST be set at the lowest level of the
// destinations
}, multistream(streams, opts))

log.debug('this will be written ONLY to process.stdout')
log.info('this will be written ONLY to process.stdout')
log.error('this will be written ONLY to process.stderr')
log.fatal('this will be written ONLY to process.stderr')
```

<a id="pino-stdserializers"></a>
### `pino.stdSerializers` (Object)

Expand Down
23 changes: 4 additions & 19 deletions docs/help.md
Expand Up @@ -118,20 +118,7 @@ Given a similar scenario as in the [Log rotation](#rotate) section a basic
<a id="multiple"></a>
## Saving to multiple files

Let's assume we want to store all error messages to a separate log file.

Install [pino-tee](https://npm.im/pino-tee) with:

```bash
npm i pino-tee -g
```

The following writes the log output of `app.js` to `./all-logs`, while
writing only warnings and errors to `./warn-log:

```bash
node app.js | pino-tee warn ./warn-logs > ./all-logs
```
See [`pino.multistream`](/doc/api.md#pino-multistream).

<a id="filter-logs"></a>
## Log Filtering
Expand Down Expand Up @@ -164,14 +151,13 @@ ExecStart=/bin/sh -c '/path/to/node app.js | pino-transport'

Pino's default log destination is the singular destination of `stdout`. While
not recommended for performance reasons, multiple destinations can be targeted
by using [`pino-multi-stream`](https://github.com/pinojs/pino-multi-stream).
by using [`pino.multistream`](/doc/api.md#pino-multistream).

In this example we use `stderr` for `error` level logs and `stdout` as default
for all other levels (e.g. `debug`, `info`, and `warn`).

```js
const pino = require('pino')
const { multistream } = require('pino-multi-stream')
var streams = [
{level: 'debug', stream: process.stdout},
{level: 'error', stream: process.stderr},
Expand All @@ -180,11 +166,10 @@ var streams = [

const logger = pino({
name: 'my-app',
level: 'info',
}, multistream(streams))
level: 'debug', // must be the lower level of all streams
}, pino.multistream(streams))
```


<a id="dupe-keys"></a>
## How Pino handles duplicate keys

Expand Down
138 changes: 138 additions & 0 deletions lib/multistream.js
@@ -0,0 +1,138 @@
'use strict'

const metadata = Symbol.for('pino.metadata')

const defaultLevels = {
silent: Infinity,
fatal: 60,
error: 50,
warn: 40,
info: 30,
debug: 20,
trace: 10
}

function multistream (streamsArray, opts) {
let counter = 0

streamsArray = streamsArray || []
opts = opts || { dedupe: false }

let levels = defaultLevels
if (opts.levels && typeof opts.levels === 'object') {
levels = opts.levels
}

const res = {
write,
add,
flushSync,
minLevel: 0,
streams: [],
clone,
[metadata]: true
}

if (Array.isArray(streamsArray)) {
streamsArray.forEach(add, res)
} else {
add.call(res, streamsArray)
}

// clean this object up
// or it will stay allocated forever
// as it is closed on the following closures
streamsArray = null

return res

// we can exit early because the streams are ordered by level
function write (data) {
let dest
const level = this.lastLevel
const { streams } = this
let stream
for (let i = 0; i < streams.length; i++) {
dest = streams[i]
if (dest.level <= level) {
stream = dest.stream
if (stream[metadata]) {
const { lastTime, lastMsg, lastObj, lastLogger } = this
stream.lastLevel = level
stream.lastTime = lastTime
stream.lastMsg = lastMsg
stream.lastObj = lastObj
stream.lastLogger = lastLogger
}
if (!opts.dedupe) {
stream.write(data)
}
} else {
break
}
}

if (opts.dedupe && stream) {
stream.write(data)
}
}

function flushSync () {
for (const { stream } of this.streams) {
if (typeof stream.flushSync === 'function') {
stream.flushSync()
}
}
}

function add (dest) {
const { streams } = this
if (typeof dest.write === 'function') {
return add.call(this, { stream: dest })
} else if (typeof dest.levelVal === 'number') {
return add.call(this, Object.assign({}, dest, { level: dest.levelVal, levelVal: undefined }))
} else if (typeof dest.level === 'string') {
return add.call(this, Object.assign({}, dest, { level: levels[dest.level] }))
} else if (typeof dest.level !== 'number') {
// we default level to 'info'
dest = Object.assign({}, dest, { level: 30 })
} else {
dest = Object.assign({}, dest)
}
dest.id = counter++

streams.unshift(dest)
streams.sort(compareByLevel)

this.minLevel = streams[0].level

return res
}

function clone (level) {
const streams = new Array(this.streams.length)

for (let i = 0; i < streams.length; i++) {
streams[i] = {
level: level,
stream: this.streams[i].stream
}
}

return {
write,
add,
minLevel: level,
streams,
clone,
flushSync,
[metadata]: true
}
}
}

function compareByLevel (a, b) {
return a.level - b.level
}

module.exports = multistream

0 comments on commit 8af6198

Please sign in to comment.