Skip to content

Commit 0b5f164

Browse files
authoredAug 25, 2016
Improvements...
* Buffer messages to avoid quota overruns, use object streams, use GCP-standard option names, remove custom formatter, clean up... * Allow projectId via env var. Use strict JSON. * Update example.js * Update README.md * update to google-cloud/logging dep See full change description in #1
1 parent fecc373 commit 0b5f164

File tree

4 files changed

+154
-130
lines changed

4 files changed

+154
-130
lines changed
 

‎README.md

+45-43
Original file line numberDiff line numberDiff line change
@@ -4,81 +4,83 @@
44

55
## Installation
66

7-
First install bunyan...
7+
Install bunyan and bunyan-stackdriver...
88

99
```bash
10-
npm install bunyan
11-
```
12-
13-
Then install bunyan-stackdriver
14-
15-
```bash
16-
npm install bunyan-stackdriver
10+
npm install bunyan bunyan-stackdriver
1711
```
1812

1913
## Setup
2014

21-
1. Enable [Google Cloud Logging API](https://console.cloud.google.com/apis/api/logging.googleapis.com/overview) in your Google Developer Console.
22-
2. [Create a new Client ID](https://console.cloud.google.com/apis/credentials) for a Service Account (JSON Key) and download it.
23-
3. Start using `bunyan-stackdriver` to create log your messages
15+
1. Enable [Google Cloud Logging API](https://console.cloud.google.com/apis/api/logging.googleapis.com/overview)
16+
in your Google Developer Console.
17+
1. Start using `bunyan-stackdriver` to create log your messages
2418

2519
## Basic usage
2620

2721
```javascript
2822
var bunyan = require("bunyan"),
29-
BunyanStackDriver = require('bunyan-stackdriver'),
23+
BunyanStackDriver = require("bunyan-stackdriver"),
3024
log;
3125

3226
log = bunyan.createLogger({
3327
name: "myApp",
34-
stream: new BunyanStackDriver({
35-
authJSON: require("./your-JSON-key.json"),
36-
project: "your_project_id",
37-
log_id: "default"
38-
}),
28+
streams: [{
29+
type: "raw", // faster; makes Bunyan send objects instead of stringifying messages
30+
stream: new BunyanStackDriver({
31+
projectId: "your_project_id"
32+
})
33+
}],
3934
level: "error"
4035
});
4136

4237
log.error("hello bunyan user");
4338
```
4439

45-
You can also pass an optional error handler.
40+
## Full options
4641

4742
```javascript
4843
new BunyanStackDriver({
49-
authJSON: require("./your-JSON-key.json"),
50-
project: "your_project_id",
51-
log_id: "default"
52-
}, function(error){
53-
console.log(error);
54-
});
44+
keyFilename: "/path/to/keyfile.json",
45+
logName: "logname",
46+
projectId: "project-id",
47+
writeInterval: 500, // ms
48+
resource: {
49+
type: "resource_type",
50+
labels: {key1: value1}
51+
}
52+
}, function errorCallback(err) { console.log(err); });
5553
```
5654

57-
##Custom Formatters
55+
* If you are running on Google Cloud Platform, authentication will be taken
56+
care of automatically. If you're running elsewhere, or wish to provide
57+
alternative authentication, you can specify the `keyFilename` pointing to a
58+
service account JSON key.
5859

59-
By default the logs are formatted like so: `[LOG_LEVEL] message`, unless you specify a `customFormatter` function.
60+
* logName. Must be less than 512 characters and include only alphanumeric
61+
characters, forward-slash, underscore, hyphen and period.
6062

61-
```javascript
62-
log = bunyan.createLogger({
63-
name: "myApp",
64-
stream: new BunyanStackDriver({
65-
authJSON: require("./your-JSON-key.json"),
66-
project: "your_project_id",
67-
log_id: "default"
68-
customFormatter: function(record, levelName){
69-
return {text: "[" + levelName + "] " + record.msg }
70-
}
71-
}),
72-
level: "error"
73-
});
74-
```
63+
* projectId. The id of the project. This can be omitted if the environment
64+
variable "GCLOUD_PROJECT" is set.
7565

76-
## Links
66+
* writeInterval. Specifies the maximum write frequency. Messages will be
67+
batched between writes to avoid exceeding API rate limits. (The default GCP
68+
limit is 20 RPS. The default setting for BunyanStackDriver is 500 ms.)
7769

78-
[Stackdriver Logging - Method entries.write](https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/entries/write)
70+
* resource. See https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/MonitoredResource.
71+
72+
* errorCallback. Will report errors during the logging process itself.
7973

80-
[Google Cloud Logging API beta nodejs client source code](https://github.com/google/google-api-nodejs-client/blob/master/apis/logging/v2beta1.js)
74+
## Known issues
8175

76+
* Circular objects will cause a stack overflow. If you need to log object with
77+
circular references, either (a) preprocess them to remove the circles, or (b)
78+
do not use the `type: "raw"` setting and instead let Bunyan's stringifier
79+
remove circular references.
80+
81+
## Links
82+
83+
[Stackdriver Logging - Method entries.write](https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/entries/write)
8284

8385
## License
8486

‎example.js

+6-5
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ var bunyan = require("bunyan"),
44

55
log = bunyan.createLogger({
66
name: "myApp",
7-
stream: new BunyanStackDriver({
8-
authJSON: require("./your-JSON-key.json"),
9-
project: "your_project_id",
10-
log_id: "default"
11-
}),
7+
streams: [{
8+
type: "raw",
9+
stream: new BunyanStackDriver({
10+
projectId: "your_project_id"
11+
})
12+
}],
1213
level: "error"
1314
});
1415

‎lib/bunyan-stackdriver.js

+99-79
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,6 @@
1-
var util = require('util'),
2-
google = require('googleapis');
3-
4-
var glogging = google.logging("v2beta1");
5-
6-
const loggingScopes = [
7-
//'https://www.googleapis.com/auth/logging.read',
8-
'https://www.googleapis.com/auth/logging.write',
9-
//'https://www.googleapis.com/auth/logging.admin',
10-
//'https://www.googleapis.com/auth/cloud-platform'
11-
];
1+
var util = require('util');
2+
var logging = require('@google-cloud/logging');
3+
var Writable = require('stream').Writable;
124

135
const nameFromLevel = {
146
10: 'trace',
@@ -24,98 +16,126 @@ const mapLevelToSeverity = {
2416
info: 'INFO',
2517
warn: 'WARNING',
2618
error: 'ERROR',
27-
fatal: 'EMERGENCY'
28-
}
29-
30-
function getNow() {
31-
var d = new Date();
32-
return JSON.parse(JSON.stringify(d).replace('Z', '000Z'));
33-
}
19+
fatal: 'ALERT'
20+
};
3421

22+
util.inherits(BunyanStackDriver, Writable);
3523
function BunyanStackDriver(options, error) {
24+
Writable.call(this, {objectMode: true});
25+
3626
options = options || {};
37-
if (!options.project) {
38-
throw new Error('Project cannot be null');
27+
28+
var gopts = {};
29+
30+
if (options.projectId) {
31+
gopts.projectId = options.projectId;
32+
} else if (process.env.GCLOUD_PROJECT) {
33+
// Noop, gcloud will pick this up.
3934
} else {
35+
throw new Error('option "projectId" or env variable GCLOUD_PROJECT required');
36+
}
4037

41-
this.customFormatter = options.customFormatter;
42-
this.project_id = options.project;
43-
this.log_id = options.log_id || 'default';
44-
this.error = error || function() {};
38+
var logName = options.logName || 'default';
4539

46-
// https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/LogEntry#LogSeverity
47-
if (options.severity) {
48-
this.severity = options.severity || 'DEFAULT';
49-
}
40+
this.error = error || function() {};
5041

51-
// object(MonitoredResource)
52-
// https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/MonitoredResource
53-
this.resource = options.resource || {type: 'global'};
42+
// ms. GCP's default limit is 20 RPS.
43+
this.writeInterval = 'writeInterval' in options ? options.writeInterval : 500;
5444

45+
// object(MonitoredResource)
46+
// https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/MonitoredResource
47+
if (options.resource) {
48+
if (options.resource.type) {
49+
options.resource.labels = options.resource.labels || {}; // required
50+
this.resource = options.resource;
51+
} else {
52+
throw new Error('Property "type" required when specifying a resource');
53+
}
54+
} else {
55+
this.resource = {
56+
type: 'global',
57+
labels: {}
58+
};
5559
}
5660

57-
this.getLoggingClient = function (callback) {
58-
google.auth.fromJSON(options.authJSON, function (err, authClient){
59-
if (err) {
60-
return callback(err);
61-
}
62-
if (authClient.createScopedRequired && authClient.createScopedRequired()) {
63-
authClient = authClient.createScoped(loggingScopes);
64-
}
65-
callback(null, authClient);
66-
});
61+
// If not provided, gcloud will attempt automatic auth.
62+
if (options.keyFilename) {
63+
gopts.keyFilename = options.authJSON;
6764
}
65+
66+
var loggingClient = logging(gopts);
67+
68+
this.log = loggingClient.log(logName);
69+
70+
this.entryQueue = [];
6871
}
6972

70-
BunyanStackDriver.prototype.write = function write(record, callback) {
71-
var self = this,
72-
levelName,
73-
message;
73+
var once = true;
7474

75+
BunyanStackDriver.prototype._write = function write(record, encoding, callback) {
76+
var timestamp;
7577
if (typeof record === 'string') {
78+
if (once) {
79+
once = false;
80+
console.warn('BunyanStackDriver: use "streams: [ type: \"raw\", stream: new BunyanStackDriver(...) ]" for better performance.');
81+
}
7682
record = JSON.parse(record);
83+
timestamp = new Date(record.time);
84+
} else {
85+
timestamp = record.time;
7786
}
7887

79-
levelName = nameFromLevel[record.level];
88+
strictJSON(record);
8089

81-
try {
90+
var entry = this.log.entry(this.resource, record);
91+
92+
// There are no public APIs for this yet:
93+
// https://github.com/GoogleCloudPlatform/gcloud-node/issues/1348
94+
entry.timestamp = timestamp;
95+
entry.severity = mapLevelToSeverity[nameFromLevel[record.level]] || 'DEFAULT';
96+
97+
this.entryQueue.push(entry);
98+
99+
if (!this.writeQueued) {
100+
this.writeQueued = true;
101+
setTimeout(this._writeToServer.bind(this), this.writeInterval);
102+
}
82103

83-
if(self.customFormatter){
84-
message = self.customFormatter(record, levelName);
85-
}else if(typeof(record) == "string"){
86-
message = { text: util.format('[%s] %s', levelName.toUpperCase(), record.msg)}
87-
}else{
88-
message = record;
104+
callback();
105+
};
106+
107+
/**
108+
* Convert JS standard objects to strings, remove undefined. This is not
109+
* cycle-safe.
110+
* https://github.com/GoogleCloudPlatform/gcloud-node/issues/1354
111+
*/
112+
function strictJSON(o) {
113+
for (var k in o) {
114+
var v = o[k];
115+
if (v instanceof Date || v instanceof Error || v instanceof RegExp) {
116+
o[k] = v.toJSON();
117+
} else if (v === undefined) {
118+
delete o[k];
119+
} else if (typeof v === 'object') {
120+
strictJSON(v);
89121
}
90-
} catch(err) {
91-
return self.error(err);
92122
}
123+
}
93124

94-
self.getLoggingClient(function (err, authClient) {
95-
var params = {
96-
auth: authClient,
97-
resource: {
98-
//logName: "projects/" + self.project_id + "/logs/" + self.log_id,
99-
//resource: self.resource,
100-
//labels: {},
101-
entries: [{
102-
logName: "projects/" + self.project_id + "/logs/" + self.log_id,
103-
resource: self.resource,
104-
//timestamp: getNow(),
105-
severity: mapLevelToSeverity[levelName] || 'DEFAULT',
106-
//insertId,
107-
//httpRequest,
108-
//labels,
109-
//operation,
110-
[(message instanceof Object)?'jsonPayload':'textPayload']: message
111-
}],
112-
partialSuccess: true
113-
}
114-
};
125+
BunyanStackDriver.prototype._writeToServer = function () {
126+
var self = this;
127+
128+
this.writeQueued = false;
129+
130+
// Atomically get the entries to send and clear the queue
131+
var entries = this.entryQueue.splice(0);
132+
133+
var options = {
134+
partialSuccess: true
135+
};
115136

116-
glogging.entries.write(params, function(err,data){
117-
if(err) return self.error(err);
118-
});
137+
this.log.write(entries, options, function (err, response) {
138+
if (err) return self.error(err);
119139
});
120140
};
121141

‎package.json

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "bunyan-stackdriver",
3-
"version": "0.0.1",
3+
"version": "1.0.0",
44
"description": "StackDriver stream for Bunyan",
55
"main": "./lib/bunyan-stackdriver",
66
"repository": {
@@ -21,13 +21,14 @@
2121
},
2222
"homepage": "https://github.com/mlazarov/bunyan-stackdriver",
2323
"dependencies": {
24-
"googleapis": "^4.0.0"
24+
"@google-cloud/logging": "^0.1.1"
2525
},
2626
"devDependencies": {
2727
"bunyan": "^1.3.3"
2828
},
2929
"maintainers": [
30-
"mlazarov <martin@lazarov.bg>"
30+
"mlazarov <martin@lazarov.bg>",
31+
"zbjornson <zbbjornson@gmail.com>"
3132
],
3233
"scripts": {
3334
"test": "echo \"Error: no test specified\" && exit 1"

0 commit comments

Comments
 (0)
Failed to load comments.