From 02e4eaa5950e73c3f4848798309a6504f305dadd Mon Sep 17 00:00:00 2001 From: Bennett Buchanan Date: Wed, 14 Mar 2018 15:14:13 -0700 Subject: [PATCH 1/3] FT: Add CRR statistics --- lib/utilities/reportHandler.js | 53 ++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/lib/utilities/reportHandler.js b/lib/utilities/reportHandler.js index 8a38ed1d55..175e71d5e6 100644 --- a/lib/utilities/reportHandler.js +++ b/lib/utilities/reportHandler.js @@ -3,6 +3,7 @@ const os = require('os'); const { errors, ipCheck } = require('arsenal'); const async = require('async'); +const request = require('request'); const config = require('../Config').config; const data = require('../data/wrapper'); @@ -73,6 +74,56 @@ function getSystemStats() { }; } +function getCRRStats(log, cb) { + log.debug('getting CRR stats', { method: 'getCRRStats' }); + // TODO: Reuse metrics code from Backbeat by moving it to Arsenal instead of + // making an HTTP request to the Backbeat metrics route. + const params = { url: 'http://localhost:8900/_/metrics/crr/all' }; + return request.get(params, (err, res) => { + if (err) { + log.error('failed to get CRR stats', { + method: 'getCRRStats', + error: err, + }); + return cb(err); + } + let body; + try { + body = JSON.parse(res.body); + } catch (parseErr) { + log.error('could not parse backbeat response', { + method: 'getCRRStats', + error: parseErr, + }); + return cb(errors.InternalError + .customizeDescription('could not parse backbeat response')); + } + const { completions, backlog, throughput } = body; + if (!completions || !backlog || !throughput) { + log.error('could not get metrics from backbeat', { + method: 'getCRRStats', + }); + return cb(errors.InternalError + .customizeDescription('could not get metrics from backbeat')); + } + const stats = { + completions: { + count: parseFloat(completions.results.count), + size: parseFloat(completions.results.size), + }, + backlog: { + count: parseFloat(backlog.results.count), + size: parseFloat(backlog.results.size), + }, + throughput: { + count: parseFloat(throughput.results.count), + size: parseFloat(throughput.results.size), + }, + }; + return cb(null, stats); + }); +} + /** * Sends back a report * @@ -97,6 +148,7 @@ function reportHandler(clientIP, req, res, log) { getDataDiskUsage: cb => data.getDiskUsage(log, cb), getVersion: cb => getGitVersion(cb), getObjectCount: cb => metadata.countItems(log, cb), + getCRRStats: cb => getCRRStats(log, cb), }, (err, results) => { if (err) { @@ -114,6 +166,7 @@ function reportHandler(clientIP, req, res, log) { serverVersion: results.getVersion, systemStats: getSystemStats(), itemCounts: results.getObjectCount, + crrStats: results.getCRRStats, config: cleanup(config), }; From d2719e1d36ad3acdd9e1a78c550c80ac15f5b5f2 Mon Sep 17 00:00:00 2001 From: Bennett Buchanan Date: Thu, 15 Mar 2018 13:35:34 -0700 Subject: [PATCH 2/3] [squash] Address comments --- config.json | 4 ++++ lib/Config.js | 9 +++++++++ lib/utilities/reportHandler.js | 35 ++++++++++++---------------------- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/config.json b/config.json index 8721973621..8ec879df6a 100644 --- a/config.json +++ b/config.json @@ -34,6 +34,10 @@ "site": "us-east-2", "type": "aws_s3" }], + "backbeat": { + "host": "localhost", + "port": 8900 + }, "cdmi": { "host": "localhost", "port": 81, diff --git a/lib/Config.js b/lib/Config.js index 457b91792a..5ac8c44219 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -433,6 +433,15 @@ class Config extends EventEmitter { this.replicationEndpoints = replicationEndpoints; } + if (config.backbeat) { + const { backbeat } = config; + assert.strictEqual(typeof backbeat.host, 'string', + 'bad config: backbeat host must be a string'); + assert(Number.isInteger(backbeat.port) && backbeat.port > 0, + 'bad config: backbeat port must be a positive integer'); + this.backbeat = backbeat; + } + // legacy if (config.regions !== undefined) { throw new Error('bad config: regions key is deprecated. ' + diff --git a/lib/utilities/reportHandler.js b/lib/utilities/reportHandler.js index 175e71d5e6..1eb5a1a808 100644 --- a/lib/utilities/reportHandler.js +++ b/lib/utilities/reportHandler.js @@ -78,50 +78,39 @@ function getCRRStats(log, cb) { log.debug('getting CRR stats', { method: 'getCRRStats' }); // TODO: Reuse metrics code from Backbeat by moving it to Arsenal instead of // making an HTTP request to the Backbeat metrics route. - const params = { url: 'http://localhost:8900/_/metrics/crr/all' }; + const { host, port } = config.backbeat; + const params = { url: `http://${host}:${port}/_/metrics/crr/all` }; return request.get(params, (err, res) => { if (err) { log.error('failed to get CRR stats', { method: 'getCRRStats', error: err, }); - return cb(err); + return cb(null, {}); } - let body; - try { - body = JSON.parse(res.body); - } catch (parseErr) { - log.error('could not parse backbeat response', { - method: 'getCRRStats', - error: parseErr, - }); - return cb(errors.InternalError - .customizeDescription('could not parse backbeat response')); - } - const { completions, backlog, throughput } = body; + const { completions, backlog, throughput } = res.body; if (!completions || !backlog || !throughput) { log.error('could not get metrics from backbeat', { method: 'getCRRStats', }); - return cb(errors.InternalError - .customizeDescription('could not get metrics from backbeat')); + return cb(null, {}); } const stats = { completions: { - count: parseFloat(completions.results.count), - size: parseFloat(completions.results.size), + count: completions.results.count, + size: parseFloat(completions.results.size) * 1000, }, backlog: { - count: parseFloat(backlog.results.count), - size: parseFloat(backlog.results.size), + count: backlog.results.count, + size: parseFloat(backlog.results.size) * 1000, }, throughput: { - count: parseFloat(throughput.results.count), - size: parseFloat(throughput.results.size), + count: parseFloat(throughput.results.count) * 1000, + size: parseFloat(throughput.results.size) * 1000, }, }; return cb(null, stats); - }); + }).json(); } /** From 487e65f47aa4f7fa92b8663f0f3e19b703a97fec Mon Sep 17 00:00:00 2001 From: Bennett Buchanan Date: Thu, 15 Mar 2018 19:30:52 -0700 Subject: [PATCH 3/3] [squash] Add backbeat host and port to entrypoint --- docker-entrypoint.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 9344bc572a..ff26adae2a 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -102,6 +102,14 @@ if [[ "$RECORDLOG_ENABLED" ]]; then JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .recordLog.enabled=true" fi +if [[ "$CRR_METRICS_HOST" ]]; then + JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.host=\"$CRR_METRICS_HOST\"" +fi + +if [[ "$CRR_METRICS_PORT" ]]; then + JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT" +fi + if [[ $JQ_FILTERS_CONFIG != "." ]]; then jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp mv config.json.tmp config.json