Skip to content

Commit

Permalink
Caliper terminates if prometheus is not available (#1288)
Browse files Browse the repository at this point in the history 
This is due to the error event not being correctly captured when a
request is made to prometheus

Also added some extra code to output a warning and stop trying to do any
more queries for the round.

It won't stop it for all rounds but checks on every round.

closes #1267

Signed-off-by: D <d_kelsey@uk.ibm.com>

Co-authored-by: D <d_kelsey@uk.ibm.com>
  • Loading branch information
@davidkel
davidkel and davidkel committed Mar 24, 2022
1 parent 6a76834 commit ada19c3
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 23 deletions.
9 changes: 5 additions & 4 deletions packages/caliper-core/lib/common/prometheus/prometheus-query-client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,10 +149,11 @@ class PrometheusQueryClient {
resolve();
}
});
res.on('error', err => {
Logger.error(err);
reject(err);
});
});

req.on('error', err => {
Logger.error(err);
reject(err);
});

req.end();
Expand Down
19 changes: 13 additions & 6 deletions packages/caliper-core/lib/manager/monitors/monitor-prometheus.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class PrometheusMonitor extends MonitorInterface {
* Retrieve the query client
* @returns {PrometheusQueryClient} the query client
*/
getQueryClient(){
getQueryClient() {
return this.prometheusQueryClient;
}

Expand All @@ -74,7 +74,7 @@ class PrometheusMonitor extends MonitorInterface {
* @async
*/
async start() {
this.startTime = Date.now()/1000;
this.startTime = Date.now() / 1000;
}

/**
Expand Down Expand Up @@ -114,7 +114,7 @@ class PrometheusMonitor extends MonitorInterface {
* @async
*/
async getStatistics(testLabel) {
this.endTime = Date.now()/1000;
this.endTime = Date.now() / 1000;

const resourceStats = [];
const chartArray = [];
Expand All @@ -131,7 +131,14 @@ class PrometheusMonitor extends MonitorInterface {
// label: a matching label for the component of interest
// }
const queryString = PrometheusQueryHelper.buildStringRangeQuery(queryObject.query, this.startTime, this.endTime, queryObject.step);
const response = await this.prometheusQueryClient.getByEncodedUrl(queryString);

let response;
try {
response = await this.prometheusQueryClient.getByEncodedUrl(queryString);
} catch (error) {
Logger.warn('Failed to connect to Prometheus, unable to perform queries');
break;
}

// Retrieve map of component names and corresponding values for the issued query
const componentNameValueMap = PrometheusQueryHelper.extractStatisticFromRange(response, queryObject.statistic, queryObject.label);
Expand All @@ -146,13 +153,13 @@ class PrometheusMonitor extends MonitorInterface {
newQueryObjectIteration = false;
watchItemStat.set('Name', key);
const multiplier = queryObject.multiplier ? queryObject.multiplier : 1;
watchItemStat.set('Value', (value*multiplier).toPrecision(this.precision));
watchItemStat.set('Value', (value * multiplier).toPrecision(this.precision));
// Store
resourceStats.push(watchItemStat);

// Build separate charting information
const metricMap = new Map();
metricMap.set('Name', watchItemStat.get('Name'));
metricMap.set('Name', watchItemStat.get('Name'));
metricMap.set(queryObject.name, watchItemStat.get('Value'));
metricArray.push(metricMap);
}
Expand Down
119 changes: 106 additions & 13 deletions packages/caliper-core/test/manager/monitors/monitor-prometheus.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,33 +21,58 @@ const chai = require('chai');
const should = chai.should();
const sinon = require('sinon');

class FakeQueryClient {
static getByEncodedUrlCount = 0;

static reset() {
FakeQueryClient.getByEncodedUrlCount = 0;
}

static setGetByEncodedUrlResponse(ableToConnect, response) {
FakeQueryClient.ableToConnect = ableToConnect;
FakeQueryClient.response = response;
}

async getByEncodedUrl() {
FakeQueryClient.getByEncodedUrlCount++;
if (!FakeQueryClient.ableToConnect) {
throw new Error('ECONNREFUSED');
}
return FakeQueryClient.response;
}
}

describe('Prometheus monitor implementation', () => {

const fakeQueryClient = sinon.stub();
PrometheusMonitorRewire.__set__('PrometheusQueryClient', fakeQueryClient);
PrometheusMonitorRewire.__set__('PrometheusQueryClient', FakeQueryClient);

// Before/After
let clock;
beforeEach( () => {
beforeEach(() => {
clock = sinon.useFakeTimers();
});

afterEach( () => {
afterEach(() => {
clock.restore();
});

// Test data
const monitorOptions = {
metrics : {
include: ['peer', 'pushgateway', 'dev.*'],
url: 'http://localhost:9090',
include: ['peer', 'orderer', 'dev.*'],
queries: [
{
name: 'avg cpu',
label: 'name',
query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100',
statistic: 'average'
statistic: 'avg'
},
{
name: 'max cpu',
label: 'name',
query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100',
statistic: 'maximum'
statistic: 'max'
}
]
}
Expand Down Expand Up @@ -78,7 +103,7 @@ describe('Prometheus monitor implementation', () => {
});
});

describe('#getQueryClient', ()=>{
describe('#getQueryClient', () => {

it('should return the internal Query Client', () => {
const mon = new PrometheusMonitorRewire({});
Expand All @@ -92,7 +117,7 @@ describe('Prometheus monitor implementation', () => {

it('should set the start time with the current time', () => {
clock.tick(42);
const mon = new PrometheusMonitorRewire({push_url: '123'});
const mon = new PrometheusMonitorRewire({ push_url: '123' });
mon.start();
mon.startTime.should.equal(0.042);
});
Expand All @@ -102,7 +127,7 @@ describe('Prometheus monitor implementation', () => {
describe('#stop', () => {
it('should remove startTime if it exists', () => {
clock.tick(42);
const mon = new PrometheusMonitorRewire({push_url: '123'});
const mon = new PrometheusMonitorRewire({ push_url: '123' });
mon.start();
mon.startTime.should.equal(0.042);
mon.stop();
Expand All @@ -114,7 +139,7 @@ describe('Prometheus monitor implementation', () => {

it('should reset the start time', () => {
clock.tick(42);
const mon = new PrometheusMonitorRewire({push_url: '123'});
const mon = new PrometheusMonitorRewire({ push_url: '123' });
mon.start();
clock.tick(42);
mon.restart();
Expand Down Expand Up @@ -149,15 +174,83 @@ describe('Prometheus monitor implementation', () => {
const mon = new PrometheusMonitorRewire(monitorOptions);
mon.includeStatistic('peer0.org0.example.com').should.equal(true);

mon.includeStatistic('pushgateway').should.equal(true);
mon.includeStatistic('pushgateway').should.equal(false);

mon.includeStatistic('dev-org0.example.com').should.equal(true);

mon.includeStatistic('penuin').should.equal(false);

mon.includeStatistic('orderer0.example.com').should.equal(false);
mon.includeStatistic('orderer0.example.com').should.equal(true);
});
});

describe('When getting statistics', () => {
const response = {
status: 'success',
data: {
resultType: 'matrix',
result: [
{
metric: {
name: 'orderer.example.com'
},
values: [
[
1648125000.736,
'37318656'
],
[
1648125010.736,
'37318656'
],
[
1648125020.736,
'37318656'
]
]
},
{
metric: {
name: 'peer0.org1.example.com'
},
values: [
[
1648125000.736,
'80855040'
],
[
1648125010.736,
'80855040'
],
[
1648125020.736,
'80855040'
]
]
}
]
}
};

it('should stop processing further queries and return an empty set of results if it fails to connect to prometheus ', async () => {
const prometheusMonitor = new PrometheusMonitorRewire(monitorOptions);
FakeQueryClient.reset();
FakeQueryClient.setGetByEncodedUrlResponse(false);
const res = await prometheusMonitor.getStatistics();
FakeQueryClient.getByEncodedUrlCount.should.equal(1);
res.resourceStats.length.should.equal(0);
res.chartStats.length.should.equal(0);
});

it('should process all queries successfully if able to connect to prometheus', async () => {
const prometheusMonitor = new PrometheusMonitorRewire(monitorOptions);
FakeQueryClient.reset();
FakeQueryClient.setGetByEncodedUrlResponse(true, response);
const res = await prometheusMonitor.getStatistics();
FakeQueryClient.getByEncodedUrlCount.should.equal(2);
res.resourceStats.length.should.equal(4);
res.chartStats.length.should.equal(0);
});
});

});

0 comments on commit ada19c3

Please sign in to comment.