Caliper terminates if prometheus is not available (#1288)

davidkel · davek-at-ibm · web-flow · commit ada19c3dfb42 · 2022-03-24T17:57:43.000Z
This is due to the error event not being correctly captured when a request is made to prometheus Also added some extra code to output a warning and stop trying to do any more queries for the round. It won't stop it for all rounds but checks on every round. closes #1267 Signed-off-by: D <d_kelsey@uk.ibm.com> Co-authored-by: D <d_kelsey@uk.ibm.com>
diff --git a/packages/caliper-core/lib/common/prometheus/prometheus-query-client.js b/packages/caliper-core/lib/common/prometheus/prometheus-query-client.js
@@ -149,10 +149,11 @@ class PrometheusQueryClient {
                         resolve();
                     }
                 });
-                res.on('error', err => {
-                    Logger.error(err);
-                    reject(err);
-                });
+            });
+
+            req.on('error', err => {
+                Logger.error(err);
+                reject(err);
             });
 
             req.end();
diff --git a/packages/caliper-core/lib/manager/monitors/monitor-prometheus.js b/packages/caliper-core/lib/manager/monitors/monitor-prometheus.js
@@ -65,7 +65,7 @@ class PrometheusMonitor extends MonitorInterface {
      * Retrieve the query client
      * @returns {PrometheusQueryClient} the query client
      */
-    getQueryClient(){
+    getQueryClient() {
         return this.prometheusQueryClient;
     }
 
@@ -74,7 +74,7 @@ class PrometheusMonitor extends MonitorInterface {
     * @async
     */
     async start() {
-        this.startTime = Date.now()/1000;
+        this.startTime = Date.now() / 1000;
     }
 
     /**
@@ -114,7 +114,7 @@ class PrometheusMonitor extends MonitorInterface {
      * @async
      */
     async getStatistics(testLabel) {
-        this.endTime = Date.now()/1000;
+        this.endTime = Date.now() / 1000;
 
         const resourceStats = [];
         const chartArray = [];
@@ -131,7 +131,14 @@ class PrometheusMonitor extends MonitorInterface {
                 //     label: a matching label for the component of interest
                 // }
                 const queryString = PrometheusQueryHelper.buildStringRangeQuery(queryObject.query, this.startTime, this.endTime, queryObject.step);
-                const response = await this.prometheusQueryClient.getByEncodedUrl(queryString);
+
+                let response;
+                try {
+                    response = await this.prometheusQueryClient.getByEncodedUrl(queryString);
+                } catch (error) {
+                    Logger.warn('Failed to connect to Prometheus, unable to perform queries');
+                    break;
+                }
 
                 // Retrieve map of component names and corresponding values for the issued query
                 const componentNameValueMap = PrometheusQueryHelper.extractStatisticFromRange(response, queryObject.statistic, queryObject.label);
@@ -146,13 +153,13 @@ class PrometheusMonitor extends MonitorInterface {
                         newQueryObjectIteration = false;
                         watchItemStat.set('Name', key);
                         const multiplier = queryObject.multiplier ? queryObject.multiplier : 1;
-                        watchItemStat.set('Value', (value*multiplier).toPrecision(this.precision));
+                        watchItemStat.set('Value', (value * multiplier).toPrecision(this.precision));
                         // Store
                         resourceStats.push(watchItemStat);
 
                         // Build separate charting information
                         const metricMap = new Map();
-                        metricMap.set('Name',  watchItemStat.get('Name'));
+                        metricMap.set('Name', watchItemStat.get('Name'));
                         metricMap.set(queryObject.name, watchItemStat.get('Value'));
                         metricArray.push(metricMap);
                     }
diff --git a/packages/caliper-core/test/manager/monitors/monitor-prometheus.js b/packages/caliper-core/test/manager/monitors/monitor-prometheus.js
@@ -21,33 +21,58 @@ const chai = require('chai');
 const should = chai.should();
 const sinon = require('sinon');
 
+class FakeQueryClient {
+    static getByEncodedUrlCount = 0;
+
+    static reset() {
+        FakeQueryClient.getByEncodedUrlCount = 0;
+    }
+
+    static setGetByEncodedUrlResponse(ableToConnect, response) {
+        FakeQueryClient.ableToConnect = ableToConnect;
+        FakeQueryClient.response = response;
+    }
+
+    async getByEncodedUrl() {
+        FakeQueryClient.getByEncodedUrlCount++;
+        if (!FakeQueryClient.ableToConnect) {
+            throw new Error('ECONNREFUSED');
+        }
+        return FakeQueryClient.response;
+    }
+}
+
 describe('Prometheus monitor implementation', () => {
 
-    const fakeQueryClient = sinon.stub();
-    PrometheusMonitorRewire.__set__('PrometheusQueryClient', fakeQueryClient);
+    PrometheusMonitorRewire.__set__('PrometheusQueryClient', FakeQueryClient);
 
     // Before/After
     let clock;
-    beforeEach( () => {
+    beforeEach(() => {
         clock = sinon.useFakeTimers();
     });
 
-    afterEach( () => {
+    afterEach(() => {
         clock.restore();
     });
 
     // Test data
     const monitorOptions = {
         metrics : {
-            include: ['peer', 'pushgateway', 'dev.*'],
+            url: 'http://localhost:9090',
+            include: ['peer', 'orderer', 'dev.*'],
             queries: [
                 {
+                    name: 'avg cpu',
+                    label: 'name',
                     query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100',
-                    statistic: 'average'
+                    statistic: 'avg'
                 },
                 {
+                    name: 'max cpu',
+                    label: 'name',
                     query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100',
-                    statistic: 'maximum'
+                    statistic: 'max'
                 }
             ]
         }
@@ -78,7 +103,7 @@ describe('Prometheus monitor implementation', () => {
         });
     });
 
-    describe('#getQueryClient', ()=>{
+    describe('#getQueryClient', () => {
 
         it('should return the internal Query Client', () => {
             const mon = new PrometheusMonitorRewire({});
@@ -92,7 +117,7 @@ describe('Prometheus monitor implementation', () => {
 
         it('should set the start time with the current time', () => {
             clock.tick(42);
-            const mon = new PrometheusMonitorRewire({push_url: '123'});
+            const mon = new PrometheusMonitorRewire({ push_url: '123' });
             mon.start();
             mon.startTime.should.equal(0.042);
         });
@@ -102,7 +127,7 @@ describe('Prometheus monitor implementation', () => {
     describe('#stop', () => {
         it('should remove startTime if it exists', () => {
             clock.tick(42);
-            const mon = new PrometheusMonitorRewire({push_url: '123'});
+            const mon = new PrometheusMonitorRewire({ push_url: '123' });
             mon.start();
             mon.startTime.should.equal(0.042);
             mon.stop();
@@ -114,7 +139,7 @@ describe('Prometheus monitor implementation', () => {
 
         it('should reset the start time', () => {
             clock.tick(42);
-            const mon = new PrometheusMonitorRewire({push_url: '123'});
+            const mon = new PrometheusMonitorRewire({ push_url: '123' });
             mon.start();
             clock.tick(42);
             mon.restart();
@@ -149,15 +174,83 @@ describe('Prometheus monitor implementation', () => {
             const mon = new PrometheusMonitorRewire(monitorOptions);
             mon.includeStatistic('peer0.org0.example.com').should.equal(true);
 
-            mon.includeStatistic('pushgateway').should.equal(true);
+            mon.includeStatistic('pushgateway').should.equal(false);
 
             mon.includeStatistic('dev-org0.example.com').should.equal(true);
 
             mon.includeStatistic('penuin').should.equal(false);
 
-            mon.includeStatistic('orderer0.example.com').should.equal(false);
+            mon.includeStatistic('orderer0.example.com').should.equal(true);
         });
     });
 
+    describe('When getting statistics', () => {
+        const response = {
+            status: 'success',
+            data: {
+                resultType: 'matrix',
+                result: [
+                    {
+                        metric: {
+                            name: 'orderer.example.com'
+                        },
+                        values: [
+                            [
+                                1648125000.736,
+                                '37318656'
+                            ],
+                            [
+                                1648125010.736,
+                                '37318656'
+                            ],
+                            [
+                                1648125020.736,
+                                '37318656'
+                            ]
+                        ]
+                    },
+                    {
+                        metric: {
+                            name: 'peer0.org1.example.com'
+                        },
+                        values: [
+                            [
+                                1648125000.736,
+                                '80855040'
+                            ],
+                            [
+                                1648125010.736,
+                                '80855040'
+                            ],
+                            [
+                                1648125020.736,
+                                '80855040'
+                            ]
+                        ]
+                    }
+                ]
+            }
+        };
+
+        it('should stop processing further queries and return an empty set of results if it fails to connect to prometheus ', async () => {
+            const prometheusMonitor = new PrometheusMonitorRewire(monitorOptions);
+            FakeQueryClient.reset();
+            FakeQueryClient.setGetByEncodedUrlResponse(false);
+            const res = await prometheusMonitor.getStatistics();
+            FakeQueryClient.getByEncodedUrlCount.should.equal(1);
+            res.resourceStats.length.should.equal(0);
+            res.chartStats.length.should.equal(0);
+        });
+
+        it('should process all queries successfully if able to connect to prometheus', async () => {
+            const prometheusMonitor = new PrometheusMonitorRewire(monitorOptions);
+            FakeQueryClient.reset();
+            FakeQueryClient.setGetByEncodedUrlResponse(true, response);
+            const res = await prometheusMonitor.getStatistics();
+            FakeQueryClient.getByEncodedUrlCount.should.equal(2);
+            res.resourceStats.length.should.equal(4);
+            res.chartStats.length.should.equal(0);
+        });
+    });
 
 });
