Bench mark report (#280)

feat(benchmark): give reports score by a defined benchmark
Zooz · Apr 7, 2020 · e467f57 · e467f57
1 parent 8e3441f
commit e467f57
Show file tree

Hide file tree

Showing 36 changed files with 1,306 additions and 307 deletions.
diff --git a/docs/devguide/docs/swagger-docs.yaml b/docs/devguide/docs/swagger-docs.yaml
@@ -319,6 +319,40 @@ paths:
               $ref: '#/components/schemas/benchmark_request'
         description: The bench mark to add
         required: true
+    get:
+      tags:
+        - Tests
+      parameters:
+        - in: path
+          name: test_id
+          description: The test id.
+          required: true
+          schema:
+            type: string
+            format: uuid
+            example: 4bf5d7ab-f310-4a64-8ec2-d65c06188ec1
+      summary: get a bench mark for test
+      description:  Get a benchmark for a test
+      responses:
+        '200':
+          description: Success
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/benchmark_request'
+        '404':
+          description: Not found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error_response'
+        '500':
+          description: Internal server error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error_response'
+
   '/v1/tests/{test_id}':
     get:
       operationId: retrieve-a-test
@@ -1182,6 +1216,11 @@ components:
     benchmark_request:
       type: object
       additionalProperties: false
+      required:
+        - errors
+        - codes
+        - rps
+        - latency
       properties:
         errors:
           type: object
@@ -1196,24 +1235,22 @@ components:
             500: 53
         rps:
           type: object
+          required:
+            - mean
           properties:
-            count:
-              type: number
             mean:
               type: number
         latency:
           type: object
+          required:
+            - median
+            - p95
           properties:
-            min:
-              type: number
-            max:
-              type: number
             median:
               type: number
             p95:
               type: number
-            p99:
-              type: number
+
     dsl:
       description: A test that is made of scenarios base on domain specific language
       allOf:
@@ -1770,6 +1807,13 @@ components:
           type: number
           description: The duration of the test in seconds.
           example: 60
+        score:
+          type: number
+          description: The score of a report, calculated based on a benchmark and benchmark weights
+          example: 9.25
+        benchmark_weights_data:
+          type: object
+          description: The way score calualated
         arrival_rate:
           type: number
           description: The arrival rate that was set for the test. This is the number of times per second that the test scenarios will run.
@@ -1911,14 +1955,14 @@ components:
               type: object
               description: benchmark_weights
               required:
-              - percentile_ninety
+              - percentile_ninety_five
               - percentile_fifty
               - server_errors
               - client_errors
               - rps
               properties:
-                percentile_ninety:
-                  description: P90 benchmark weight
+                percentile_ninety_five:
+                  description: P95 benchmark weight
                   allOf:
                     - $ref: '#/components/schemas/benchmark_weights'
                 percentile_fifty:
@@ -1940,12 +1984,8 @@ components:
     benchmark_weights:
       type: object
       required:
-        - factor
         - percentage
       properties:
-        factor:
-          type: number
-          description: benchmark factor weight
         percentage:
             type: number
             description: benchmark percentage weight
diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml
@@ -323,6 +323,40 @@ paths:
               $ref: '#/components/schemas/benchmark_request'
         description: The bench mark to add
         required: true
+    get:
+      tags:
+        - Tests
+      parameters:
+        - in: path
+          name: test_id
+          description: The test id.
+          required: true
+          schema:
+            type: string
+            format: uuid
+            example: 4bf5d7ab-f310-4a64-8ec2-d65c06188ec1
+      summary: get a bench mark for test
+      description:  Get a benchmark for a test
+      responses:
+        '200':
+          description: Success
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/benchmark_request'
+        '404':
+          description: Not found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error_response'
+        '500':
+          description: Internal server error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error_response'
+
   '/v1/tests/file/{file_id}':
     get:
       operationId: retrieve-a-file
@@ -1465,6 +1499,11 @@ components:
     benchmark_request:
       type: object
       additionalProperties: false
+      required:
+        - errors
+        - codes
+        - rps
+        - latency
       properties:
         errors:
           type: object
@@ -1479,24 +1518,21 @@ components:
             500: 53
         rps:
           type: object
+          required:
+            - mean
           properties:
-            count:
-              type: number
             mean:
               type: number
         latency:
           type: object
+          required:
+            - median
+            - p95
           properties:
-            min:
-              type: number
-            max:
-              type: number
             median:
               type: number
             p95:
               type: number
-            p99:
-              type: number
 
     dsl:
       description: A test that is made of scenarios base on domain specific language
@@ -2047,6 +2083,13 @@ components:
           type: number
           description: The duration of the test in seconds.
           example: 60
+        score:
+          type: number
+          description: The score of a report, calculated based on a benchmark and benchmark weights
+          example: 9.25
+        benchmark_weights_data:
+          type: object
+          description: The way score calualated
         arrival_rate:
           type: number
           description: The arrival rate that was set for the test. This is the number of times per second that the test scenarios will run.
@@ -2192,14 +2235,14 @@ components:
               type: object
               description: benchmark_weights
               required:
-              - percentile_ninety
+              - percentile_ninety_five
               - percentile_fifty
               - server_errors
               - client_errors
               - rps
               properties:
-                percentile_ninety:
-                  description: P90 benchmark weight
+                percentile_ninety_five:
+                  description: P95 benchmark weight
                   allOf:
                     - $ref: '#/components/schemas/benchmark_weights'
                 percentile_fifty:
@@ -2221,12 +2264,8 @@ components:
     benchmark_weights:
       type: object
       required:
-        - factor
-        - percentage
+         - percentage
       properties:
-        factor:
-          type: number
-          description: benchmark factor weight
         percentage:
             type: number
             description: benchmark percentage weight
diff --git a/src/configManager/helpers/configDataMap.js b/src/configManager/helpers/configDataMap.js
@@ -1,5 +1,13 @@
 const constConfig = require('../../common/consts').CONFIG;
 
+const BENCHMARK_WEIGHTS_DEFAULT = {
+    percentile_ninety_five: { percentage: 20 },
+    percentile_fifty: { percentage: 20 },
+    server_errors: { percentage: 20 },
+    client_errors: { percentage: 20 },
+    rps: { percentage: 20 }
+};
+
 let configDataMap = {
     [constConfig.GRFANA_URL]: { value: process.env.GRAFANA_URL },
     [constConfig.DELAY_RUNNER_MS]: { value: process.env.DELAY_RUNNER_MS || 0, type: 'int' },
@@ -32,7 +40,7 @@ let configDataMap = {
     },
     [constConfig.BENCHMARK_THRESHOLD]: { value: process.env.BENCHMARK_THRESHOLD, type: 'int' },
     [constConfig.BENCHMARK_THRESHOLD_WEBHOOK_URL]: { value: process.env.BENCHMARK_THRESHOLD_WEBHOOK_URL, type: 'string' },
-    [constConfig.BENCHMARK_WEIGHTS]: { value: process.env.BENCHMARK_WEIGHTS, type: 'json' }
+    [constConfig.BENCHMARK_WEIGHTS]: { value: process.env.BENCHMARK_WEIGHTS || JSON.stringify(BENCHMARK_WEIGHTS_DEFAULT), type: 'json' }
 };
 
 module.exports.getConstType = (configValue) => {

diff --git a/src/configManager/helpers/validators.js b/src/configManager/helpers/validators.js
@@ -1,13 +1,13 @@
 module.exports.validateBenchmarkWeights = (req, res, next) => {
     const benchmarkWeights = req.body.benchmark_weights;
     if (benchmarkWeights){
-        const p90Percentage = benchmarkWeights.percentile_ninety.percentage;
+        const p95Percentage = benchmarkWeights.percentile_ninety_five.percentage;
         const p50Percentage = benchmarkWeights.percentile_fifty.percentage;
         const serverErrorsPercentage = benchmarkWeights.server_errors.percentage;
         const clientErrorsPercentage = benchmarkWeights.client_errors.percentage;
         const rpsPercentage = benchmarkWeights.rps.percentage;
 
-        const percentageSum = p90Percentage + p50Percentage + serverErrorsPercentage + clientErrorsPercentage + rpsPercentage;
+        const percentageSum = p95Percentage + p50Percentage + serverErrorsPercentage + clientErrorsPercentage + rpsPercentage;
         if (percentageSum !== 100){
             const error = new Error('Benchmark weights needs to sum up to 100%');
             error.statusCode = 422;

diff --git a/src/database/cassandra-handler/init-scripts/19__bench_mark_data_reports.cql b/src/database/cassandra-handler/init-scripts/19__bench_mark_data_reports.cql
@@ -0,0 +1,8 @@
+ALTER TABLE reports_summary ADD benchmark_weights_data text;
+---
+ALTER TABLE reports_summary ADD score float;
+---
+ALTER TABLE last_reports ADD benchmark_weights_data text;
+---
+ALTER TABLE last_reports ADD score float;
+