Add Db queries verification to tfb toolset verif procedure

Dockerfile

@@ -15,6 +15,8 @@ RUN apt-get -yqq install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::
 
 RUN pip install colorama==0.3.1 requests MySQL-python psycopg2-binary pymongo docker==4.0.2 psutil
 
+RUN apt-get install -yqq siege
+
 # Fix for docker-py trying to import one package from the wrong location
 RUN cp -r /usr/local/lib/python2.7/dist-packages/backports/ssl_match_hostname/ /usr/lib/python2.7/dist-packages/backports
 

toolset/benchmark/test_types/db_type.py

@@ -1,5 +1,5 @@
 from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object
+from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object, verify_queries_count
 
 
 class DBTestType(FrameworkTestType):
@@ -9,7 +9,7 @@ def __init__(self, config):
             'name': 'db',
             'accept_header': self.accept('json'),
             'requires_db': True,
-            'args': ['db_url']
+            'args': ['db_url', 'database']
         }
         FrameworkTestType.__init__(self, config, **kwargs)
 
@@ -22,6 +22,10 @@ def verify(self, base_url):
         'randomNumber', both of which must map to integers
         '''
 
+        # Initialization for query counting
+        repetitions = 2
+        expected_queries = repetitions * 512
+
         url = base_url + self.db_url
         headers, body = self.request_headers_and_body(url)
 
@@ -52,6 +56,8 @@ def verify(self, base_url):
         problems += verify_randomnumber_object(response, url)
         problems += verify_headers(self.request_headers_and_body, headers, url, should_be='json')
 
+        if len(problems) == 0:
+            problems += verify_queries_count(self, "World", url, 512, repetitions, expected_queries, expected_queries)
         if len(problems) == 0:
             return [('pass', '', url)]
         else:

toolset/benchmark/test_types/fortune_type.py

@@ -1,6 +1,6 @@
 from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
 from toolset.benchmark.fortune_html_parser import FortuneHTMLParser
-from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers
+from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_queries_count
 
 
 class FortuneTestType(FrameworkTestType):
@@ -10,7 +10,7 @@ def __init__(self, config):
             'name': 'fortune',
             'accept_header': self.accept('html'),
             'requires_db': True,
-            'args': ['fortune_url']
+            'args': ['fortune_url','database']
         }
         FrameworkTestType.__init__(self, config, **kwargs)
 
@@ -23,6 +23,10 @@ def verify(self, base_url):
         FortuneHTMLParser whether the parsed string is a
         valid fortune response
         '''
+        # Initialization for query counting
+        repetitions = 2
+        expected_queries = repetitions * 512
+        expected_rows = 12 * expected_queries
 
         url = base_url + self.fortune_url
         headers, body = self.request_headers_and_body(url)
@@ -38,7 +42,8 @@ def verify(self, base_url):
 
         if valid:
             problems += verify_headers(self.request_headers_and_body, headers, url, should_be='html')
-
+            if len(problems) == 0:
+                problems += verify_queries_count(self, "fortune", url, 512, repetitions, expected_queries, expected_rows)
             if len(problems) == 0:
                 return [('pass', '', url)]
             else:

toolset/benchmark/test_types/query_type.py

@@ -9,7 +9,7 @@ def __init__(self, config):
             'name': 'query',
             'accept_header': self.accept('json'),
             'requires_db': True,
-            'args': ['query_url']
+            'args': ['query_url', 'database']
         }
         FrameworkTestType.__init__(self, config, **kwargs)
 
@@ -29,7 +29,7 @@ def verify(self, base_url):
         cases = [('2', 'fail'), ('0', 'fail'), ('foo', 'fail'),
                  ('501', 'warn'), ('', 'fail')]
 
-        problems = verify_query_cases(self, cases, url)
+        problems = verify_query_cases(self, cases, url, False)
 
         if len(problems) == 0:
             return [('pass', '', url + case) for case, _ in cases]

toolset/benchmark/test_types/verifications.py

@@ -7,6 +7,9 @@
 from toolset.databases import databases
 from time import sleep
 
+# Cross-platform colored text
+from colorama import Fore, Style
+
 def basic_body_verification(body, url, is_json_check=True):
     '''
     Takes in a raw (stringy) response body, checks that it is non-empty,
@@ -330,11 +333,16 @@ def verify_query_cases(self, cases, url, check_updates=False):
     problems = []
     MAX = 500
     MIN = 1
+    # Initialization for query counting
+    repetitions = 2
+    expected_queries = 20 * repetitions * 512
+    expected_rows = expected_queries
 
     # Only load in the World table if we are doing an Update verification
     world_db_before = {}
     if check_updates:
         world_db_before = databases[self.database.lower()].get_current_world_table(self.config)
+        expected_queries = 2 * expected_queries
 
     for q, max_infraction in cases:
         case_url = url + q
@@ -384,4 +392,38 @@ def verify_query_cases(self, cases, url, check_updates=False):
                     expected_len, headers, body, case_url, max_infraction)
                 problems += verify_headers(self.request_headers_and_body, headers, case_url)
 
+    if hasattr(self, 'database'):
+        # verify the number of queries and rows read for 20 queries, with a concurrency level of 512, with 2 repetitions
+        problems += verify_queries_count(self, "world", url+"20", 512, repetitions, expected_queries, expected_rows, check_updates)
+    return problems
+
+
+def verify_queries_count(self, tbl_name, url, concurrency=512, count=2, expected_queries=1024, expected_rows = 1024, check_updates = False):
+    '''
+    Checks that the number of executed queries, at the given concurrency level, 
+    corresponds to: the total number of http requests made * the number of queries per request
+    '''
+    log("VERIFYING QUERY COUNT FOR %s" % url, border='-', color=Fore.WHITE + Style.BRIGHT)
+
+    problems = []
+
+    queries, rows, rows_updated, margin = databases[self.database.lower()].verify_queries(self.config, tbl_name, url, concurrency, count, check_updates)
+
+    problems.append(display_queries_count_result(queries, expected_queries, queries, "executed queries", url))
+    problems.append(display_queries_count_result(rows, expected_rows, int(rows / margin), "rows read", url))
+    if check_updates:
+        problems.append(display_queries_count_result(rows_updated, expected_rows, int(rows_updated / margin), "rows updated", url))
+
     return problems
+
+def display_queries_count_result(result, expected_result, displayed_result, caption, url):
+    '''
+    Returns a single result in counting queries, rows read or updated 
+    '''
+    if result < expected_result :
+        return (
+        "fail",
+        "Only %s %s in the database out of roughly %s expected."
+        % (displayed_result, caption, expected_result), url)
+    else:
+        return ("pass","%s: %s/%s" % (caption.capitalize(), displayed_result,expected_result), url)