WT-7312 Keys/Values updated to String type and save the created keys (#…

…6424)

* Fixed configuration parsing

* Removed key_format and value_format parameters from the test configuration

* Key and values are now of type string.

* Created a Database struct to represent the data model to keep track of the collections, keys and values during a test. Keys are now using the key_size given in the test configuration during the populate stage.

dist/test_data.py

@@ -48,12 +48,8 @@ def __ge__(self, other):
 record_config = [
     Config('key_size', 0, r'''
         The size of the keys created''', min=0, max=10000),
-    Config('key_format', 'i', r'''
-        The format of the keys in the database'''),
     Config('value_size', 0, r'''
         The size of the values created''', min=0, max=1000000000),
-    Config('value_format', 'S', r'''
-        The format of the values stored in the database.''')
 ]
 
 #

src/config/test_config.c

@@ -17,33 +17,27 @@ static const WT_CONFIG_CHECK confchk_timestamp_manager_subconfigs[] = {
   {"stable_lag", "int", NULL, "min=0,max=1000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
 
 static const WT_CONFIG_CHECK confchk_insert_config_subconfigs[] = {
-  {"key_format", "string", NULL, NULL, NULL, 0},
   {"key_size", "int", NULL, "min=0,max=10000", NULL, 0},
-  {"value_format", "string", NULL, NULL, NULL, 0},
   {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
 
 static const WT_CONFIG_CHECK confchk_ops_per_transaction_subconfigs[] = {
   {"max", "string", NULL, NULL, NULL, 0}, {"min", "string", NULL, NULL, NULL, 0},
   {NULL, NULL, NULL, NULL, NULL, 0}};
 
 static const WT_CONFIG_CHECK confchk_update_config_subconfigs[] = {
-  {"key_format", "string", NULL, NULL, NULL, 0},
   {"key_size", "int", NULL, "min=0,max=10000", NULL, 0},
-  {"value_format", "string", NULL, NULL, NULL, 0},
   {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
 
 static const WT_CONFIG_CHECK confchk_workload_generator_subconfigs[] = {
   {"collection_count", "int", NULL, "min=0,max=200000", NULL, 0},
-  {"insert_config", "category", NULL, NULL, confchk_insert_config_subconfigs, 4},
+  {"insert_config", "category", NULL, NULL, confchk_insert_config_subconfigs, 2},
   {"insert_threads", "int", NULL, "min=0,max=20", NULL, 0},
   {"key_count", "int", NULL, "min=0,max=1000000", NULL, 0},
-  {"key_format", "string", NULL, NULL, NULL, 0},
   {"key_size", "int", NULL, "min=0,max=10000", NULL, 0},
   {"ops_per_transaction", "category", NULL, NULL, confchk_ops_per_transaction_subconfigs, 2},
   {"read_threads", "int", NULL, "min=0,max=100", NULL, 0},
-  {"update_config", "category", NULL, NULL, confchk_update_config_subconfigs, 4},
+  {"update_config", "category", NULL, NULL, confchk_update_config_subconfigs, 2},
   {"update_threads", "int", NULL, "min=0,max=20", NULL, 0},
-  {"value_format", "string", NULL, NULL, NULL, 0},
   {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
 
 static const WT_CONFIG_CHECK confchk_workload_tracking_subconfigs[] = {
@@ -55,7 +49,7 @@ static const WT_CONFIG_CHECK confchk_poc_test[] = {
   {"enable_logging", "boolean", NULL, NULL, NULL, 0},
   {"runtime_monitor", "category", NULL, NULL, confchk_runtime_monitor_subconfigs, 2},
   {"timestamp_manager", "category", NULL, NULL, confchk_timestamp_manager_subconfigs, 3},
-  {"workload_generator", "category", NULL, NULL, confchk_workload_generator_subconfigs, 12},
+  {"workload_generator", "category", NULL, NULL, confchk_workload_generator_subconfigs, 10},
   {"workload_tracking", "category", NULL, NULL, confchk_workload_tracking_subconfigs, 1},
   {NULL, NULL, NULL, NULL, NULL, 0}};
 
@@ -65,13 +59,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {
     "runtime_monitor=(rate_per_second=1,"
     "stat_cache_size=(enabled=false,limit=)),"
     "timestamp_manager=(enabled=false,oldest_lag=0,stable_lag=0),"
-    "workload_generator=(collection_count=1,"
-    "insert_config=(key_format=i,key_size=0,value_format=S,"
-    "value_size=0),insert_threads=0,key_count=0,key_format=i,"
-    "key_size=0,ops_per_transaction=(max=1,min=),read_threads=0,"
-    "update_config=(key_format=i,key_size=0,value_format=S,"
-    "value_size=0),update_threads=0,value_format=S,value_size=0),"
-    "workload_tracking=(enabled=false)",
+    "workload_generator=(collection_count=1,insert_config=(key_size=0"
+    ",value_size=0),insert_threads=0,key_count=0,key_size=0,"
+    "ops_per_transaction=(max=1,min=),read_threads=0,"
+    "update_config=(key_size=0,value_size=0),update_threads=0,"
+    "value_size=0),workload_tracking=(enabled=false)",
     confchk_poc_test, 7},
   {NULL, NULL, NULL, 0}};
 

test/cppsuite/configs/config_poc_test_default.txt

@@ -23,7 +23,6 @@ workload_generator=
 {
     collection_count=2
     key_count=5
-    key_format=i
     key_size=1
     ops_per_transaction=
     {
@@ -32,7 +31,6 @@ workload_generator=
     }
     read_threads=1
     value_size=10
-    value_format=S
 }
 workload_tracking=
 {

test/cppsuite/test_harness/api_const.h

@@ -45,6 +45,7 @@ static const char *DURATION_SECONDS = "duration_seconds";
 static const char *ENABLED = "enabled";
 static const char *ENABLE_LOGGING = "enable_logging";
 static const char *KEY_COUNT = "key_count";
+static const char *KEY_SIZE = "key_size";
 static const char *LIMIT = "limit";
 static const char *MAX = "max";
 static const char *MIN = "min";
@@ -63,6 +64,7 @@ static const char *OLDEST_TS = "oldest_timestamp";
 static const char *STABLE_TS = "stable_timestamp";
 
 /* Test harness consts. */
+static const char *DEFAULT_FRAMEWORK_SCHEMA = "key_format=S,value_format=S";
 static const char *TABLE_OPERATION_TRACKING = "table:operation_tracking";
 static const char *TABLE_SCHEMA_TRACKING = "table:schema_tracking";
 static const char *STATISTICS_URI = "statistics:";

test/cppsuite/test_harness/configuration.h

@@ -83,7 +83,7 @@ class configuration {
     {
         WT_CONFIG_ITEM temp_value;
         testutil_check(_config_parser->get(_config_parser, key.c_str(), &temp_value));
-        if (temp_value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRING ||
+        if (temp_value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRING &&
           temp_value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_ID)
             return (-1);
         value = std::string(temp_value.str, temp_value.len);

test/cppsuite/test_harness/database_model.h

@@ -0,0 +1,72 @@
+/*-
+ * Public Domain 2014-present MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef DATABASE_MODEL_H
+#define DATABASE_MODEL_H
+
+#include <map>
+#include <string>
+
+namespace test_harness {
+
+/* Key/Value type. */
+typedef std::string key_value_t;
+
+/* Representation of key states. */
+struct key_t {
+    bool exists;
+};
+
+/* Representation of a value. */
+struct value_t {
+    key_value_t value;
+};
+
+/* A collection is made of mapped Key objects. */
+struct collection_t {
+    std::map<key_value_t, key_t> keys;
+    std::map<key_value_t, value_t> *values;
+};
+
+/* Representation of the collections in memory. */
+class database {
+    public:
+    const std::vector<std::string>
+    get_collection_names() const
+    {
+        std::vector<std::string> collection_names;
+        for (auto const &it : collections)
+            collection_names.push_back(it.first);
+        return (collection_names);
+    }
+
+    std::map<std::string, collection_t> collections;
+};
+} // namespace test_harness
+
+#endif

test/cppsuite/test_harness/test.h

@@ -139,7 +139,7 @@ class test {
         if (_workload_tracking->is_enabled()) {
             workload_validation wv;
             is_success = wv.validate(_workload_tracking->get_operation_table_name(),
-              _workload_tracking->get_schema_table_name());
+              _workload_tracking->get_schema_table_name(), _workload_generator->get_database());
         }
 
         debug_print(is_success ? "SUCCESS" : "FAILED", DEBUG_INFO);

test/cppsuite/test_harness/workload_generator.h

@@ -33,6 +33,7 @@
 #include <atomic>
 #include <map>
 
+#include "database_model.h"
 #include "random_generator.h"
 #include "workload_tracking.h"
 
@@ -68,44 +69,56 @@ class workload_generator : public component {
      *      - Open a cursor on each collection.
      *      - Insert m key/value pairs in each collection. Values are random strings which size is
      * defined by the configuration.
+     *      - Store in memory the created collections and the generated keys that were inserted.
      */
     void
-    populate()
+    populate(database &database)
     {
         WT_CURSOR *cursor;
         WT_SESSION *session;
         wt_timestamp_t ts;
-        int64_t collection_count, key_count, value_size;
-        std::string collection_name, config, generated_value, home;
+        int64_t collection_count, key_count, key_cpt, key_size, value_size;
+        std::string collection_name, config, home;
+        key_value_t generated_key, generated_value;
         bool ts_enabled = _timestamp_manager->is_enabled();
 
         cursor = nullptr;
-        collection_count = key_count = value_size = 0;
-        collection_name = "";
+        collection_count = key_count = key_size = value_size = 0;
 
         /* Get a session. */
         session = connection_manager::instance().create_session();
         /* Create n collections as per the configuration and store each collection name. */
         testutil_check(_config->get_int(COLLECTION_COUNT, collection_count));
         for (int i = 0; i < collection_count; ++i) {
             collection_name = "table:collection" + std::to_string(i);
-            testutil_check(session->create(session, collection_name.c_str(), DEFAULT_TABLE_SCHEMA));
+            database.collections[collection_name] = {};
+            testutil_check(
+              session->create(session, collection_name.c_str(), DEFAULT_FRAMEWORK_SCHEMA));
             ts = _timestamp_manager->get_next_ts();
             testutil_check(_tracking->save(tracking_operation::CREATE, collection_name, 0, "", ts));
-            _collection_names.push_back(collection_name);
         }
         debug_print(std::to_string(collection_count) + " collections created", DEBUG_TRACE);
 
         /* Open a cursor on each collection and use the configuration to insert key/value pairs. */
         testutil_check(_config->get_int(KEY_COUNT, key_count));
         testutil_check(_config->get_int(VALUE_SIZE, value_size));
-        testutil_assert(value_size >= 0);
-        for (const auto &collection_name : _collection_names) {
+        testutil_assert(value_size > 0);
+        testutil_check(_config->get_int(KEY_SIZE, key_size));
+        testutil_assert(key_size > 0);
+        /* Keys must be unique. */
+        testutil_assert(key_count <= pow(10, key_size));
+
+        for (const auto &it_collections : database.collections) {
+            collection_name = it_collections.first;
+            key_cpt = 0;
             /* WiredTiger lets you open a cursor on a collection using the same pointer. When a
              * session is closed, WiredTiger APIs close the cursors too. */
             testutil_check(
               session->open_cursor(session, collection_name.c_str(), NULL, NULL, &cursor));
             for (size_t j = 0; j < key_count; ++j) {
+                /* Generation of a unique key. */
+                generated_key = number_to_string(key_size, key_cpt);
+                ++key_cpt;
                 /*
                  * Generation of a random string value using the size defined in the test
                  * configuration.
@@ -115,11 +128,16 @@ class workload_generator : public component {
                 ts = _timestamp_manager->get_next_ts();
                 if (ts_enabled)
                     testutil_check(session->begin_transaction(session, ""));
-                testutil_check(insert(cursor, collection_name, j + 1, generated_value.c_str(), ts));
+                testutil_check(insert(
+                  cursor, collection_name, generated_key.c_str(), generated_value.c_str(), ts));
                 if (ts_enabled) {
                     config = std::string(COMMIT_TS) + "=" + _timestamp_manager->decimal_to_hex(ts);
                     testutil_check(session->commit_transaction(session, config.c_str()));
                 }
+                /* Update the memory representation of the collections. */
+                database.collections[collection_name].keys[generated_key].exists = true;
+                /* Values are not stored here. */
+                database.collections[collection_name].values = nullptr;
             }
         }
         debug_print("Populate stage done", DEBUG_TRACE);
@@ -132,9 +150,10 @@ class workload_generator : public component {
         configuration *sub_config;
         int64_t read_threads, min_operation_per_transaction, max_operation_per_transaction,
           value_size;
+        std::vector<std::string> collection_names;
 
         /* Populate the database. */
-        populate();
+        populate(_database);
 
         /* Retrieve useful parameters from the test configuration. */
         testutil_check(_config->get_int(READ_THREADS, read_threads));
@@ -147,11 +166,13 @@ class workload_generator : public component {
 
         delete sub_config;
 
+        collection_names = _database.get_collection_names();
+
         /* Generate threads to execute read operations on the collections. */
         for (int i = 0; i < read_threads; ++i) {
-            thread_context *tc = new thread_context(_timestamp_manager, _tracking,
-              _collection_names, thread_operation::READ, max_operation_per_transaction,
-              min_operation_per_transaction, value_size);
+            thread_context *tc = new thread_context(_timestamp_manager, _tracking, collection_names,
+              thread_operation::READ, max_operation_per_transaction, min_operation_per_transaction,
+              value_size);
             _workers.push_back(tc);
             _thread_manager.add_thread(tc, &execute_operation);
         }
@@ -167,6 +188,12 @@ class workload_generator : public component {
         debug_print("Workload generator: run stage done", DEBUG_TRACE);
     }
 
+    database &
+    get_database()
+    {
+        return _database;
+    }
+
     /* Workload threaded operations. */
     static void
     execute_operation(thread_context &context)
@@ -205,8 +232,9 @@ class workload_generator : public component {
         WT_CURSOR *cursor;
         wt_timestamp_t ts;
         std::vector<WT_CURSOR *> cursors;
+        std::string collection_name;
         std::vector<std::string> collection_names;
-        std::string generated_value;
+        key_value_t generated_value, key;
         bool has_committed = true;
         int64_t cpt, value_size = context.get_value_size();
 
@@ -223,11 +251,13 @@ class workload_generator : public component {
             context.begin_transaction(session, "");
             ts = context.set_commit_timestamp(session);
             cpt = 0;
+            /* The key to update is hard coded to 1 for now. */
+            key = 1;
             for (const auto &it : cursors) {
+                collection_name = collection_names[cpt];
                 generated_value =
                   random_generator::random_generator::instance().generate_string(value_size);
-                /* Key is hard coded for now. */
-                testutil_check(update(context.get_tracking(), it, collection_names[cpt], 1,
+                testutil_check(update(context.get_tracking(), it, collection_name, key.c_str(),
                   generated_value.c_str(), ts));
                 ++cpt;
             }
@@ -265,7 +295,8 @@ class workload_generator : public component {
     /* WiredTiger APIs wrappers for single operations. */
     template <typename K, typename V>
     int
-    insert(WT_CURSOR *cursor, const std::string &collection_name, K key, V value, wt_timestamp_t ts)
+    insert(WT_CURSOR *cursor, const std::string &collection_name, const K &key, const V &value,
+      wt_timestamp_t ts)
     {
         int error_code;
 
@@ -322,7 +353,22 @@ class workload_generator : public component {
     }
 
     private:
-    std::vector<std::string> _collection_names;
+    /*
+     * Convert a number to a string. If the resulting string is less than the given length, padding
+     * of '0' is added.
+     */
+    static std::string
+    number_to_string(uint64_t size, uint64_t value)
+    {
+        std::string str, value_str = std::to_string(value);
+        testutil_assert(size >= value_str.size());
+        uint64_t diff = size - value_str.size();
+        std::string s(diff, '0');
+        str = s.append(value_str);
+        return (str);
+    }
+
+    database _database;
     thread_manager _thread_manager;
     timestamp_manager *_timestamp_manager;
     workload_tracking *_tracking;