diff --git a/db_stress_tool/batched_ops_stress.cc b/db_stress_tool/batched_ops_stress.cc index e4e9f664fb8..a5760207e95 100644 --- a/db_stress_tool/batched_ops_stress.cc +++ b/db_stress_tool/batched_ops_stress.cc @@ -308,6 +308,10 @@ class BatchedOpsStressTest : public StressTest { std::array column_results; std::array attribute_group_results; + std::string error_msg_header = FLAGS_use_attribute_group + ? "GetEntity (AttributeGroup) error" + : "GetEntity error"; + for (size_t i = 0; i < num_keys; ++i) { const std::string key = std::to_string(i) + key_suffix; @@ -323,7 +327,8 @@ class BatchedOpsStressTest : public StressTest { } if (!s.ok() && !s.IsNotFound()) { - fprintf(stderr, "GetEntity error: %s\n", s.ToString().c_str()); + fprintf(stderr, "%s: %s\n", error_msg_header.c_str(), + s.ToString().c_str()); thread->stats.AddErrors(1); } else if (s.IsNotFound()) { thread->stats.AddGets(1, 0); @@ -343,9 +348,8 @@ class BatchedOpsStressTest : public StressTest { : column_results[i].columns(); if (!CompareColumns(columns_to_compare, columns)) { - fprintf(stderr, - "GetEntity error: inconsistent entities for key %s: %s, %s\n", - StringToHex(key_suffix).c_str(), + fprintf(stderr, "%s: inconsistent entities for key %s: %s, %s\n", + error_msg_header.c_str(), StringToHex(key_suffix).c_str(), WideColumnsToHex(columns_to_compare).c_str(), WideColumnsToHex(columns).c_str()); } @@ -360,20 +364,18 @@ class BatchedOpsStressTest : public StressTest { if (value.empty() || value[value.size() - 1] != expected) { fprintf(stderr, - "GetEntity error: incorrect column value for key " + "%s: incorrect column value for key " "%s, entity %s, column value %s, expected %c\n", - StringToHex(key_suffix).c_str(), + error_msg_header.c_str(), StringToHex(key_suffix).c_str(), WideColumnsToHex(columns).c_str(), value.ToString(/* hex */ true).c_str(), expected); } } if (!VerifyWideColumns(columns)) { - fprintf( - stderr, - "GetEntity error: inconsistent columns for key %s, entity %s\n", - StringToHex(key_suffix).c_str(), - WideColumnsToHex(columns).c_str()); + fprintf(stderr, "%s: inconsistent columns for key %s, entity %s\n", + error_msg_header.c_str(), StringToHex(key_suffix).c_str(), + WideColumnsToHex(columns).c_str()); } } } @@ -407,66 +409,140 @@ class BatchedOpsStressTest : public StressTest { std::array keys; std::array key_slices; - std::array results; - std::array statuses; for (size_t j = 0; j < num_prefixes; ++j) { keys[j] = std::to_string(j) + key_suffix; key_slices[j] = keys[j]; } - db_->MultiGetEntity(read_opts_copy, cfh, num_prefixes, key_slices.data(), - results.data(), statuses.data()); + if (FLAGS_use_attribute_group) { + // AttributeGroup MultiGetEntity verification + + std::vector results; + results.reserve(num_prefixes); + for (size_t j = 0; j < num_prefixes; ++j) { + PinnableAttributeGroups attribute_groups; + attribute_groups.emplace_back(cfh); + results.emplace_back(std::move(attribute_groups)); + } + db_->MultiGetEntity(read_opts_copy, num_prefixes, key_slices.data(), + results.data()); + + const WideColumns& cmp_columns = results[0][0].columns(); + + for (size_t j = 0; j < num_prefixes; ++j) { + const auto& attribute_groups = results[j]; + assert(attribute_groups.size() == 1); + const Status& s = attribute_groups[0].status(); + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "MultiGetEntity (AttributeGroup) error: %s\n", + s.ToString().c_str()); + thread->stats.AddErrors(1); + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + } else { + thread->stats.AddGets(1, 1); + } - for (size_t j = 0; j < num_prefixes; ++j) { - const Status& s = statuses[j]; + const WideColumns& columns = results[j][0].columns(); + if (!CompareColumns(cmp_columns, columns)) { + fprintf(stderr, + "MultiGetEntity (AttributeGroup) error: inconsistent " + "entities for key %s: %s, " + "%s\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(cmp_columns).c_str(), + WideColumnsToHex(columns).c_str()); + } + if (!columns.empty()) { + // The last character of each column value should be 'j' as a + // decimal digit + const char expected = static_cast('0' + j); + + for (const auto& column : columns) { + const Slice& value = column.value(); + + if (value.empty() || value[value.size() - 1] != expected) { + fprintf(stderr, + "MultiGetEntity (AttributeGroup) error: incorrect " + "column value for key " + "%s, entity %s, column value %s, expected %c\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(columns).c_str(), + value.ToString(/* hex */ true).c_str(), expected); + } + } - if (!s.ok() && !s.IsNotFound()) { - fprintf(stderr, "MultiGetEntity error: %s\n", s.ToString().c_str()); - thread->stats.AddErrors(1); - } else if (s.IsNotFound()) { - thread->stats.AddGets(1, 0); - } else { - thread->stats.AddGets(1, 1); + if (!VerifyWideColumns(columns)) { + fprintf(stderr, + "MultiGetEntity (AttributeGroup) error: inconsistent " + "columns for key %s, " + "entity %s\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(columns).c_str()); + } + } } + } else { + // Non-AttributeGroup MultiGetEntity verification + + std::array results; + std::array statuses; + + db_->MultiGetEntity(read_opts_copy, cfh, num_prefixes, + key_slices.data(), results.data(), statuses.data()); const WideColumns& cmp_columns = results[0].columns(); - const WideColumns& columns = results[j].columns(); - if (!CompareColumns(cmp_columns, columns)) { - fprintf(stderr, - "MultiGetEntity error: inconsistent entities for key %s: %s, " - "%s\n", - StringToHex(key_suffix).c_str(), - WideColumnsToHex(cmp_columns).c_str(), - WideColumnsToHex(columns).c_str()); - } + for (size_t j = 0; j < num_prefixes; ++j) { + const Status& s = statuses[j]; + + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "MultiGetEntity error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + } else { + thread->stats.AddGets(1, 1); + } + const WideColumns& columns = results[j].columns(); - if (!columns.empty()) { - // The last character of each column value should be 'j' as a decimal - // digit - const char expected = static_cast('0' + j); + if (!CompareColumns(cmp_columns, columns)) { + fprintf( + stderr, + "MultiGetEntity error: inconsistent entities for key %s: %s, " + "%s\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(cmp_columns).c_str(), + WideColumnsToHex(columns).c_str()); + } - for (const auto& column : columns) { - const Slice& value = column.value(); + if (!columns.empty()) { + // The last character of each column value should be 'j' as a + // decimal digit + const char expected = static_cast('0' + j); + + for (const auto& column : columns) { + const Slice& value = column.value(); + + if (value.empty() || value[value.size() - 1] != expected) { + fprintf(stderr, + "MultiGetEntity error: incorrect column value for key " + "%s, entity %s, column value %s, expected %c\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(columns).c_str(), + value.ToString(/* hex */ true).c_str(), expected); + } + } - if (value.empty() || value[value.size() - 1] != expected) { + if (!VerifyWideColumns(columns)) { fprintf(stderr, - "MultiGetEntity error: incorrect column value for key " - "%s, entity %s, column value %s, expected %c\n", + "MultiGetEntity error: inconsistent columns for key %s, " + "entity %s\n", StringToHex(key_suffix).c_str(), - WideColumnsToHex(columns).c_str(), - value.ToString(/* hex */ true).c_str(), expected); + WideColumnsToHex(columns).c_str()); } } - - if (!VerifyWideColumns(columns)) { - fprintf(stderr, - "MultiGetEntity error: inconsistent columns for key %s, " - "entity %s\n", - StringToHex(key_suffix).c_str(), - WideColumnsToHex(columns).c_str()); - } } } } diff --git a/db_stress_tool/cf_consistency_stress.cc b/db_stress_tool/cf_consistency_stress.cc index bcc893c1c8c..cd0beabf359 100644 --- a/db_stress_tool/cf_consistency_stress.cc +++ b/db_stress_tool/cf_consistency_stress.cc @@ -506,95 +506,220 @@ class CfConsistencyStressTest : public StressTest { const size_t num_keys = rand_keys.size(); - for (size_t i = 0; i < num_keys; ++i) { - const std::string key = Key(rand_keys[i]); + if (FLAGS_use_attribute_group) { + // AttributeGroup MultiGetEntity verification + + std::vector results; + std::vector key_slices; + std::vector key_strs; + results.reserve(num_keys); + key_slices.reserve(num_keys); + key_strs.reserve(num_keys); + + for (size_t i = 0; i < num_keys; ++i) { + key_strs.emplace_back(Key(rand_keys[i])); + key_slices.emplace_back(key_strs.back()); + PinnableAttributeGroups attribute_groups; + for (auto* cfh : cfhs) { + attribute_groups.emplace_back(cfh); + } + results.emplace_back(std::move(attribute_groups)); + } + db_->MultiGetEntity(read_opts_copy, num_keys, key_slices.data(), + results.data()); - std::vector key_slices(num_cfs, key); - std::vector results(num_cfs); - std::vector statuses(num_cfs); + bool is_consistent = true; - db_->MultiGetEntity(read_opts_copy, num_cfs, cfhs.data(), - key_slices.data(), results.data(), statuses.data()); + for (size_t i = 0; i < num_keys; ++i) { + const auto& result = results[i]; + const Status& cmp_s = result[0].status(); + const WideColumns& cmp_columns = result[0].columns(); - bool is_consistent = true; + bool has_error = false; - for (size_t j = 0; j < num_cfs; ++j) { - const Status& s = statuses[j]; - const Status& cmp_s = statuses[0]; - const WideColumns& columns = results[j].columns(); - const WideColumns& cmp_columns = results[0].columns(); + for (size_t j = 0; j < num_cfs; ++j) { + const Status& s = result[j].status(); + const WideColumns& columns = result[j].columns(); + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "TestMultiGetEntity (AttributeGroup) error: %s\n", + s.ToString().c_str()); + thread->stats.AddErrors(1); + has_error = true; + break; + } - if (!s.ok() && !s.IsNotFound()) { - fprintf(stderr, "TestMultiGetEntity error: %s\n", - s.ToString().c_str()); - thread->stats.AddErrors(1); - break; - } + assert(cmp_s.ok() || cmp_s.IsNotFound()); + + if (s.IsNotFound()) { + if (cmp_s.ok()) { + fprintf(stderr, + "MultiGetEntity (AttributeGroup) returns different " + "results for key %s: CF %s " + "returns entity %s, CF %s returns not found\n", + key_slices[i].ToString(true).c_str(), + column_family_names_[0].c_str(), + WideColumnsToHex(cmp_columns).c_str(), + column_family_names_[j].c_str()); + is_consistent = false; + break; + } - assert(cmp_s.ok() || cmp_s.IsNotFound()); + continue; + } - if (s.IsNotFound()) { - if (cmp_s.ok()) { - fprintf( - stderr, - "MultiGetEntity returns different results for key %s: CF %s " - "returns entity %s, CF %s returns not found\n", - StringToHex(key).c_str(), column_family_names_[0].c_str(), - WideColumnsToHex(cmp_columns).c_str(), - column_family_names_[j].c_str()); + assert(s.ok()); + if (cmp_s.IsNotFound()) { + fprintf(stderr, + "MultiGetEntity (AttributeGroup) returns different results " + "for key %s: CF %s " + "returns not found, CF %s returns entity %s\n", + key_slices[i].ToString(true).c_str(), + column_family_names_[0].c_str(), + column_family_names_[j].c_str(), + WideColumnsToHex(columns).c_str()); is_consistent = false; break; } - continue; - } + if (columns != cmp_columns) { + fprintf(stderr, + "MultiGetEntity (AttributeGroup) returns different results " + "for key %s: CF %s " + "returns entity %s, CF %s returns entity %s\n", + key_slices[i].ToString(true).c_str(), + column_family_names_[0].c_str(), + WideColumnsToHex(cmp_columns).c_str(), + column_family_names_[j].c_str(), + WideColumnsToHex(columns).c_str()); + is_consistent = false; + break; + } - assert(s.ok()); - if (cmp_s.IsNotFound()) { + if (!VerifyWideColumns(columns)) { + fprintf(stderr, + "MultiGetEntity (AttributeGroup) error: inconsistent " + "columns for key %s, " + "entity %s\n", + key_slices[i].ToString(true).c_str(), + WideColumnsToHex(columns).c_str()); + is_consistent = false; + break; + } + } + if (has_error) { + break; + } else if (!is_consistent) { fprintf(stderr, - "MultiGetEntity returns different results for key %s: CF %s " - "returns not found, CF %s returns entity %s\n", - StringToHex(key).c_str(), column_family_names_[0].c_str(), - column_family_names_[j].c_str(), - WideColumnsToHex(columns).c_str()); - is_consistent = false; + "TestMultiGetEntity (AttributeGroup) error: results are not " + "consistent\n"); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); break; + } else if (cmp_s.ok()) { + thread->stats.AddGets(1, 1); + } else if (cmp_s.IsNotFound()) { + thread->stats.AddGets(1, 0); } + } - if (columns != cmp_columns) { - fprintf(stderr, + } else { + // Non-AttributeGroup MultiGetEntity verification + + for (size_t i = 0; i < num_keys; ++i) { + const std::string key = Key(rand_keys[i]); + + std::vector key_slices(num_cfs, key); + std::vector results(num_cfs); + std::vector statuses(num_cfs); + + db_->MultiGetEntity(read_opts_copy, num_cfs, cfhs.data(), + key_slices.data(), results.data(), statuses.data()); + + bool is_consistent = true; + + const Status& cmp_s = statuses[0]; + const WideColumns& cmp_columns = results[0].columns(); + + for (size_t j = 0; j < num_cfs; ++j) { + const Status& s = statuses[j]; + const WideColumns& columns = results[j].columns(); + + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "TestMultiGetEntity error: %s\n", + s.ToString().c_str()); + thread->stats.AddErrors(1); + break; + } + + assert(cmp_s.ok() || cmp_s.IsNotFound()); + + if (s.IsNotFound()) { + if (cmp_s.ok()) { + fprintf( + stderr, "MultiGetEntity returns different results for key %s: CF %s " - "returns entity %s, CF %s returns entity %s\n", + "returns entity %s, CF %s returns not found\n", StringToHex(key).c_str(), column_family_names_[0].c_str(), WideColumnsToHex(cmp_columns).c_str(), - column_family_names_[j].c_str(), - WideColumnsToHex(columns).c_str()); - is_consistent = false; - break; + column_family_names_[j].c_str()); + is_consistent = false; + break; + } + + continue; + } + + assert(s.ok()); + if (cmp_s.IsNotFound()) { + fprintf( + stderr, + "MultiGetEntity returns different results for key %s: CF %s " + "returns not found, CF %s returns entity %s\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + column_family_names_[j].c_str(), + WideColumnsToHex(columns).c_str()); + is_consistent = false; + break; + } + + if (columns != cmp_columns) { + fprintf( + stderr, + "MultiGetEntity returns different results for key %s: CF %s " + "returns entity %s, CF %s returns entity %s\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + WideColumnsToHex(cmp_columns).c_str(), + column_family_names_[j].c_str(), + WideColumnsToHex(columns).c_str()); + is_consistent = false; + break; + } + + if (!VerifyWideColumns(columns)) { + fprintf(stderr, + "MultiGetEntity error: inconsistent columns for key %s, " + "entity %s\n", + StringToHex(key).c_str(), + WideColumnsToHex(columns).c_str()); + is_consistent = false; + break; + } } - if (!VerifyWideColumns(columns)) { + if (!is_consistent) { fprintf(stderr, - "MultiGetEntity error: inconsistent columns for key %s, " - "entity %s\n", - StringToHex(key).c_str(), WideColumnsToHex(columns).c_str()); - is_consistent = false; + "TestMultiGetEntity error: results are not consistent\n"); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); break; + } else if (statuses[0].ok()) { + thread->stats.AddGets(1, 1); + } else if (statuses[0].IsNotFound()) { + thread->stats.AddGets(1, 0); } } - - if (!is_consistent) { - fprintf(stderr, - "TestMultiGetEntity error: results are not consistent\n"); - thread->stats.AddErrors(1); - // Fail fast to preserve the DB state. - thread->shared->SetVerificationFailure(); - break; - } else if (statuses[0].ok()) { - thread->stats.AddGets(1, 1); - } else if (statuses[0].IsNotFound()) { - thread->stats.AddGets(1, 0); - } } } diff --git a/db_stress_tool/no_batched_ops_stress.cc b/db_stress_tool/no_batched_ops_stress.cc index 127bd974b61..c18cf506b64 100644 --- a/db_stress_tool/no_batched_ops_stress.cc +++ b/db_stress_tool/no_batched_ops_stress.cc @@ -1077,130 +1077,257 @@ class NonBatchedOpsStressTest : public StressTest { std::vector keys(num_keys); std::vector key_slices(num_keys); + if (fault_fs_guard) { + fault_fs_guard->EnableErrorInjection(); + SharedState::ignore_read_error = false; + } + for (size_t i = 0; i < num_keys; ++i) { keys[i] = Key(rand_keys[i]); key_slices[i] = keys[i]; } - std::vector results(num_keys); - std::vector statuses(num_keys); + int error_count = 0; - if (fault_fs_guard) { - fault_fs_guard->EnableErrorInjection(); - SharedState::ignore_read_error = false; - } + auto handle_result = [](ThreadState* _thread, const Status& s, + bool is_consistent, int err_count) { + if (!is_consistent) { + fprintf(stderr, + "TestMultiGetEntity%s error: results are not consistent\n", + FLAGS_use_attribute_group ? "(AttributeGroup)" : ""); + _thread->stats.AddErrors(1); + // Fail fast to preserve the DB state + _thread->shared->SetVerificationFailure(); + } else if (s.ok()) { + _thread->stats.AddGets(1, 1); + } else if (s.IsNotFound()) { + _thread->stats.AddGets(1, 0); + } else { + if (err_count == 0) { + fprintf(stderr, "MultiGetEntity%s error: %s\n", + FLAGS_use_attribute_group ? "(AttributeGroup)" : "", + s.ToString().c_str()); + _thread->stats.AddErrors(1); + } else { + _thread->stats.AddVerifiedErrors(1); + } + } + }; - db_->MultiGetEntity(read_opts_copy, cfh, num_keys, key_slices.data(), - results.data(), statuses.data()); + if (FLAGS_use_attribute_group) { + // AttributeGroup MultiGetEntity verification - int error_count = 0; + std::vector results; + results.reserve(num_keys); + for (size_t i = 0; i < num_keys; ++i) { + PinnableAttributeGroups attribute_groups; + attribute_groups.emplace_back(cfh); + results.emplace_back(std::move(attribute_groups)); + } + db_->MultiGetEntity(read_opts_copy, num_keys, key_slices.data(), + results.data()); - if (fault_fs_guard) { - error_count = fault_fs_guard->GetAndResetErrorCount(); + if (fault_fs_guard) { + error_count = fault_fs_guard->GetAndResetErrorCount(); - if (error_count && !SharedState::ignore_read_error) { - int stat_nok = 0; - for (const auto& s : statuses) { - if (!s.ok() && !s.IsNotFound()) { - stat_nok++; + if (error_count && !SharedState::ignore_read_error) { + int stat_nok = 0; + for (size_t i = 0; i < num_keys; ++i) { + const Status& s = results[i][0].status(); + if (!s.ok() && !s.IsNotFound()) { + stat_nok++; + } } - } - if (stat_nok < error_count) { - // Grab mutex so multiple threads don't try to print the - // stack trace at the same time - assert(thread->shared); - MutexLock l(thread->shared->GetMutex()); + if (stat_nok < error_count) { + // Grab mutex so multiple threads don't try to print the + // stack trace at the same time + assert(thread->shared); + MutexLock l(thread->shared->GetMutex()); - fprintf(stderr, "Didn't get expected error from MultiGetEntity\n"); - fprintf(stderr, "num_keys %zu Expected %d errors, seen %d\n", - num_keys, error_count, stat_nok); - fprintf(stderr, "Call stack that injected the fault\n"); - fault_fs_guard->PrintFaultBacktrace(); - std::terminate(); + fprintf(stderr, + "Didn't get expected error from MultiGetEntity " + "(AttributeGroup)\n"); + fprintf(stderr, "num_keys %zu Expected %d errors, seen %d\n", + num_keys, error_count, stat_nok); + fprintf(stderr, "Call stack that injected the fault\n"); + fault_fs_guard->PrintFaultBacktrace(); + std::terminate(); + } } + fault_fs_guard->DisableErrorInjection(); } - fault_fs_guard->DisableErrorInjection(); - } + // Compare against non-attribute-group GetEntity result + const bool check_get_entity = + !error_count && FLAGS_check_multiget_entity_consistency; - const bool check_get_entity = - !error_count && FLAGS_check_multiget_entity_consistency; + for (size_t i = 0; i < num_keys; ++i) { + assert(results[i].size() == 1); + const Status& s = results[i][0].status(); - for (size_t i = 0; i < num_keys; ++i) { - const Status& s = statuses[i]; - - bool is_consistent = true; + bool is_consistent = true; - if (s.ok() && !VerifyWideColumns(results[i].columns())) { - fprintf( - stderr, - "error : inconsistent columns returned by MultiGetEntity for key " - "%s: %s\n", - StringToHex(keys[i]).c_str(), - WideColumnsToHex(results[i].columns()).c_str()); - is_consistent = false; - } else if (check_get_entity && (s.ok() || s.IsNotFound())) { - PinnableWideColumns cmp_result; - ThreadStatusUtil::SetThreadOperation( - ThreadStatus::OperationType::OP_GETENTITY); - const Status cmp_s = - db_->GetEntity(read_opts_copy, cfh, key_slices[i], &cmp_result); - - if (!cmp_s.ok() && !cmp_s.IsNotFound()) { - fprintf(stderr, "GetEntity error: %s\n", cmp_s.ToString().c_str()); + if (s.ok() && !VerifyWideColumns(results[i][0].columns())) { + fprintf(stderr, + "error : inconsistent columns returned by MultiGetEntity " + "(AttributeGroup) for key " + "%s: %s\n", + StringToHex(keys[i]).c_str(), + WideColumnsToHex(results[i][0].columns()).c_str()); is_consistent = false; - } else if (cmp_s.IsNotFound()) { - if (s.ok()) { - fprintf(stderr, - "Inconsistent results for key %s: MultiGetEntity returned " - "ok, GetEntity returned not found\n", - StringToHex(keys[i]).c_str()); + } else if (check_get_entity && (s.ok() || s.IsNotFound())) { + PinnableWideColumns cmp_result; + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_GETENTITY); + const Status cmp_s = + db_->GetEntity(read_opts_copy, cfh, key_slices[i], &cmp_result); + + if (!cmp_s.ok() && !cmp_s.IsNotFound()) { + fprintf(stderr, "GetEntity error: %s\n", cmp_s.ToString().c_str()); is_consistent = false; + } else if (cmp_s.IsNotFound()) { + if (s.ok()) { + fprintf(stderr, + "Inconsistent results for key %s: MultiGetEntity " + "(AttributeGroup) returned " + "ok, GetEntity returned not found\n", + StringToHex(keys[i]).c_str()); + is_consistent = false; + } + } else { + assert(cmp_s.ok()); + + if (s.IsNotFound()) { + fprintf(stderr, + "Inconsistent results for key %s: MultiGetEntity " + "(AttributeGroup) returned " + "not found, GetEntity returned ok\n", + StringToHex(keys[i]).c_str()); + is_consistent = false; + } else { + assert(s.ok()); + + if (results[i][0].columns() != cmp_result.columns()) { + fprintf(stderr, + "Inconsistent results for key %s: MultiGetEntity " + "(AttributeGroup) returned " + "%s, GetEntity returned %s\n", + StringToHex(keys[i]).c_str(), + WideColumnsToHex(results[i][0].columns()).c_str(), + WideColumnsToHex(cmp_result.columns()).c_str()); + is_consistent = false; + } + } } - } else { - assert(cmp_s.ok()); + } + handle_result(thread, s, is_consistent, error_count); + if (!is_consistent) { + break; + } + } + } else { + // Non-AttributeGroup MultiGetEntity verification - if (s.IsNotFound()) { - fprintf(stderr, - "Inconsistent results for key %s: MultiGetEntity returned " - "not found, GetEntity returned ok\n", - StringToHex(keys[i]).c_str()); + std::vector results(num_keys); + std::vector statuses(num_keys); + + db_->MultiGetEntity(read_opts_copy, cfh, num_keys, key_slices.data(), + results.data(), statuses.data()); + + if (fault_fs_guard) { + error_count = fault_fs_guard->GetAndResetErrorCount(); + + if (error_count && !SharedState::ignore_read_error) { + int stat_nok = 0; + for (const auto& s : statuses) { + if (!s.ok() && !s.IsNotFound()) { + stat_nok++; + } + } + + if (stat_nok < error_count) { + // Grab mutex so multiple threads don't try to print the + // stack trace at the same time + assert(thread->shared); + MutexLock l(thread->shared->GetMutex()); + + fprintf(stderr, "Didn't get expected error from MultiGetEntity\n"); + fprintf(stderr, "num_keys %zu Expected %d errors, seen %d\n", + num_keys, error_count, stat_nok); + fprintf(stderr, "Call stack that injected the fault\n"); + fault_fs_guard->PrintFaultBacktrace(); + std::terminate(); + } + } + + fault_fs_guard->DisableErrorInjection(); + } + + const bool check_get_entity = + !error_count && FLAGS_check_multiget_entity_consistency; + + for (size_t i = 0; i < num_keys; ++i) { + const Status& s = statuses[i]; + + bool is_consistent = true; + + if (s.ok() && !VerifyWideColumns(results[i].columns())) { + fprintf( + stderr, + "error : inconsistent columns returned by MultiGetEntity for key " + "%s: %s\n", + StringToHex(keys[i]).c_str(), + WideColumnsToHex(results[i].columns()).c_str()); + is_consistent = false; + } else if (check_get_entity && (s.ok() || s.IsNotFound())) { + PinnableWideColumns cmp_result; + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_GETENTITY); + const Status cmp_s = + db_->GetEntity(read_opts_copy, cfh, key_slices[i], &cmp_result); + + if (!cmp_s.ok() && !cmp_s.IsNotFound()) { + fprintf(stderr, "GetEntity error: %s\n", cmp_s.ToString().c_str()); is_consistent = false; + } else if (cmp_s.IsNotFound()) { + if (s.ok()) { + fprintf( + stderr, + "Inconsistent results for key %s: MultiGetEntity returned " + "ok, GetEntity returned not found\n", + StringToHex(keys[i]).c_str()); + is_consistent = false; + } } else { - assert(s.ok()); + assert(cmp_s.ok()); - if (results[i] != cmp_result) { + if (s.IsNotFound()) { fprintf( stderr, "Inconsistent results for key %s: MultiGetEntity returned " - "%s, GetEntity returned %s\n", - StringToHex(keys[i]).c_str(), - WideColumnsToHex(results[i].columns()).c_str(), - WideColumnsToHex(cmp_result.columns()).c_str()); + "not found, GetEntity returned ok\n", + StringToHex(keys[i]).c_str()); is_consistent = false; + } else { + assert(s.ok()); + + if (results[i] != cmp_result) { + fprintf( + stderr, + "Inconsistent results for key %s: MultiGetEntity returned " + "%s, GetEntity returned %s\n", + StringToHex(keys[i]).c_str(), + WideColumnsToHex(results[i].columns()).c_str(), + WideColumnsToHex(cmp_result.columns()).c_str()); + is_consistent = false; + } } } } - } - - if (!is_consistent) { - fprintf(stderr, - "TestMultiGetEntity error: results are not consistent\n"); - thread->stats.AddErrors(1); - // Fail fast to preserve the DB state - thread->shared->SetVerificationFailure(); - break; - } else if (s.ok()) { - thread->stats.AddGets(1, 1); - } else if (s.IsNotFound()) { - thread->stats.AddGets(1, 0); - } else { - if (error_count == 0) { - fprintf(stderr, "MultiGetEntity error: %s\n", s.ToString().c_str()); - thread->stats.AddErrors(1); - } else { - thread->stats.AddVerifiedErrors(1); + handle_result(thread, s, is_consistent, error_count); + if (!is_consistent) { + break; } } }