diff --git a/dbms/src/Storages/Page/V3/PageEntry.h b/dbms/src/Storages/Page/V3/PageEntry.h index c2c530f5203..277684de653 100644 --- a/dbms/src/Storages/Page/V3/PageEntry.h +++ b/dbms/src/Storages/Page/V3/PageEntry.h @@ -136,7 +136,7 @@ struct fmt::formatter return fmt::format_to( ctx.out(), - "PageEntry{{file: {}, offset: 0x{:X}, size: {}, checksum: 0x{:X}, tag: {}, field_offsets: [{}], " + "PageEntry{{file: {}, offset: {}, size: {}, checksum: 0x{:X}, tag: {}, field_offsets: [{}], " "checkpoint_info: {}}}", entry.file_id, entry.offset, diff --git a/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp index bcd6bb91470..404951f74fe 100644 --- a/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp @@ -13,14 +13,18 @@ // limitations under the License. #include +#include #include #include #include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -29,6 +33,7 @@ #include #include +#include #include #include @@ -47,12 +52,15 @@ struct ControlOptions CHECK_ALL_DATA_CRC = 4, DISPLAY_WAL_ENTRIES = 5, DISPLAY_REGION_INFO = 6, + DISPLAY_BLOB_DATA = 7, }; std::vector paths; DisplayType mode = DisplayType::DISPLAY_SUMMARY_INFO; UInt64 page_id = UINT64_MAX; - UInt32 blob_id = UINT32_MAX; + BlobFileId blob_id = INVALID_BLOBFILE_ID; + BlobFileOffset blob_offset = INVALID_BLOBFILE_OFFSET; + size_t blob_size = UINT64_MAX; UInt64 namespace_id = DB::TEST_NAMESPACE_ID; StorageType storage_type = StorageType::Unknown; // only useful for universal page storage UInt32 keyspace_id = NullspaceID; // only useful for universal page storage @@ -85,6 +93,7 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) 4 is check every data is valid 5 is dump entries in WAL log files 6 is display all region info + 7 is display blob data (in hex) )") // ("show_entries", value()->default_value(true), @@ -106,8 +115,14 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) value()->default_value(UINT64_MAX), "Query a single Page id, and print its version chain.") // ("blob_id,B", - value()->default_value(UINT32_MAX), - "Query a single Blob id, and print its data distribution.") // + value()->default_value(INVALID_BLOBFILE_ID), + "Specify the blob_id") // + ("blob_offset", + value()->default_value(INVALID_BLOBFILE_OFFSET), + "Specify the offset.") // + ("blob_size", + value()->default_value(0), + "Specify the size.") // // ("imitative,I", value()->default_value(true), @@ -140,7 +155,9 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) opt.paths = options["paths"].as>(); auto mode_int = options["mode"].as(); opt.page_id = options["page_id"].as(); - opt.blob_id = options["blob_id"].as(); + opt.blob_id = options["blob_id"].as(); + opt.blob_offset = options["blob_offset"].as(); + opt.blob_size = options["blob_size"].as(); opt.show_entries = options["show_entries"].as(); opt.check_fields = options["check_fields"].as(); auto storage_type_int = options["storage_type"].as(); @@ -251,29 +268,36 @@ class PageStorageControlV3 } private: - static int getPageStorageV3Info(Context & context, const ControlOptions & options) + static DB::PSDiskDelegatorPtr getDelegator(const ControlOptions & opts) { - DB::PSDiskDelegatorPtr delegator; - if (options.paths.size() == 1) + if (opts.paths.size() == 1) { - delegator = std::make_shared(options.paths[0]); + return std::make_shared(opts.paths[0]); } else { - delegator = std::make_shared(options.paths); + return std::make_shared(opts.paths); } + } - FileProviderPtr provider; - if (options.is_imitative) + static FileProviderPtr getProvider(Context & context, const ControlOptions & opts) + { + if (opts.is_imitative) { auto key_manager = std::make_shared(false); - provider = std::make_shared(key_manager, false); + return std::make_shared(key_manager, false); } else { - provider = context.getFileProvider(); + return context.getFileProvider(); } + } + + static int getPageStorageV3Info(Context & context, const ControlOptions & options) + { + DB::PSDiskDelegatorPtr delegator = getDelegator(options); + FileProviderPtr provider = getProvider(context, options); constexpr static std::string_view NAME = "PageStorageControlV3"; PageStorageConfig config; @@ -294,13 +318,13 @@ class PageStorageControlV3 { case ControlOptions::DisplayType::DISPLAY_SUMMARY_INFO: { - std::cout << getSummaryInfo(mvcc_table_directory, blob_store) << std::endl; + fmt::println("{}", getSummaryInfo(mvcc_table_directory, blob_store)); break; } case ControlOptions::DisplayType::DISPLAY_DIRECTORY_INFO: { - fmt::print( - "{}\n", + fmt::println( + "{}", getDirectoryInfo( mvcc_table_directory, opts.show_entries, @@ -312,25 +336,26 @@ class PageStorageControlV3 } case ControlOptions::DisplayType::DISPLAY_BLOBS_INFO: { - std::cout << getBlobsInfo(blob_store, opts.blob_id) << std::endl; + fmt::println("{}", getBlobsInfo(blob_store, opts.blob_id)); break; } case ControlOptions::DisplayType::CHECK_ALL_DATA_CRC: { if (opts.page_id != UINT64_MAX) { - std::cout << checkSinglePage( - mvcc_table_directory, - blob_store, - opts.storage_type, - opts.keyspace_id, - opts.namespace_id, - opts.page_id) - << std::endl; + fmt::println( + "{}", + checkSinglePage( + mvcc_table_directory, + blob_store, + opts.storage_type, + opts.keyspace_id, + opts.namespace_id, + opts.page_id)); } else { - std::cout << checkAllDataCrc(mvcc_table_directory, blob_store, opts.check_fields) << std::endl; + fmt::println("{}", checkAllDataCrc(mvcc_table_directory, blob_store, opts.check_fields)); } break; } @@ -338,14 +363,20 @@ class PageStorageControlV3 { if constexpr (std::is_same_v) { - std::cout << getAllRegionInfo(mvcc_table_directory) << std::endl; + fmt::println("{}", getAllRegionInfo(mvcc_table_directory)); } else { - std::cout << "Only UniversalPageStorage support this mode." << std::endl; + fmt::println("Only UniversalPageStorage support this mode."); } break; } + case ControlOptions::DisplayType::DISPLAY_BLOB_DATA: + { + String hex_data = getBlobData(blob_store, opts.blob_id, opts.blob_offset, opts.blob_size); + fmt::println("hex:{}", hex_data); + break; + } default: std::cout << "Invalid display mode." << std::endl; break; @@ -372,7 +403,7 @@ class PageStorageControlV3 return 0; } - static String getBlobsInfo(typename Trait::BlobStore & blob_store, UInt32 blob_id) + static String getBlobsInfo(typename Trait::BlobStore & blob_store, BlobFileId blob_id) { auto stat_info = [](const BlobStats::BlobStatPtr & stat, const String & path) { FmtBuffer stat_str; @@ -402,7 +433,7 @@ class PageStorageControlV3 { for (const auto & stat : stats) { - if (blob_id != UINT32_MAX) + if (blob_id != INVALID_BLOBFILE_ID) { if (stat->id == blob_id) { @@ -416,7 +447,7 @@ class PageStorageControlV3 } } - if (blob_id != UINT32_MAX) + if (blob_id != INVALID_BLOBFILE_ID) { stats_info.fmtAppend(" no found blob {}", blob_id); } @@ -821,6 +852,32 @@ class PageStorageControlV3 return error_msg.toString(); } + static String getBlobData( + typename Trait::BlobStore & blob_store, + BlobFileId blob_id, + BlobFileOffset offset, + size_t size) + { + auto page_id = []() { + if constexpr (std::is_same_v) + return PageIdV3Internal(0, 0); + else + return UniversalPageId(""); + }(); + char * buffer = new char[size]; + blob_store.read(page_id, blob_id, offset, buffer, size, nullptr, false); + + using ChecksumClass = Digest::CRC64; + ChecksumClass digest; + digest.update(buffer, size); + auto checksum = digest.checksum(); + fmt::print("checksum: 0x{:X}\n", checksum); + + auto hex_str = Redact::keyToHexString(buffer, size); + delete[] buffer; + return hex_str; + } + private: ControlOptions options; };