diff --git a/HISTORY.md b/HISTORY.md index 18daf4bd4c0..2781c0ce49d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -17,6 +17,8 @@ ### Public API Change * Expose kTypeDeleteWithTimestamp in EntryType and update GetEntryType() accordingly. +* Added file_checksum and file_checksum_func_name to TableFileCreationInfo, which can pass the table file checksum information through the OnTableFileCreated callback during flush and compaction. + ## 6.12 (2020-07-28) ### Public API Change diff --git a/db/builder.cc b/db/builder.cc index 0a4adeb6fa6..8507135a8ea 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -106,6 +106,8 @@ Status BuildTable( std::string fname = TableFileName(ioptions.cf_paths, meta->fd.GetNumber(), meta->fd.GetPathId()); + std::string file_checksum = kUnknownFileChecksum; + std::string file_checksum_func_name = kUnknownFileChecksumFuncName; #ifndef ROCKSDB_LITE EventHelpers::NotifyTableFileCreationStarted( ioptions.listeners, dbname, column_family_name, fname, job_id, reason); @@ -133,7 +135,8 @@ Status BuildTable( if (!s.ok()) { EventHelpers::LogAndNotifyTableFileCreationFinished( event_logger, ioptions.listeners, dbname, column_family_name, fname, - job_id, meta->fd, kInvalidBlobFileNumber, tp, reason, s); + job_id, meta->fd, kInvalidBlobFileNumber, tp, reason, s, + file_checksum, file_checksum_func_name); return s; } file->SetIOPriority(io_priority); @@ -232,6 +235,8 @@ Status BuildTable( // Add the checksum information to file metadata. meta->file_checksum = file_writer->GetFileChecksum(); meta->file_checksum_func_name = file_writer->GetFileChecksumFuncName(); + file_checksum = meta->file_checksum; + file_checksum_func_name = meta->file_checksum_func_name; } if (s.ok()) { @@ -292,7 +297,8 @@ Status BuildTable( // Output to event logger and fire events. EventHelpers::LogAndNotifyTableFileCreationFinished( event_logger, ioptions.listeners, dbname, column_family_name, fname, - job_id, meta->fd, meta->oldest_blob_file_number, tp, reason, s); + job_id, meta->fd, meta->oldest_blob_file_number, tp, reason, s, + file_checksum, file_checksum_func_name); return s; } diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index 5e0e5bd479f..e097dbdfd4b 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -1198,6 +1198,8 @@ Status CompactionJob::FinishCompactionOutputFile( ColumnFamilyData* cfd = sub_compact->compaction->column_family_data(); const Comparator* ucmp = cfd->user_comparator(); + std::string file_checksum = kUnknownFileChecksum; + std::string file_checksum_func_name = kUnknownFileChecksumFuncName; // Check for iterator errors Status s = input_status; @@ -1397,6 +1399,8 @@ Status CompactionJob::FinishCompactionOutputFile( meta->file_checksum = sub_compact->outfile->GetFileChecksum(); meta->file_checksum_func_name = sub_compact->outfile->GetFileChecksumFuncName(); + file_checksum = meta->file_checksum; + file_checksum_func_name = meta->file_checksum_func_name; } if (s.ok()) { s = io_s; @@ -1453,7 +1457,8 @@ Status CompactionJob::FinishCompactionOutputFile( EventHelpers::LogAndNotifyTableFileCreationFinished( event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname, job_id_, output_fd, oldest_blob_file_number, tp, - TableFileCreationReason::kCompaction, s); + TableFileCreationReason::kCompaction, s, file_checksum, + file_checksum_func_name); #ifndef ROCKSDB_LITE // Report new file to SstFileManagerImpl @@ -1582,7 +1587,8 @@ Status CompactionJob::OpenCompactionOutputFile( EventHelpers::LogAndNotifyTableFileCreationFinished( event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname, job_id_, FileDescriptor(), kInvalidBlobFileNumber, - TableProperties(), TableFileCreationReason::kCompaction, s); + TableProperties(), TableFileCreationReason::kCompaction, s, + kUnknownFileChecksum, kUnknownFileChecksumFuncName); return s; } diff --git a/db/event_helpers.cc b/db/event_helpers.cc index b7f49140910..0b92d569a92 100644 --- a/db/event_helpers.cc +++ b/db/event_helpers.cc @@ -71,14 +71,17 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished( const std::string& db_name, const std::string& cf_name, const std::string& file_path, int job_id, const FileDescriptor& fd, uint64_t oldest_blob_file_number, const TableProperties& table_properties, - TableFileCreationReason reason, const Status& s) { + TableFileCreationReason reason, const Status& s, + const std::string& file_checksum, + const std::string& file_checksum_func_name) { if (s.ok() && event_logger) { JSONWriter jwriter; AppendCurrentTime(&jwriter); jwriter << "cf_name" << cf_name << "job" << job_id << "event" << "table_file_creation" << "file_number" << fd.GetNumber() << "file_size" - << fd.GetFileSize(); + << fd.GetFileSize() << "file_checksum" << file_checksum + << "file_checksum_func_name" << file_checksum_func_name; // table_properties { @@ -154,6 +157,8 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished( info.table_properties = table_properties; info.reason = reason; info.status = s; + info.file_checksum = file_checksum; + info.file_checksum_func_name = file_checksum_func_name; for (auto& listener : listeners) { listener->OnTableFileCreated(info); } diff --git a/db/event_helpers.h b/db/event_helpers.h index 87cc1cb8c17..abc00981c1c 100644 --- a/db/event_helpers.h +++ b/db/event_helpers.h @@ -35,7 +35,9 @@ class EventHelpers { const std::string& db_name, const std::string& cf_name, const std::string& file_path, int job_id, const FileDescriptor& fd, uint64_t oldest_blob_file_number, const TableProperties& table_properties, - TableFileCreationReason reason, const Status& s); + TableFileCreationReason reason, const Status& s, + const std::string& file_checksum, + const std::string& file_checksum_func_name); static void LogAndNotifyTableFileDeletion( EventLogger* event_logger, int job_id, uint64_t file_number, const std::string& file_path, diff --git a/db/listener_test.cc b/db/listener_test.cc index c3df30d6a62..9239068ce9a 100644 --- a/db/listener_test.cc +++ b/db/listener_test.cc @@ -227,6 +227,8 @@ class TestFlushListener : public EventListener { ASSERT_GT(info.table_properties.raw_value_size, 0U); ASSERT_GT(info.table_properties.num_data_blocks, 0U); ASSERT_GT(info.table_properties.num_entries, 0U); + ASSERT_EQ(info.file_checksum, kUnknownFileChecksum); + ASSERT_EQ(info.file_checksum_func_name, kUnknownFileChecksumFuncName); #ifdef ROCKSDB_USING_THREAD_STATUS // Verify the id of the current thread that created this table @@ -751,6 +753,8 @@ class TableFileCreationListener : public EventListener { ASSERT_GT(info.cf_name.size(), 0U); ASSERT_GT(info.file_path.size(), 0U); ASSERT_GT(info.job_id, 0); + ASSERT_EQ(info.file_checksum, kUnknownFileChecksum); + ASSERT_EQ(info.file_checksum_func_name, kUnknownFileChecksumFuncName); if (info.status.ok()) { ASSERT_GT(info.table_properties.data_size, 0U); ASSERT_GT(info.table_properties.raw_key_size, 0U); diff --git a/include/rocksdb/listener.h b/include/rocksdb/listener.h index 19145e81624..1c15fc0fe47 100644 --- a/include/rocksdb/listener.h +++ b/include/rocksdb/listener.h @@ -58,6 +58,10 @@ struct TableFileCreationInfo : public TableFileCreationBriefInfo { TableProperties table_properties; // The status indicating whether the creation was successful or not. Status status; + // The checksum of the table file being created + std::string file_checksum; + // The checksum function name of checksum generator used for this table file + std::string file_checksum_func_name; }; enum class CompactionReason : int {