[#10076]: docdb: [DST] Enhance rocksdb UI page with info about OPTIONS

Summary:
Updated `TabletServerPathHandlers` to display RocksDB Regular/Intents options on tablet page within `Options` section. The content of the section is similar to RocksDB options file without comments header and file version.

Collapsed view:
{F20907}
Expanded view when pointer is hovering section's title ('Options'):
{F20908}

Test Plan:
Auto:
```
ybd --cxx-test rocksdb_options_test
ybd --cxx-test rocksdb_options_util_test
ybd --cxx-test rocksdb_column_family_test
ybd --cxx-test rocksdb_db_test
ybd --cxx-test rocksdb_compaction_job_stats_test
ybd --cxx-test rocksdb_db_dynamic_level_test
ybd --cxx-test rocksdb_fault_injection_test
```

Manual:
1.Run `yb-ctl` to start up a local cluster:
```
yb-ctl --rf=1 create
```
2. Run `yb-sample-apps.jar` for some time to upload the data
```
java -jar yb-sample-apps.jar --workload CassandraTransactionalKeyValue --nodes 127.0.0.1:9042
```
3. Flush memtables to force SST files appear
```
yb-admin flush_table ybdemo_keyspace cassandratransactionalkeyvalue
```
4. Open `http://127.0.0.1:9000/`, navigate to `Tablets`, open a tablet for `cassandratransactionalkeyvalue` table and then open `RocksDB` info
5. Expand `Options` section for Regular instance, make sure the content is similar to Regular instance options file, possible location:
```
~/yugabyte-data/node-1/disk-1/yb-data/tserver/data/rocksdb/table-%uuid0%/tablet-%uuid1%/OPTIONS-000009
```
6. Expand `Options` section for Intents instance, make sure the content is similar to Intents instance options file, possible location:
```
~/yugabyte-data/node-1/disk-1/yb-data/tserver/data/rocksdb/table-%uuid0%/tablet-%uuid1%.intents/OPTIONS-000009
```
7. Collapse/Expand `Options` for Regular/Intents sections to make sure expanding/collapsing works fine
8. Remember the value of `max_background_flushes` option both for Regular and Intents instances: `max_background_flushes=3`
9. Update the option by restarting the local cluster:
```
yb-ctl stop
yb-ctl start --tserver_flags rocksdb_max_background_flushes=1
```
10. Open `127.0.0.1:9000`, navigate to `Tablets`, open a tablet for `cassandratransactionalkeyvalue` table and then open `RocksDB` info
11. Expand `Options` and make sure the option has been changed: `max_background_flushes=1`
12. Make sure `Options` content for both Regular/Intents instances are similar to corresponding options files

Reviewers: sergei, bogdan, timur

Reviewed By: timur

Subscribers: mbautin, ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D14221

src/yb/rocksdb/db.h

@@ -848,6 +848,12 @@ class DB {
     GetColumnFamilyMetaData(DefaultColumnFamily(), metadata);
   }
 
+  // Obtains all column family options and corresponding names,
+  // dropped columns are not included into the resulting collections.
+  virtual void GetColumnFamiliesOptions(
+      std::vector<std::string>* column_family_names,
+      std::vector<ColumnFamilyOptions>* column_family_options) = 0;
+
   // Load table file located at "file_path" into "column_family", a pointer to
   // ExternalSstFileInfo can be used instead of "file_path" to do a blind add
   // that wont need to read the file, move_file can be set to true to

src/yb/rocksdb/db/column_family_test.cc

@@ -2588,6 +2588,80 @@ TEST_F(ColumnFamilyTest, LogSyncConflictFlush) {
   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
   Close();
 }
+
+TEST_F(ColumnFamilyTest, GetColumnFamiliesOptions) {
+  column_family_options_.arena_block_size = 4096;
+
+  // Source options
+  ColumnFamilyOptions src_d = column_family_options_;
+  ColumnFamilyOptions src_1;
+  ColumnFamilyOptions src_2;
+  ColumnFamilyOptions src_3;
+  src_1.arena_block_size = 2 * column_family_options_.arena_block_size;
+  src_2.arena_block_size = 3 * column_family_options_.arena_block_size;
+  src_3.arena_block_size = 4 * column_family_options_.arena_block_size;
+
+  using OptionsRef  = std::reference_wrapper<ColumnFamilyOptions>;
+  using Descriptors = std::map<std::string, OptionsRef>;
+
+  auto compare_with = [this](Descriptors src) {
+    std::vector<std::string> cf_names;
+    std::vector<ColumnFamilyOptions> cf_options;
+    db_->GetColumnFamiliesOptions(&cf_names, &cf_options);
+
+    // Comapre sizes
+    ASSERT_EQ(cf_names.size(), cf_options.size());
+    ASSERT_EQ(cf_names.size(), src.size());
+
+    // Keep sorted order for the case descriptors are stored in an unpredictable way
+    Descriptors dst;
+    for (size_t i = 0; i < cf_names.size(); ++i) {
+      dst.insert(std::make_pair(cf_names[i], std::ref(cf_options[i])));
+    }
+
+    // Compare options
+    for (auto it1 = src.begin(), it2 = dst.begin();
+         it1 != src.end() && it2 != dst.end(); ++it1, ++it2) {
+      ASSERT_EQ(it1->first, it2->first);
+      ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(it1->second, it2->second));
+    }
+  };
+
+  Open();
+  compare_with({{"default", std::ref(src_d)}});
+
+  CreateColumnFamilies({"1", "2"}, {src_1, src_2});
+  compare_with({{"default", std::ref(src_d)}, {"1", std::ref(src_1)}, {"2", std::ref(src_2)}});
+
+  DropColumnFamilies({1});
+  compare_with({{"default", std::ref(src_d)}, {"2", std::ref(src_2)}});
+
+  CreateColumnFamilies({"3"}, {src_3});
+  compare_with({{"default", std::ref(src_d)}, {"3", std::ref(src_3)}, {"2", std::ref(src_2)}});
+  Close();
+
+  src_3.arena_block_size += 1024;
+  Open({"3", "2", "default"}, {src_3, src_2, src_d});
+  compare_with({{"default", std::ref(src_d)}, {"2", std::ref(src_2)}, {"3", std::ref(src_3)}});
+
+  DropColumnFamilies({0});
+  compare_with({{"default", std::ref(src_d)}, {"2", std::ref(src_2)}});
+  Close();
+
+  db_options_.create_missing_column_families = true;
+  src_3.arena_block_size -= 1024;
+  Open({"1", "default", "2", "3"}, {src_1, src_d, src_2, src_3});
+  compare_with({{"default", std::ref(src_d)}, {"2", std::ref(src_2)},
+                {"3", std::ref(src_3)}, {"1", std::ref(src_1)}});
+
+  DropColumnFamilies({0, 2});
+  compare_with({{"3", std::ref(src_3)}, {"default", std::ref(src_d)}});
+
+  DropColumnFamilies({3});
+  compare_with({{"default", std::ref(src_d)}});
+  Close();
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

src/yb/rocksdb/db/db_impl.cc

@@ -5444,6 +5444,27 @@ Status DBImpl::GetPropertiesOfTablesInRange(ColumnFamilyHandle* column_family,
   return s;
 }
 
+void DBImpl::GetColumnFamiliesOptions(
+    std::vector<std::string>* column_family_names,
+    std::vector<ColumnFamilyOptions>* column_family_options) {
+  DCHECK(column_family_names);
+  DCHECK(column_family_options);
+  InstrumentedMutexLock lock(&mutex_);
+  GetColumnFamiliesOptionsUnlocked(column_family_names, column_family_options);
+}
+
+void DBImpl::GetColumnFamiliesOptionsUnlocked(
+    std::vector<std::string>* column_family_names,
+    std::vector<ColumnFamilyOptions>* column_family_options) {
+  for (auto cfd : *versions_->GetColumnFamilySet()) {
+    if (cfd->IsDropped()) {
+      continue;
+    }
+    column_family_names->push_back(cfd->GetName());
+    column_family_options->push_back(
+      BuildColumnFamilyOptions(*cfd->options(), *cfd->GetLatestMutableCFOptions()));
+  }
+}
 #endif  // ROCKSDB_LITE
 
 const std::string& DBImpl::GetName() const {
@@ -6480,14 +6501,7 @@ Status DBImpl::WriteOptionsFile() {
   std::vector<ColumnFamilyOptions> cf_opts;
 
   // This part requires mutex to protect the column family options
-  for (auto cfd : *versions_->GetColumnFamilySet()) {
-    if (cfd->IsDropped()) {
-      continue;
-    }
-    cf_names.push_back(cfd->GetName());
-    cf_opts.push_back(BuildColumnFamilyOptions(
-        *cfd->options(), *cfd->GetLatestMutableCFOptions()));
-  }
+  GetColumnFamiliesOptionsUnlocked(&cf_names, &cf_opts);
 
   // Unlock during expensive operations.  New writes cannot get here
   // because the single write thread ensures all new writes get queued.

src/yb/rocksdb/db/db_impl.h

@@ -237,6 +237,12 @@ class DBImpl : public DB {
       ColumnFamilyHandle* column_family,
       ColumnFamilyMetaData* metadata) override;
 
+  // Obtains all column family options and corresponding names,
+  // dropped columns are not included into the resulting collections.
+  virtual void GetColumnFamiliesOptions(
+      std::vector<std::string>* column_family_names,
+      std::vector<ColumnFamilyOptions>* column_family_options) override;
+
   // experimental API
   Status SuggestCompactRange(ColumnFamilyHandle* column_family,
                              const Slice* begin, const Slice* end);
@@ -1015,6 +1021,12 @@ class DBImpl : public DB {
       ColumnFamilyHandle* column_family, const Range* range, std::size_t n,
       TablePropertiesCollection* props) override;
 
+  // Obtains all column family options and corresponding names,
+  // dropped columns are not included into the resulting collections.
+  // REQUIREMENT: mutex_ must be held when calling this function.
+  void GetColumnFamiliesOptionsUnlocked(
+      std::vector<std::string>* column_family_names,
+      std::vector<ColumnFamilyOptions>* column_family_options);
 #endif  // ROCKSDB_LITE
 
   // Function that Get and KeyMayExist call with no_io true or false

src/yb/rocksdb/db/db_test.cc

@@ -4758,6 +4758,10 @@ class ModelDB: public DB {
   virtual void GetColumnFamilyMetaData(
       ColumnFamilyHandle* column_family,
       ColumnFamilyMetaData* metadata) override {}
+
+  virtual void GetColumnFamiliesOptions(
+      std::vector<std::string>* column_family_names,
+      std::vector<ColumnFamilyOptions>* column_family_options) override {}
 #endif  // ROCKSDB_LITE
 
   Status GetDbIdentity(std::string* identity) const override {

src/yb/rocksdb/util/options_parser.cc

@@ -49,7 +49,9 @@ static const std::string option_file_header =
 Status PersistRocksDBOptions(const DBOptions& db_opt,
                              const std::vector<std::string>& cf_names,
                              const std::vector<ColumnFamilyOptions>& cf_opts,
-                             const std::string& file_name, Env* env) {
+                             const std::string& file_name, Env* env,
+                             const IncludeHeader include_header,
+                             const IncludeFileVersion include_file_version) {
   TEST_SYNC_POINT("PersistRocksDBOptions:start");
   if (cf_names.size() != cf_opts.size()) {
     return STATUS(InvalidArgument,
@@ -61,37 +63,43 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
   if (!s.ok()) {
     return s;
   }
-  std::string options_file_content;
 
+  // Header section
+  if (include_header) {
+    RETURN_NOT_OK(writable->Append(option_file_header));
+  }
+
+  // Versions section
   RETURN_NOT_OK(writable->Append(
-      option_file_header + "[" +
-      opt_section_titles[kOptionSectionVersion] +
-      "]\n"
+      "[" + opt_section_titles[kOptionSectionVersion] + "]\n"
       "  yugabyte_version=" + yb::VersionInfo::GetShortVersionString() + "\n"));
-  RETURN_NOT_OK(writable->Append("  options_file_version=" +
-                                 ToString(ROCKSDB_OPTION_FILE_MAJOR) + "." +
-                                 ToString(ROCKSDB_OPTION_FILE_MINOR) + "\n"));
-  RETURN_NOT_OK(writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] +
-                   "]\n  "));
+  if (include_file_version) {
+    RETURN_NOT_OK(writable->Append("  options_file_version=" +
+                                   ToString(ROCKSDB_OPTION_FILE_MAJOR) + "." +
+                                   ToString(ROCKSDB_OPTION_FILE_MINOR) + "\n"));
+  }
 
+  // DBOptions section
+  std::string options_file_content;
+  RETURN_NOT_OK(writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] + "]\n  "));
   s = GetStringFromDBOptions(&options_file_content, db_opt, "\n  ");
   if (!s.ok()) {
     WARN_NOT_OK(writable->Close(), "Failed to close writable");
     return s;
   }
-  RETURN_NOT_OK(writable->Append(options_file_content + "\n"));
+  RETURN_NOT_OK(writable->Append(options_file_content));
 
   for (size_t i = 0; i < cf_opts.size(); ++i) {
     // CFOptions section
     RETURN_NOT_OK(writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] +
-                     " \"" + EscapeOptionString(cf_names[i]) + "\"]\n  "));
-    s = GetStringFromColumnFamilyOptions(&options_file_content, cf_opts[i],
-                                         "\n  ");
+                                   " \"" + EscapeOptionString(cf_names[i]) + "\"]\n  "));
+    s = GetStringFromColumnFamilyOptions(&options_file_content, cf_opts[i], "\n  ");
     if (!s.ok()) {
       WARN_NOT_OK(writable->Close(), "Failed to close writable");
       return s;
     }
-    RETURN_NOT_OK(writable->Append(options_file_content + "\n"));
+    RETURN_NOT_OK(writable->Append(options_file_content));
+
     // TableOptions section
     auto* tf = cf_opts[i].table_factory.get();
     if (tf != nullptr) {
@@ -103,9 +111,10 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
       if (!s.ok()) {
         return s;
       }
-      RETURN_NOT_OK(writable->Append(options_file_content + "\n"));
+      RETURN_NOT_OK(writable->Append(options_file_content));
     }
   }
+
   RETURN_NOT_OK(writable->Flush());
   if (!db_opt.disableDataSync) {
     RETURN_NOT_OK(writable->Fsync());

src/yb/rocksdb/util/options_parser.h

@@ -18,6 +18,9 @@
 // under the License.
 //
 
+#ifndef YB_ROCKSDB_UTIL_OPTIONS_PARSER_H
+#define YB_ROCKSDB_UTIL_OPTIONS_PARSER_H
+
 #pragma once
 
 #include <map>
@@ -31,8 +34,6 @@
 
 namespace rocksdb {
 
-#ifndef ROCKSDB_LITE
-
 #define ROCKSDB_OPTION_FILE_MAJOR 1
 #define ROCKSDB_OPTION_FILE_MINOR 1
 
@@ -47,10 +48,15 @@ enum OptionSection : char {
 static const std::string opt_section_titles[] = {
     "Version", "DBOptions", "CFOptions", "TableOptions/", "Unknown"};
 
+YB_STRONGLY_TYPED_BOOL(IncludeHeader);
+YB_STRONGLY_TYPED_BOOL(IncludeFileVersion);
+
 Status PersistRocksDBOptions(const DBOptions& db_opt,
                              const std::vector<std::string>& cf_names,
                              const std::vector<ColumnFamilyOptions>& cf_opts,
-                             const std::string& file_name, Env* env);
+                             const std::string& file_name, Env* env,
+                             IncludeHeader include_header = IncludeHeader::kTrue,
+                             IncludeFileVersion include_file_version = IncludeFileVersion::kTrue);
 
 class RocksDBOptionsParser {
  public:
@@ -153,6 +159,6 @@ class RocksDBOptionsParser {
   int opt_file_version[3];
 };
 
-#endif  // !ROCKSDB_LITE
-
 }  // namespace rocksdb
+
+#endif // YB_ROCKSDB_UTIL_OPTIONS_PARSER_H

src/yb/rocksdb/utilities/options/options_util_test.cc

@@ -111,6 +111,61 @@ TEST_F(OptionsUtilTest, SaveAndLoad) {
   }
 }
 
+TEST_F(OptionsUtilTest, CompareIncludeOptions) {
+  const size_t kCFCount = 5;
+
+  DBOptions db_opt;
+  std::vector<std::string> cf_names;
+  std::vector<ColumnFamilyOptions> cf_opts;
+  test::RandomInitDBOptions(&db_opt, &rnd_);
+  for (size_t i = 0; i < kCFCount; ++i) {
+    cf_names.push_back(i == 0 ? kDefaultColumnFamilyName
+                              : test::RandomName(&rnd_, 10));
+    cf_opts.emplace_back();
+    test::RandomInitCFOptions(&cf_opts.back(), &rnd_);
+  }
+
+  std::tuple<IncludeHeader, IncludeFileVersion> include_opts[] {
+    std::make_tuple(IncludeHeader::kTrue, IncludeFileVersion::kTrue),
+    std::make_tuple(IncludeHeader::kTrue, IncludeFileVersion::kFalse),
+    std::make_tuple(IncludeHeader::kFalse, IncludeFileVersion::kTrue),
+    std::make_tuple(IncludeHeader::kFalse, IncludeFileVersion::kFalse)
+  };
+
+  const std::string kFileName = "OPTIONS-123456";
+
+  for (auto const& opts : include_opts) {
+    if (env_->FileExists(kFileName).ok()) {
+      ASSERT_OK(env_->DeleteFile(kFileName));
+    }
+    ASSERT_OK(PersistRocksDBOptions(
+      db_opt, cf_names, cf_opts, kFileName, env_.get(), std::get<0>(opts), std::get<1>(opts)));
+
+    DBOptions loaded_db_opt;
+    std::vector<ColumnFamilyDescriptor> loaded_cf_descs;
+    ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt, &loaded_cf_descs));
+    ASSERT_OK(env_->DeleteFile(kFileName));
+
+    ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt));
+    ASSERT_EQ(loaded_cf_descs.size(), cf_names.size());
+    ASSERT_EQ(loaded_cf_descs.size(), cf_opts.size());
+    for (size_t i = 0; i < loaded_cf_descs.size(); ++i) {
+      ASSERT_EQ(cf_names[i], loaded_cf_descs[i].name);
+      ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cf_opts[i], loaded_cf_descs[i].options));
+      if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) {
+        ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory(
+            cf_opts[i].table_factory.get(), loaded_cf_descs[i].options.table_factory.get()));
+      }
+    }
+  }
+
+  for (size_t i = 0; i < cf_opts.size(); ++i) {
+    if (cf_opts[i].compaction_filter) {
+      delete cf_opts[i].compaction_filter;
+    }
+  }
+}
+
 namespace {
 class DummyTableFactory : public TableFactory {
  public:

src/yb/rocksdb/utilities/stackable_db.h

@@ -296,6 +296,12 @@ class StackableDB : public DB {
     db_->GetColumnFamilyMetaData(column_family, cf_meta);
   }
 
+  virtual void GetColumnFamiliesOptions(
+      std::vector<std::string>* column_family_names,
+      std::vector<ColumnFamilyOptions>* column_family_options) override {
+    db_->GetColumnFamiliesOptions(column_family_names, column_family_options);
+  }
+
 #endif  // ROCKSDB_LITE
 
   virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs,