Adding option templates to wdt

Summary: Adding option templates to wdt. Right now, there is only flash and
disk. Depending on the option type, flags are initialized to different values.
Value of a specific option can be changed using flags. For disk, only number of
threads is changed and block mode is disabled.

Reviewed By: @ldemailly, @nikunjy

Differential Revision: D2363348

OptionTypeTest.cpp

@@ -0,0 +1,95 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#include "WdtFlags.h"
+#include "WdtOptions.h"
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+/*
+ * Tests in this file can not be run in the same process. That is because we
+ * rely on is_default property of gflags to determine whether a flag has been
+ * specified by the user. And if a flag is ever changed, is_default becomes
+ * false and can never be reset to true. So, after running one test, other
+ * tests would fail. To run tests in this file, use wdt/wdt_option_type_test.sh.
+ * That script runs each test in separate process and also tests for both long
+ * and short flags.
+ */
+
+namespace facebook {
+namespace wdt {
+
+const std::string NUM_PORTS_FLAG =
+    WdtFlags::getFlagNameFromOptionName("num_ports");
+const std::string BLOCK_SIZE_FLAG =
+    WdtFlags::getFlagNameFromOptionName("block_size_mbytes");
+const std::string OPTION_TYPE_FLAG =
+    WdtFlags::getFlagNameFromOptionName("option_type");
+
+void overrideTest1(const std::string &optionType) {
+  const auto &options = WdtOptions::get();
+  google::SetCommandLineOption(OPTION_TYPE_FLAG.c_str(), optionType.c_str());
+  google::SetCommandLineOption(NUM_PORTS_FLAG.c_str(), "4");
+  google::SetCommandLineOption(BLOCK_SIZE_FLAG.c_str(), "8");
+  WdtFlags::initializeFromFlags();
+  EXPECT_EQ(4, options.num_ports);
+  EXPECT_EQ(8, options.block_size_mbytes);
+}
+
+void overrideTest2(const std::string &optionType) {
+  const auto &options = WdtOptions::get();
+  google::SetCommandLineOption(OPTION_TYPE_FLAG.c_str(), optionType.c_str());
+  google::SetCommandLineOption(NUM_PORTS_FLAG.c_str(), "8");
+  google::SetCommandLineOption(BLOCK_SIZE_FLAG.c_str(), "16");
+  WdtFlags::initializeFromFlags();
+  EXPECT_EQ(8, options.num_ports);
+  EXPECT_EQ(16, options.block_size_mbytes);
+}
+
+TEST(OptionType, FlashOptionTypeTest1) {
+  const auto &options = WdtOptions::get();
+  google::SetCommandLineOption(OPTION_TYPE_FLAG.c_str(), "flash");
+  WdtFlags::initializeFromFlags();
+  EXPECT_EQ(8, options.num_ports);
+  EXPECT_EQ(16, options.block_size_mbytes);
+}
+
+TEST(OptionType, FlashOptionTypeTest2) {
+  overrideTest1("flash");
+}
+
+TEST(OptionType, FlashOptionTypeTest3) {
+  overrideTest2("flash");
+}
+
+TEST(OptionType, DiskOptionTypeTest1) {
+  const auto &options = WdtOptions::get();
+  google::SetCommandLineOption(OPTION_TYPE_FLAG.c_str(), "disk");
+  WdtFlags::initializeFromFlags();
+  EXPECT_EQ(1, options.num_ports);
+  EXPECT_EQ(-1, options.block_size_mbytes);
+}
+
+TEST(OptionType, DiskOptionTypeTest2) {
+  overrideTest1("disk");
+}
+
+TEST(OptionType, DiskOptionTypeTest3) {
+  overrideTest2("disk");
+}
+}
+}
+
+int main(int argc, char *argv[]) {
+  FLAGS_logtostderr = true;
+  testing::InitGoogleTest(&argc, argv);
+  google::ParseCommandLineFlags(&argc, &argv, true);
+  google::InitGoogleLogging(argv[0]);
+  int ret = RUN_ALL_TESTS();
+  return ret;
+}

WdtFlags.cpp

@@ -10,17 +10,70 @@
 #include "WdtFlags.cpp.inc"
 #include <glog/logging.h>
 #include "Protocol.h"
+#include <folly/Conv.h>
+
+FLAG_DEFINITION(string, PREFIX(option_type),
+                facebook::wdt::WdtOptions::FLASH_OPTION_TYPE,
+                "WDT option type. Options are initialized to different values "
+                "depending on the type. Individual options can still be "
+                "changed using specific flags.")
 
 namespace facebook {
 namespace wdt {
+
+const std::string FLAGS_PREFIX = "wdt_";
+
 void WdtFlags::initializeFromFlags() {
   LOG(INFO) << "Running WDT " << Protocol::getFullVersion();
 #define ASSIGN_OPT
 #include "WdtFlags.cpp.inc"  //nolint
+#undef ASSIGN_OPT
+  std::set<std::string> userSpecifiedFlags = getUserSpecifiedOptions();
+  WdtOptions::getMutable().modifyOptions(FLAGS_OPTION_TYPE, userSpecifiedFlags);
 }
+
 void WdtFlags::printOptions() {
 #define PRINT_OPT
 #include "WdtFlags.cpp.inc"  //nolint
+#undef PRINT_OPT
+}
+
+std::string WdtFlags::getOptionNameFromFlagName(const std::string &flagName) {
+#ifndef STANDALONE_APP
+  // extra wdt_ prefix is added in this case
+  if (flagName.compare(0, FLAGS_PREFIX.size(), FLAGS_PREFIX) == 0) {
+    // flagname begins with wdt_
+    return flagName.substr(FLAGS_PREFIX.size());
+  }
+#endif
+  return flagName;
+}
+
+std::string WdtFlags::getFlagNameFromOptionName(const std::string &optionName) {
+#ifndef STANDALONE_APP
+  // extra wdt_ prefix has to be added
+  std::string flagName;
+  folly::toAppend(FLAGS_PREFIX, optionName, &flagName);
+  return flagName;
+#endif
+  return optionName;
+}
+
+std::set<std::string> WdtFlags::getUserSpecifiedOptions() {
+  std::set<std::string> userSpecifiedFlags;
+  std::vector<google::CommandLineFlagInfo> allFlags;
+  google::GetAllFlags(&allFlags);
+  for (const auto &flag : allFlags) {
+    if (!flag.is_default) {
+      // is_default is false if the flag has been specified in the cmd line.
+      // Even if the value specified is same as the default value, this boolean
+      // is still marked as false. If the flag is directly like
+      // FLAGS_num_ports=1, is_default won't change. But, if it set using
+      // SetCommandLineOption, this will change.
+      userSpecifiedFlags.emplace(getOptionNameFromFlagName(flag.name));
+    }
+  }
+  return userSpecifiedFlags;
 }
 }
 }

WdtFlags.h

@@ -13,6 +13,10 @@
 #define DECLARE_ONLY
 #include "WdtFlags.cpp.inc"
 #undef DECLARE_ONLY
+
+#define FLAGS_OPTION_TYPE FLAG_VALUE(option_type)
+FLAG_DECLARATION(string, PREFIX(option_type))
+
 namespace facebook {
 namespace wdt {
 class WdtFlags {
@@ -23,6 +27,19 @@ class WdtFlags {
   static void initializeFromFlags();
 
   static void printOptions();
+
+  /**
+   * Returns option name from flag name
+   */
+  static std::string getOptionNameFromFlagName(const std::string &flagName);
+
+  /**
+   * Returns flag name from option name
+   */
+  static std::string getFlagNameFromOptionName(const std::string &optionName);
+
+  /// returns list of options specified in the cmd line by the user
+  static std::set<std::string> getUserSpecifiedOptions();
 };
 }
 }

WdtFlagsMacros.h

@@ -1,11 +1,4 @@
 /// override-include-guard
-#ifdef GENMACRO_X
-#undef GENMACRO_X
-#endif
-#ifdef GENMACRO
-#undef GENMACRO
-#endif
-
 #ifndef OPTIONS
 #define OPTIONS WdtOptions
 #endif
@@ -17,42 +10,37 @@
 #endif
 #define VALUE(A) facebook::wdt::OPTIONS::get().A
 
-#ifdef DECLARE_ONLY
-#define GENMACRO_X(type, argument, value, description) DECLARE_##type(argument);
-#else
-#define GENMACRO_X(type, argument, value, description) \
-  DEFINE_##type(argument, VALUE(value), description);
-#endif
-
-#ifdef STANDALONE_APP
-#define GENMACRO(type, argument, description) \
-  GENMACRO_X(type, argument, argument, description)
-#else
-#define GENMACRO(type, argument, description) \
-  GENMACRO_X(type, wdt_##argument, argument, description)
-#endif
-
-#ifndef WDT_OPT
-#define WDT_OPT(argument, type, description) \
-  GENMACRO(type, argument, description)
-#endif
-
 #define VALUE_X(argument) FLAGS_##argument
 #define CAT(argument) VALUE_X(argument)
 #define FLAG_VALUE(argument) CAT(PREFIX(argument))
 
-#ifdef ASSIGN_OPT
 #ifdef WDT_OPT
 #undef WDT_OPT
 #endif
+
+#ifdef ASSIGN_OPT
+// Assign option from flags
 #define WDT_OPT(argument, type, description) \
   facebook::wdt::OPTIONS::getMutable().argument = FLAG_VALUE(argument);
-#endif
-
+#else
 #ifdef PRINT_OPT
-#ifdef WDT_OPT
-#undef WDT_OPT
-#endif
+// print options
 #define WDT_OPT(argument, type, description) \
   LOG(INFO) << #argument << " " << facebook::wdt::OPTIONS::get().argument;
+#else
+
+#define FLAG_DECLARATION(type, argument) DECLARE_##type(argument);
+#define FLAG_DEFINITION(type, argument, value, description) \
+  DEFINE_##type(argument, value, description);
+
+#ifdef DECLARE_ONLY
+// declare flags
+#define WDT_OPT(argument, type, description) \
+  FLAG_DECLARATION(type, PREFIX(argument))
+#else
+// define flags
+#define WDT_OPT(argument, type, description) \
+  FLAG_DEFINITION(type, PREFIX(argument), VALUE(argument), description)
+#endif
+#endif
 #endif

WdtOptions.cpp

@@ -7,10 +7,33 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  */
 #include "WdtOptions.h"
+#include <glog/logging.h>
 namespace facebook {
 namespace wdt {
 
+#define CHANGE_IF_NOT_SPECIFIED(option, specifiedOptions, value)  \
+  if (specifiedOptions.find(#option) == specifiedOptions.end()) { \
+    option = value;                                               \
+  }
+
 WdtOptions* WdtOptions::instance_ = nullptr;
+const std::string WdtOptions::FLASH_OPTION_TYPE = "flash";
+const std::string WdtOptions::DISK_OPTION_TYPE = "disk";
+
+void WdtOptions::modifyOptions(
+    const std::string& optionType,
+    const std::set<std::string>& userSpecifiedOptions) {
+  if (optionType == DISK_OPTION_TYPE) {
+    CHANGE_IF_NOT_SPECIFIED(num_ports, userSpecifiedOptions, 1)
+    CHANGE_IF_NOT_SPECIFIED(block_size_mbytes, userSpecifiedOptions, -1)
+    return;
+  }
+  if (optionType != FLASH_OPTION_TYPE) {
+    LOG(WARNING) << "Invalid option type " << optionType << ". Valid types are "
+                 << FLASH_OPTION_TYPE << ", " << DISK_OPTION_TYPE;
+  }
+  // options are initialized for flash. So, no need to change anything
+}
 
 const WdtOptions& WdtOptions::get() {
   return getMutable();

WdtOptions.h

@@ -11,6 +11,7 @@
 #include <cstdint>
 #include <string>
 #include <unistd.h>
+#include <set>
 namespace facebook {
 namespace wdt {
 /**
@@ -20,6 +21,10 @@ namespace wdt {
  */
 class WdtOptions {
  public:
+  // WDT option types
+  static const std::string FLASH_OPTION_TYPE;
+  static const std::string DISK_OPTION_TYPE;
+
   /**
    * A static method that can be called to create
    * the singleton copy of WdtOptions through the lifetime
@@ -30,6 +35,17 @@ class WdtOptions {
    * Method to get mutable copy of the singleton
    */
   static WdtOptions& getMutable();
+
+  /**
+   * Modifies options based on the type specified
+   *
+   * @param optionType           option type
+   * @param userSpecifiedOptions options specified by user, this options are not
+   *                             changed
+   */
+  virtual void modifyOptions(const std::string& optionType,
+                             const std::set<std::string>& userSpecifiedOptions);
+
   /**
    * Use ipv6 while establishing connection.
    * When both ipv6 and ipv4 are false we will try both

wdtCmdLine.cpp

@@ -62,6 +62,10 @@ DEFINE_string(recovery_id, "", "Recovery-id to use for download resumption");
 DEFINE_bool(treat_fewer_port_as_error, false,
             "If the receiver is unable to bind to all the ports, treat that as "
             "an error.");
+DEFINE_bool(print_options, false,
+            "If true, wdt prints the option values and exits. Option values "
+            "printed take into account option type and other command line "
+            "flags specified.");
 
 using namespace facebook::wdt;
 template <typename T>
@@ -120,6 +124,10 @@ int main(int argc, char *argv[]) {
   signal(SIGPIPE, SIG_IGN);
 
   FLAGS::initializeFromFlags();
+  if (FLAGS_print_options) {
+    FLAGS::printOptions();
+    return 0;
+  }
 
   ErrorCode retCode = OK;
 

wdt_option_type_test.sh

@@ -0,0 +1,23 @@
+#! /bin/sh
+
+# Tests for option types can not be run in the same process because gflags
+# can not be reset to default condition. This script runs all the tests one by
+# one in separate processes
+
+binaries=(_bin/wdt/option_type_test_long_flags \
+  _bin/wdt/short_flags/option_type_test_short_flags)
+
+for binary in "${binaries[@]}"
+do
+  for test_name in $($binary --gtest_list_tests | \
+    tail -n +2 | sed 's/^ *//g')
+  do
+    echo "$binary --gtest_filter=*.$test_name"
+    $binary --gtest_filter="*.$test_name"
+    LAST_STATUS=$?
+    if [ $LAST_STATUS -ne 0 ] ; then
+      exit $LAST_STATUS
+    fi
+  done
+done
+exit 0