Merge pull request ceph#56735 from samarahu/wip-rgw-d4n-long-descript…

…ions

d4n: Add more in-depth config descriptions and improve naming

Reviewed-by: Casey Bodley <cbodley@redhat.com>
Reviewed-by: Samarah Uriarte <samarah.uriarte@ibm.com>

doc/radosgw/config-ref.rst

@@ -312,8 +312,15 @@ Currently, the architecture can only function on one Redis instance at a time.
 The address is configurable and can be changed by accessing the parameters 
 below.
 
-.. confval:: rgw_d4n_host
-.. confval:: rgw_d4n_port
+.. confval:: rgw_d4n_address
+.. confval:: rgw_d4n_l1_datacache_persistent_path
+.. confval:: rgw_d4n_l1_datacache_size
+.. confval:: rgw_d4n_l1_evict_cache_on_start
+.. confval:: rgw_d4n_l1_fadvise
+.. confval:: rgw_d4n_libaio_aio_threads
+.. confval:: rgw_d4n_libaio_aio_num
+.. confval:: rgw_lfuda_sync_frequency
+.. confval:: rgw_d4n_l1_datacache_address
 
 Topic persistency settings
 ==========================
@@ -335,4 +342,4 @@ and retried as frequently as possible).
 .. confval:: rgw_topic_persistency_max_retries
 .. confval:: rgw_topic_persistency_sleep_duration
 
-.. _Bucket Notifications: ../notifications
+.. _Bucket Notifications: ../notifications

src/common/options/rgw.yaml.in

@@ -3649,23 +3649,33 @@ options:
 - name: rgw_d4n_l1_datacache_persistent_path
   type: str
   level: advanced
-  desc: path for the directory for storing the local cache objects data for d4n
+  desc: path used for storing locally cached object data
+  long_desc: One cache backend option for D4N is the local SSD, which uses this path to
+    write and read object data. This is the default cache backend chosen by the D4N filter.
+    Only the SSD cache backend uses this path for object data storage since the RedisDriver
+    uses a Redis server instead and there are no additional cache backend implementations
+    available at the moment. 
   default: /tmp/rgw_d4n_datacache/
   services:
   - rgw
   with_legacy: true
 - name: rgw_d4n_l1_datacache_size
   type: size
   level: advanced
-  desc: datacache maximum size on disk in bytes
+  desc: maximum size on disk for datacache
+  long_desc: The local SSD cache uses this option to configure its size in bytes. This 
+    option is not used by the Redis cache backend. 
   default: 1_G
   services:
   - rgw
   with_legacy: true
 - name: rgw_d4n_l1_evict_cache_on_start
   type: bool
   level: advanced
-  desc: clear the content of the persistent data cache directory on start
+  desc: clear the contents of the persistent datacache on start
+  long_desc: The local SSD cache uses this option to clear the contents of the path supplied
+    by the rgw_d4n_l1_datacache_persistent_path config option on start. If false, the path's 
+    contents will be retained. 
   default: true
   services:
   - rgw
@@ -3674,7 +3684,7 @@ options:
   type: int
   level: advanced
   desc: posix_fadvise() flag for access pattern of cache files
-  long_desc: for example to bypass the page-cache -
+  long_desc: For example, to bypass the page-cache -
     POSIX_FADV_DONTNEED=4
   default: 4
   services:
@@ -3684,6 +3694,8 @@ options:
   type: int
   level: advanced
   desc: specifies the maximum number of worker threads that may be used by libaio
+  long_desc: This option is used by the SSD cache backend during initialization to set the maximum
+    number of worker threads libaio may use. It does not apply to the Redis cache backend. 
   default: 20
   services:
   - rgw
@@ -3694,6 +3706,9 @@ options:
   type: int
   level: advanced
   desc: specifies the maximum number of simultaneous I/O requests that libaio expects to enqueue
+  long_desc: This option is used by the SSD cache backend during initialization to set the maximum
+    number of simultaneous I/O requests that libaio can expect to enqueue. It
+    does not apply to the Redis cache backend. 
   default: 64
   services:
   - rgw
@@ -3704,6 +3719,12 @@ options:
   type: int
   level: advanced
   desc: LFUDA variables' sync frequency in seconds 
+  long_desc: By default, the D4N cache uses the Least Frequently Used with Dynamic Aging (LFUDA) 
+    cache replacement policy. This class globally stores values that are used by the policy's 
+    algorithm. However, strong consistency for these values is not necessary and adds additional
+    overhead to support. As a result, a thread periodically retrieves these global values and posts
+    updates when certain conditions are satisfied. This Redis thread completes this logic in a loop
+    that is called once every interval, with the interval being set by this option.  
   default: 60
   services:
   - rgw
@@ -3957,21 +3978,15 @@ options:
   default: true
   services:
   - rgw
-- name: rgw_d4n_host
+- name: rgw_d4n_address
   type: str
   level: advanced
-  desc: The rgw directory host
-  default: 127.0.0.1
-  services: 
-  - rgw
-  flags:
-  - startup
-  with_legacy: true
-- name: rgw_d4n_port
-  type: int
-  level: advanced
-  desc: The rgw directory port
-  default: 6379
+  desc: address for the D4N Redis connection
+  long_desc: The current D4N implementation supports one Redis node
+    which the D4N directory, policy, and overall filter communicate
+    with. This default value is also the address that a Redis server 
+    with no additional configuration will use.
+  default: 127.0.0.1:6379
   services: 
   - rgw
   flags:
@@ -4114,12 +4129,15 @@ options:
   services:
   - rgw
   with_legacy: true
-- name: rgw_local_cache_address
+- name: rgw_d4n_l1_datacache_address
   type: str
   level: advanced
-  desc: Local CacheDriver address 
+  desc: local Redis cache address 
+  long_desc: This is the address used to configure the Redis cache backend connection. The default
+    value is the same address used by Redis without any additional configuration. The SSD cache 
+    does not use this option.
   default: 127.0.0.1:6379
-  services: 
+  services:
   - rgw
   flags:
   - startup

src/rgw/driver/d4n/d4n_policy.cc

@@ -58,7 +58,7 @@ int LFUDAPolicy::init(CephContext *cct, const DoutPrefixProvider* dpp, asio::io_
     req.push("HEXISTS", "lfuda", "age"); 
     req.push("HSET", "lfuda", "minLocalWeights_sum", std::to_string(weightSum)); /* New cache node will always have the minimum average weight */
     req.push("HSET", "lfuda", "minLocalWeights_size", std::to_string(entries_map.size()));
-    req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_local_cache_address);
+    req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_d4n_l1_datacache_address);
 
     redis_exec(conn, ec, req, resp, y);
 
@@ -169,7 +169,7 @@ int LFUDAPolicy::local_weight_sync(const DoutPrefixProvider* dpp, optional_yield
 	response<int, int, int> value;
 	req.push("HSET", "lfuda", "minLocalWeights_sum", std::to_string(weightSum));
 	req.push("HSET", "lfuda", "minLocalWeights_size", std::to_string(entries_map.size()));
-	req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_local_cache_address);
+	req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_d4n_l1_datacache_address);
 	redis_exec(conn, ec, req, resp, y);
 
 	if (ec) {
@@ -191,8 +191,8 @@ int LFUDAPolicy::local_weight_sync(const DoutPrefixProvider* dpp, optional_yield
     boost::system::error_code ec;
     request req;
     response<int, int> resp;
-    req.push("HSET", dpp->get_cct()->_conf->rgw_local_cache_address, "avgLocalWeight_sum", std::to_string(weightSum));
-    req.push("HSET", dpp->get_cct()->_conf->rgw_local_cache_address, "avgLocalWeight_size", std::to_string(entries_map.size()));
+    req.push("HSET", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, "avgLocalWeight_sum", std::to_string(weightSum));
+    req.push("HSET", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, "avgLocalWeight_size", std::to_string(entries_map.size()));
     redis_exec(conn, ec, req, resp, y);
 
     if (ec) {
@@ -291,7 +291,7 @@ int LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, uint64_t size, optional
 
     int avgWeight = weightSum / entries_map.size();
 
-    if (victim->hostsList.size() == 1 && victim->hostsList[0] == dir->cct->_conf->rgw_local_cache_address) { /* Last copy */
+    if (victim->hostsList.size() == 1 && victim->hostsList[0] == dir->cct->_conf->rgw_d4n_l1_datacache_address) { /* Last copy */
       if (victim->globalWeight) {
 	it->second->localWeight += victim->globalWeight;
         (*it->second->handle)->localWeight = it->second->localWeight;
@@ -321,7 +321,7 @@ int LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, uint64_t size, optional
       return ret;
     }
 
-    if (int ret = dir->remove_host(victim, dir->cct->_conf->rgw_local_cache_address, y) < 0) {
+    if (int ret = dir->remove_host(victim, dir->cct->_conf->rgw_d4n_l1_datacache_address, y) < 0) {
       delete victim;
       return ret;
     }

src/rgw/driver/d4n/rgw_sal_d4n.cc

@@ -66,7 +66,7 @@ int D4NFilterDriver::initialize(CephContext *cct, const DoutPrefixProvider *dpp)
   namespace net = boost::asio;
   using boost::redis::config;
 
-  std::string address = cct->_conf->rgw_local_cache_address;
+  std::string address = cct->_conf->rgw_d4n_address;
   config cfg;
   cfg.addr.host = address.substr(0, address.find(":"));
   cfg.addr.port = address.substr(address.find(":") + 1, address.length());
@@ -699,7 +699,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
   if (write_to_cache) {
     rgw::d4n::CacheBlock block, existing_block;
     rgw::d4n::BlockDirectory* blockDir = source->driver->get_block_dir();
-    block.hostsList.push_back(blockDir->cct->_conf->rgw_local_cache_address); 
+    block.hostsList.push_back(blockDir->cct->_conf->rgw_d4n_l1_datacache_address); 
     block.cacheObj.objName = source->get_key().get_oid();
     block.cacheObj.bucketName = source->get_bucket()->get_name();
     std::stringstream s;
@@ -747,7 +747,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
                   if (blockDir->set(&block, *y) < 0) //new versioned block will have new version, hostsList etc, how about globalWeight?
                     ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set method failed." << dendl;
                 } else {
-                if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_local_cache_address, *y) < 0)
+                if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_d4n_l1_datacache_address, *y) < 0)
                   ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory update_field method failed for hostsList." << dendl;
                 }
               }
@@ -787,7 +787,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
                   if (blockDir->set(&block, *y) < 0)
                     ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set method failed." << dendl;
               } else {
-                if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_local_cache_address, *y) < 0)
+                if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_d4n_l1_datacache_address, *y) < 0)
                   ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory update_field method failed for blockHosts." << dendl;
               }
             }
@@ -834,7 +834,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
 		    if (blockDir->set(&block, *y) < 0)
 		      ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set method failed." << dendl;
 		  } else {
-		  if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_local_cache_address, *y) < 0)
+		  if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_d4n_l1_datacache_address, *y) < 0)
 		    ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory update_field method failed." << dendl;
 		  }
 		}
@@ -926,7 +926,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
 				 .bucketName = obj->get_bucket()->get_name(),
 				 .creationTime = to_iso_8601(*mtime), 
 				 .dirty = false,
-				 .hostsList = { /*driver->get_block_dir()->cct->_conf->rgw_local_cache_address*/ } //TODO: Object is not currently being cached 
+				 .hostsList = { /*driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address*/ } //TODO: Object is not currently being cached 
                                };
 
   if (driver->get_obj_dir()->set(&object, y) < 0) 

src/rgw/rgw_redis_driver.cc

@@ -68,7 +68,7 @@ int RedisDriver::initialize(const DoutPrefixProvider* dpp)
     partition_info.location += "/";
   }
 
-  std::string address = dpp->get_cct()->_conf->rgw_local_cache_address;
+  std::string address = dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address;
 
   config cfg;
   cfg.addr.host = address.substr(0, address.find(":"));

src/test/rgw/test_d4n_directory.cc

@@ -34,7 +34,7 @@ class Environment : public ::testing::Environment {
       cct = common_preinit(iparams, CODE_ENVIRONMENT_UTILITY, {}); 
       dpp = new DoutPrefix(cct->get(), dout_subsys, "D4N Object Directory Test: ");
 
-      redisHost = cct->_conf->rgw_d4n_host + ":" + std::to_string(cct->_conf->rgw_d4n_port); 
+      redisHost = cct->_conf->rgw_d4n_address; 
     }
 
     void TearDown() override {
@@ -67,8 +67,8 @@ class ObjectDirectoryFixture: public ::testing::Test {
 
       /* Run fixture's connection */
       config cfg;
-      cfg.addr.host = env->cct->_conf->rgw_d4n_host;
-      cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port);
+      cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":"));
+      cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length()); 
 
       conn->async_run(cfg, {}, net::detached);
     } 
@@ -115,8 +115,8 @@ class BlockDirectoryFixture: public ::testing::Test {
 
       /* Run fixture's connection */
       config cfg;
-      cfg.addr.host = env->cct->_conf->rgw_d4n_host;
-      cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port);
+      cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":"));
+      cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length()); 
 
       conn->async_run(cfg, {}, net::detached);
     } 

src/test/rgw/test_d4n_policy.cc

@@ -36,7 +36,7 @@ class Environment : public ::testing::Environment {
       cct = common_preinit(iparams, CODE_ENVIRONMENT_UTILITY, {}); 
       dpp = new DoutPrefix(cct->get(), dout_subsys, "D4N Object Directory Test: ");
 
-      redisHost = cct->_conf->rgw_d4n_host + ":" + std::to_string(cct->_conf->rgw_d4n_port); 
+      redisHost = cct->_conf->rgw_d4n_address; 
     }
 
     std::string redisHost;
@@ -82,8 +82,8 @@ class LFUDAPolicyFixture : public ::testing::Test {
 
       /* Run fixture's connection */
       config cfg;
-      cfg.addr.host = env->cct->_conf->rgw_d4n_host;
-      cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port);
+      cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":"));
+      cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length()); 
 
       conn->async_run(cfg, {}, net::detached);
     } 
@@ -130,7 +130,7 @@ class LFUDAPolicyFixture : public ::testing::Test {
 	    }
 	  }
 	} else if (!exists) { /* No remote copy */
-	  block->hostsList.push_back(dir->cct->_conf->rgw_local_cache_address);
+	  block->hostsList.push_back(dir->cct->_conf->rgw_d4n_l1_datacache_address);
 	  if (dir->set(block, y) < 0)
 	    return -1;
 

src/test/rgw/test_redis_driver.cc

@@ -74,7 +74,7 @@ class Environment : public ::testing::Environment {
       cct = common_preinit(iparams, CODE_ENVIRONMENT_UTILITY, {});
       dpp = new DoutPrefix(cct->get(), dout_subsys, "D4N Object Directory Test: ");
 
-      redisHost = cct->_conf->rgw_d4n_host + ":" + std::to_string(cct->_conf->rgw_d4n_port);
+      redisHost = cct->_conf->rgw_d4n_address; 
     }
 
     std::string redisHost;
@@ -102,8 +102,8 @@ class RedisDriverFixture: public ::testing::Test {
 
       /* Run fixture's connection */
       config cfg;
-      cfg.addr.host = env->cct->_conf->rgw_d4n_host;
-      cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port);
+      cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":"));
+      cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length()); 
 
       conn->async_run(cfg, {}, net::detached);
     }