Make policy retry delay configurable

The retry delay should be configurable, with a default initial backoff value of 10 seconds. This can be set to larger amounts for servers that crash under heavy loads from frequent re-requests.
noironetworks · Mar 3, 2023 · ac6b294 · ac6b294
1 parent 5468de0
commit ac6b294
Show file tree

Hide file tree

Showing 7 changed files with 32 additions and 2 deletions.
diff --git a/agent-ovs/lib/Agent.cpp b/agent-ovs/lib/Agent.cpp
@@ -181,6 +181,7 @@ void Agent::setProperties(const boost::property_tree::ptree& properties) {
     static const std::string OPFLEX_PRR_INTERVAL("opflex.timers.prr");
     static const std::string OPFLEX_HANDSHAKE("opflex.timers.handshake-timeout");
     static const std::string OPFLEX_KEEPALIVE("opflex.timers.keepalive-timeout");
+    static const std::string OPFLEX_POLICY_RETRY_DELAY("opflex.timers.policy-retry-delay");
     static const std::string DISABLED_FEATURES("feature.disabled");
     static const std::string BEHAVIOR_L34FLOWS_WITHOUT_SUBNET("behavior.l34flows-without-subnet");
     static const std::string OPFLEX_ASYC_JSON("opflex.asyncjson.enabled");
@@ -453,6 +454,16 @@ void Agent::setProperties(const boost::property_tree::ptree& properties) {
         LOG(INFO) << "prr timer set to " << prr_timer << " secs";
     }
 
+    optional<boost::uint_t<64>::fast> policy_retry_delay_present =
+        properties.get_optional<boost::uint_t<64>::fast>(OPFLEX_POLICY_RETRY_DELAY);
+    if (policy_retry_delay_present) {
+        policy_retry_delay_timer = policy_retry_delay_present.get();
+        if (policy_retry_delay_timer < 10) {
+           policy_retry_delay_timer = 10;  /* min is 10 seconds */
+        }
+        LOG(INFO) << "policy retry delay timer set to " << policy_retry_delay_timer << " secs";
+    }
+
     optional<uint32_t> handshakeOpt = properties.get_optional<uint32_t>(OPFLEX_HANDSHAKE);
     if (handshakeOpt) {
         peerHandshakeTimeout = handshakeOpt.get();
@@ -542,6 +553,7 @@ void Agent::applyProperties() {
     }
 
     framework.setPrrTimerDuration(prr_timer);
+    framework.setPolicyRetryDelayTimerDuration(policy_retry_delay_timer);
     framework.setHandshakeTimeout(peerHandshakeTimeout);
     framework.setKeepaliveTimeout(keepaliveTimeout);
 }

diff --git a/agent-ovs/lib/include/opflexagent/Agent.h b/agent-ovs/lib/include/opflexagent/Agent.h
@@ -328,6 +328,8 @@ typedef opflex::ofcore::OFConstants::OpflexElementMode opflex_elem_t;
     // timers
     // prr timer - policy resolve request timer
     boost::uint_t<64>::fast prr_timer = 7200;  /* seconds */
+    // initial policy retry delay
+    boost::uint_t<64>::fast policy_retry_delay_timer = 10;  /* seconds */
     /* handshake timeout */
     uint32_t peerHandshakeTimeout = 45000;
     /* keepalive timeout */

diff --git a/agent-ovs/opflex-agent-ovs.conf.in b/agent-ovs/opflex-agent-ovs.conf.in
@@ -97,6 +97,11 @@
            // default 7200 secs, min 15 secs
            // "prr": 7200,
            //
+           // How long to wait for initial re-request, either
+           // due to a backoff, or for no response.
+           // default 10 secs, min 10 secs
+           // "policy-retry-delay": 10
+           //
            // How long to wait for the initial peer
            // handshake to complete (in ms)
            // "handshake-timeout" : 45000,

diff --git a/libopflex/engine/Processor.cpp b/libopflex/engine/Processor.cpp
@@ -49,7 +49,6 @@ using util::ThreadManager;
 using namespace internal;
 
 static const uint64_t DEFAULT_PROC_DELAY = 250;
-static const uint64_t DEFAULT_RETRY_DELAY = 1000*60*2;
 static const uint64_t FIRST_XID = (uint64_t)1 << 63;
 static const uint32_t MAX_PROCESS = 1024;
 
@@ -100,6 +99,7 @@ void Processor::setPrrTimerDuration(uint64_t duration) {
     prrTimerDuration = duration;
     policyRefTimerDuration = 1000*prrTimerDuration/2;
 }
+
 // check whether the object state index has work for us
 bool Processor::hasWork(/* out */ obj_state_by_exp::iterator& it) {
     if (obj_state.empty()) return false;

diff --git a/libopflex/engine/include/opflex/engine/Processor.h b/libopflex/engine/include/opflex/engine/Processor.h
@@ -529,7 +529,8 @@ class Processor : public internal::AbstractObjectListener,
     /**
      * Amount of time to wait before retrying message sends
      */
-    uint64_t retryDelay;
+    static const uint64_t DEFAULT_RETRY_DELAY = 1000*60*2;
+    uint64_t retryDelay = DEFAULT_RETRY_DELAY;
 
     /**
      * prr timer duration in secs

diff --git a/libopflex/include/opflex/ofcore/OFFramework.h b/libopflex/include/opflex/ofcore/OFFramework.h
@@ -735,6 +735,12 @@ class OFFramework : private boost::noncopyable {
      */
     void setPrrTimerDuration(const uint64_t duration);
 
+    /**
+     * set the policy resolve retry delay (prrd) timer durarion.
+     * @param duration timer duration in milliseconds
+     */
+    void setPolicyRetryDelayTimerDuration(const uint64_t duration);
+
     /**
      * Set the peer handshake timeout
      * @param timeout peer handshake timeout in milliseconds

diff --git a/libopflex/ofcore/OFFramework.cpp b/libopflex/ofcore/OFFramework.cpp
@@ -111,6 +111,10 @@ void OFFramework::setPrrTimerDuration(const uint64_t duration) {
     pimpl->processor.setPrrTimerDuration(duration);
 }
 
+void OFFramework::setPolicyRetryDelayTimerDuration(const uint64_t duration) {
+    pimpl->processor.setRetryDelay(duration);
+}
+
 void OFFramework::setHandshakeTimeout(const uint32_t timeout) {
     pimpl->processor.setHandshakeTimeout(timeout);
 }