Agent restart changes from Prabhjot

Change-Id: I9ab0786870f3b40863930a943e83a321cc4f5d75

src/ksync/ksync_sock.cc

@@ -250,6 +250,10 @@ void KSyncSock::SetMeasureQueueDelay(bool val) {
     }
 }
 
+void  KSyncSock::DisableSendQueue(bool val) {
+    sock_->send_queue_.set_disable(val);
+}
+
 void KSyncSock::Start(bool read_inline) {
     sock_->read_inline_ = read_inline;
     if (sock_->read_inline_) {

src/ksync/ksync_sock.h

@@ -282,6 +282,9 @@ class KSyncSock {
     uint32_t WaitTreeSize() const;
     void SetSeqno(uint32_t seq);
     void SetMeasureQueueDelay(bool val);
+
+    static void DisableSendQueue(bool val);
+
 protected:
     static void Init(bool use_work_queue);
     static void SetSockTableEntry(KSyncSock *sock);

src/ksync/ksync_tx_queue.cc

@@ -41,7 +41,8 @@ KSyncTxQueue::KSyncTxQueue(KSyncSock *sock) :
     write_events_(0),
     read_events_(0),
     busy_time_(0),
-    measure_busy_time_(false) {
+    measure_busy_time_(false),
+    disable_(false) {
     queue_len_ = 0;
     shutdown_ = false;
 }
@@ -88,6 +89,15 @@ void KSyncTxQueue::Shutdown() {
     close(event_fd_);
 }
 
+void KSyncTxQueue::set_disable(bool disable) {
+    if (disable_ != disable) {
+        disable_ = disable;
+        if (queue_len_ != 0 && !disable_) {
+            TriggerEventFd();
+        }
+    }
+}
+
 bool KSyncTxQueue::EnqueueInternal(IoContext *io_context) {
     if (work_queue_) {
         work_queue_->Enqueue(io_context);
@@ -98,23 +108,27 @@ bool KSyncTxQueue::EnqueueInternal(IoContext *io_context) {
     size_t ncount = queue_len_.fetch_and_increment() + 1;
     if (ncount > max_queue_len_)
         max_queue_len_ = ncount;
-    if (ncount == 1) {
-        uint64_t u = 1;
-        int res = 0;
-        while ((res = write(event_fd_, &u, sizeof(u))) < (int)sizeof(u)) {
-            int ec = errno;
-            if (ec != EINTR && ec != EIO) {
-                LOG(ERROR, "KsyncTxQueue write failure : " << ec << " : "
-                    << strerror(ec));
-                assert(0);
-            }
-        }
-
-        write_events_++;
+    if (ncount == 1 && !disable_) {
+        TriggerEventFd();
     }
     return true;
 }
 
+void KSyncTxQueue::TriggerEventFd() {
+    uint64_t u = 1;
+    int res = 0;
+    while ((res = write(event_fd_, &u, sizeof(u))) < (int)sizeof(u)) {
+        int ec = errno;
+        if (ec != EINTR && ec != EIO) {
+            LOG(ERROR, "KsyncTxQueue write failure : " << ec << " : "
+                << strerror(ec));
+            assert(0);
+        }
+    }
+
+    write_events_++;
+}
+
 bool KSyncTxQueue::Run() {
     while (1) {
         uint64_t u = 0;

src/ksync/ksync_tx_queue.h

@@ -82,8 +82,11 @@ class KSyncTxQueue {
         return EnqueueInternal(io_context);
     }
 
+    void set_disable(bool disable);
+
 private:
     bool EnqueueInternal(IoContext *io_context);
+    void TriggerEventFd();
 
     WorkQueue<IoContext *> *work_queue_;
     int event_fd_;
@@ -100,6 +103,7 @@ class KSyncTxQueue {
     mutable size_t read_events_;
     mutable uint64_t busy_time_;
     mutable bool measure_busy_time_;
+    bool disable_;
 
     DISALLOW_COPY_AND_ASSIGN(KSyncTxQueue);
 };

src/vnsw/agent/cmn/agent.cc

@@ -889,3 +889,11 @@ VrouterObjectLimits Agent::GetVrouterObjectLimits() {
    vr_limits.set_vrouter_max_oflow_entries(vrouter_max_oflow_entries());
    return vr_limits;
 }
+
+uint32_t Agent::ConfigWaitTime() {
+    return params_->config_wait_time();
+}
+
+void Agent::SetConfigWaitTime(uint32_t val) {
+    return params_->set_config_wait_time(val);
+}

src/vnsw/agent/cmn/agent.h

@@ -1066,6 +1066,10 @@ class Agent {
 
     void SetMeasureQueueDelay(bool val);
     bool MeasureQueueDelay();
+
+    uint32_t ConfigWaitTime();
+    void SetConfigWaitTime(uint32_t val);
+
     void TaskTrace(const char *file_name, uint32_t line_no, const Task *task,
                    const char *description, uint32_t delay);
 

src/vnsw/agent/contrail-vrouter-agent.conf

@@ -72,6 +72,9 @@ log_local=1
 # xmpp_server_key=/etc/contrail/ssl/private/server-privkey.pem
 # xmpp_ca_cert=/etc/contrail/ssl/certs/ca-cert.pem
 
+# Configure initial config complete wait time in seconds
+# config_wait_time=20
+
 [DISCOVERY]
 #If DEFAULT.collectors and/or CONTROL-NODE and/or DNS is not specified this
 #section is mandatory. Else this section is optional

src/vnsw/agent/init/agent_param.cc

@@ -494,6 +494,10 @@ void AgentParam::ParseDefaultSection() {
                                 "DEFAULT.measure_queue_delay")) {
         measure_queue_delay_ = false;
     }
+    if (!GetValueFromTree<uint32_t>(config_wait_time_,
+                                    "DEFAULT.config_wait_time")) {
+        config_wait_time_ = 1;
+    }
 }
 
 void AgentParam::ParseTaskSection() {
@@ -770,6 +774,8 @@ void AgentParam::ParseDefaultSectionArguments
                           "DEFAULT.pkt0_tx_buffers");
     GetOptValue<bool>(var_map, measure_queue_delay_,
                       "DEFAULT.measure_queue_delay");
+    GetOptValue<uint32_t>(var_map, config_wait_time_,
+                          "DEFAULT.config_wait_time");
 }
 
 void AgentParam::ParseTaskSectionArguments
@@ -1329,7 +1335,7 @@ AgentParam::AgentParam(bool enable_flow_options,
         enable_service_options_(enable_service_options),
         agent_mode_(agent_mode), gateway_mode_(NONE), vhost_(),
         pkt0_tx_buffer_count_(Agent::kPkt0TxBufferCount),
-        measure_queue_delay_(false),
+        measure_queue_delay_(false), config_wait_time_(1),
         agent_name_(), eth_port_(),
         eth_port_no_arp_(false), eth_port_encap_type_(),
         xmpp_instance_count_(),

src/vnsw/agent/init/agent_param.h

@@ -283,6 +283,10 @@ class AgentParam  {
     void set_pkt0_tx_buffer_count(uint32_t val) { pkt0_tx_buffer_count_ = val; }
     bool measure_queue_delay() const { return measure_queue_delay_; }
     void set_measure_queue_delay(bool val) { measure_queue_delay_ = val; }
+
+    uint32_t config_wait_time() const { return config_wait_time_; }
+    void set_config_wait_time(uint32_t val) { config_wait_time_ = val; }
+
 protected:
     void set_hypervisor_mode(HypervisorMode m) { hypervisor_mode_ = m; }
     virtual void InitFromSystem();
@@ -412,6 +416,8 @@ class AgentParam  {
     uint32_t pkt0_tx_buffer_count_;
     bool measure_queue_delay_;
 
+    uint32_t config_wait_time_;
+
     std::string agent_name_;
     std::string eth_port_;
     bool eth_port_no_arp_;

src/vnsw/agent/vrouter/ksync/ksync_init.cc

@@ -62,14 +62,21 @@ KSync::KSync(Agent *agent)
       ksync_flow_index_manager_(new KSyncFlowIndexManager(this)),
       qos_queue_ksync_obj_(new QosQueueKSyncObject(this)),
       forwarding_class_ksync_obj_(new ForwardingClassKSyncObject(this)),
-      qos_config_ksync_obj_(new QosConfigKSyncObject(this)) {
-      for (uint16_t i = 0; i < agent->flow_thread_count(); i++) {
-          FlowTableKSyncObject *obj = new FlowTableKSyncObject(this);
-          flow_table_ksync_obj_list_.push_back(obj);
-      }
+      qos_config_ksync_obj_(new QosConfigKSyncObject(this)),
+      config_wait_timer_(TimerManager::CreateTimer(
+                       *(agent->event_manager())->io_service(),
+                       "KSync Config wait Timer",
+                       agent->task_scheduler()->GetTaskId("Agent::KSync"), 0)) {
+    for (uint16_t i = 0; i < agent->flow_thread_count(); i++) {
+        FlowTableKSyncObject *obj = new FlowTableKSyncObject(this);
+        flow_table_ksync_obj_list_.push_back(obj);
+    }
+    config_wait_timer_->Start((agent->ConfigWaitTime() * 1000),
+                              boost::bind(&KSync::ConfigWaitCb, this));
 }
 
 KSync::~KSync() {
+    TimerManager::DeleteTimer(config_wait_timer_);
     STLDeleteValues(&flow_table_ksync_obj_list_);
 }
 
@@ -92,7 +99,7 @@ void KSync::Init(bool create_vhost) {
     NetlinkInit();
     VRouterInterfaceSnapshot();
     InitFlowMem();
-    ResetVRouter(true);
+    KSyncSock::DisableSendQueue(true);
     if (create_vhost) {
         CreateVhostIntf();
     }
@@ -102,7 +109,12 @@ void KSync::Init(bool create_vhost) {
         flow_table->set_ksync_object(flow_table_ksync_obj_list_[i]);
         flow_table_ksync_obj_list_[i]->Init();
     }
+}
+
+void KSync::InitConfigDone() {
+    ResetVRouter(true);
     ksync_flow_memory_.get()->Init();
+    KSyncSock::DisableSendQueue(false);
 }
 
 void KSync::InitDone() {
@@ -268,6 +280,11 @@ void KSync::CreateVhostIntf() {
 #endif
 }
 
+bool KSync::ConfigWaitCb() {
+    InitConfigDone();
+    return false;
+}
+
 void KSync::UpdateVhostMac() {
 #if defined(__linux__)
     struct  nl_client *cl;
@@ -385,13 +402,19 @@ void KSyncTcp::Init(bool create_vhost) {
     TcpInit();
     VRouterInterfaceSnapshot();
     InitFlowMem();
-    ResetVRouter(false);
-    //Start async read of socket
-    KSyncSockTcp *sock = static_cast<KSyncSockTcp *>(KSyncSock::Get(0));
-    sock->AsyncReadStart();
+    KSyncSock::DisableSendQueue(true);
     interface_ksync_obj_.get()->Init();
     for (uint16_t i = 0; i < flow_table_ksync_obj_list_.size(); i++) {
         flow_table_ksync_obj_list_[i]->Init();
     }
+}
+
+void KSyncTcp::InitConfigDone() {
+    ResetVRouter(false);
+    //Start async read of socket
+    KSyncSockTcp *sock = static_cast<KSyncSockTcp *>(KSyncSock::Get(0));
+    sock->AsyncReadStart();
     ksync_flow_memory_.get()->Init();
+    KSyncSock::DisableSendQueue(false);
 }
+

src/vnsw/agent/vrouter/ksync/ksync_init.h

@@ -5,6 +5,7 @@
 #ifndef vnsw_agent_ksync_init_h
 #define vnsw_agent_ksync_init_h
 
+#include <base/timer.h>
 #include <vrouter/ksync/flowtable_ksync.h>
 #include <vrouter/ksync/mpls_ksync.h>
 #include <vrouter/ksync/nexthop_ksync.h>
@@ -29,6 +30,7 @@ class KSync {
     virtual ~KSync();
 
     virtual void Init(bool create_vhost);
+    virtual void InitConfigDone();
     virtual void InitDone();
     virtual void RegisterDBClients(DB *db);
     void VnswInterfaceListenerInit();
@@ -100,6 +102,10 @@ class KSync {
 private:
     void NetlinkInit();
     void CreateVhostIntf();
+
+    bool ConfigWaitCb();
+
+    Timer *config_wait_timer_;
     DISALLOW_COPY_AND_ASSIGN(KSync);
 };
 
@@ -108,6 +114,7 @@ class KSyncTcp : public KSync {
     KSyncTcp(Agent *agent);
     virtual ~KSyncTcp();
     virtual void Init(bool create_vhost);
+    virtual void InitConfigDone();
     void TcpInit();
 protected:
     virtual void InitFlowMem();