Enabling introspect ssl tests to run in microservices setup

    1. Update common.sh in place of config file of
	contrail services
    2. Optimize method add_knob_to_container
    3. Optimizing few methods to verify service
	status with ssl certs
    4. Fix for intermittent failure of test_flow_action_drop_stats

Change-Id: I2ac282e8f2475fc38280dab8783043632b6c8228
Closes-bug: #1777107
Partial-bug: #1777106

common/base.py

@@ -82,45 +82,65 @@ def start_containers(self, node_ip, containers):
             nodes=[node_ip])[0]
     # end start_containers
 
-    def add_knob_to_container(self, node_ip, container_name, level='DEFAULT', knob=None):
-        ''' Add a configuration knob to container at specified level
+    def add_knob_to_container(self, node_ip, container_name, level='DEFAULT',
+        knob=None, restart_container=True, file_name='entrypoint.sh'):
+        ''' Add/update a configuration knob to container via common.sh or entrypoint.sh
+                For entrypoint.sh, it can add at specified level
+                For common.sh, it can only edit the existing knob and
+                    level must be None in this case
             Args:
                 node_ip         : Node on which containers need to be stopped
                 container_name  : Name of the container
+                file_name       : config script file name
                 level           : Hierarchy level where knob needs to be added
-                knob            : Knob which needs to be added
+                knob            : Knob which needs to be added or list of knobs
             E.g: add_knob_to_container('10.204.217.127', 'control_control_1',
             'DEFAULT', 'mvpn_ipv4_enable=1')
         '''
 
-        issue_cmd = 'docker cp %s:/entrypoint.sh .' % (container_name)
+        issue_cmd = 'docker cp %s:/%s .' % (container_name, file_name)
         username = self.inputs.host_data[node_ip]['username']
         password = self.inputs.host_data[node_ip]['password']
 
         self.logger.info('Running %s on %s' % (issue_cmd, node_ip))
         self.inputs.run_cmd_on_server(node_ip, issue_cmd, username, password, pty=True,
                                     as_sudo=True)
 
-        just_knob = knob[:knob.find('=')] 
-        issue_cmd = 'sed -i -e \'/'+just_knob+'/d\' entrypoint.sh'
-        self.logger.info('Running %s on %s' % (issue_cmd, node_ip))
-        self.inputs.run_cmd_on_server(node_ip, issue_cmd, username, password, pty=True,
-                                    as_sudo=True)
-        issue_cmd = 'grep -q -F \''+knob+'\' entrypoint.sh ||' + \
-            'sed -i  \'/\['+level+'\]/a '+knob+'\' entrypoint.sh'
-        self.logger.info('Running %s on %s' % (issue_cmd, node_ip))
-        self.inputs.run_cmd_on_server(node_ip, issue_cmd, username, password, pty=True,
+        knob_list = [knob] if isinstance(knob, str) else knob
+
+        for knob in knob_list:
+            just_knob = knob[:knob.find('=')]
+            if level is not None:
+                #Delete the existing knob
+                issue_cmd = 'sed -i -e \'/'+just_knob+'/d\' %s' % (file_name)
+                self.logger.info('Running %s on %s' % (issue_cmd, node_ip))
+                self.inputs.run_cmd_on_server(node_ip, issue_cmd, username, password, pty=True,
+                                            as_sudo=True)
+                #Insert the new knob at given level
+                issue_cmd = 'grep -q -F \''+knob+'\' %s ||' % (file_name) + \
+                    'sed -i  \'/\['+level+'\]/a '+knob+'\' %s' % (file_name)
+            else:
+                #Replace the existing knob with new value
+                #Append at next line of first match then delete first match
+                issue_cmd = 'sed -i \'/'+just_knob+'=.*/a %s\' %s' % (knob, file_name)
+                issue_cmd = issue_cmd + ';sed -i \'0,/%s=.*/{//d}\' %s' % (
+                    just_knob, file_name)
+
+            self.logger.info('Running %s on %s' % (issue_cmd, node_ip))
+            self.inputs.run_cmd_on_server(node_ip, issue_cmd, username, password, pty=True,
                                     as_sudo=True)
 
-        issue_cmd = 'docker cp entrypoint.sh %s:/entrypoint.sh' % (container_name)
+        issue_cmd = 'docker cp %s %s:/%s' % (file_name, container_name,
+            file_name)
         self.logger.info('Running %s on %s' % (issue_cmd, node_ip))
         self.inputs.run_cmd_on_server(node_ip, issue_cmd, username, password, pty=True,
                                     as_sudo=True)
 
-        issue_cmd = 'docker restart %s -t 60' % (container_name)
-        self.logger.info('Running %s on %s' % (issue_cmd, node_ip))
-        self.inputs.run_cmd_on_server(node_ip, issue_cmd, username, password, pty=True,
-                                    as_sudo=True)
+        if restart_container:
+            issue_cmd = 'docker restart %s -t 60' % (container_name)
+            self.logger.info('Running %s on %s' % (issue_cmd, node_ip))
+            self.inputs.run_cmd_on_server(node_ip, issue_cmd, username, password, pty=True,
+                                        as_sudo=True)
 
 # end _GenericTestBaseMethods
 

common/contrail_test_init.py

@@ -1526,11 +1526,14 @@ def verify_state(self):
         return result
     # end verify_state
 
-    def verify_service_state(self, host, service=None, role=None):
+    def verify_service_state(self, host, service=None, role=None,
+            tries=15, delay=5, expected_state=None,
+            keyfile=None, certfile=None, cacert=None):
         '''
         Based on name of service, it decides whether its a service name like
         "contrail-vrouter-agent", container name like "agent" or a non contrail service
         like docker.
+         expected_state: service status expected, like active, backup etc.
         '''
         contrail_svc = []
         non_contrail_svc = []
@@ -1548,7 +1551,9 @@ def verify_service_state(self, host, service=None, role=None):
                 return self.verify_non_contrail_service_state(host,
                                                               non_contrail_svc)
         return ContrailStatusChecker(self).wait_till_contrail_cluster_stable(
-            host, role, contrail_svc, tries=15, delay=5)
+            host, role, contrail_svc, tries=tries, delay=delay,
+            expected_state=expected_state,
+            keyfile=keyfile, certfile=certfile, cacert=cacert)
     #end verify_service_state
 
     def verify_service_down(self, host, service=None, role=None):
@@ -1999,6 +2004,8 @@ def copy_file_to_server(self, ip, src, dstdir, dst, force=False,
             container = None
             self.logger.debug('Container %s not in host %s, copying to '
                 ' host itself' % (container, ip))
+        else:
+            container=self.host_data[ip].get('containers', {}).get(container)
         copy_file_to_server(host, src, dstdir, dst, force, container=container)
 
     def copy_file_from_server(self, ip, src_file_path, dest_folder,

common/introspect/base.py

@@ -11,7 +11,7 @@
 from common.contrail_test_init import DEFAULT_CERT, DEFAULT_PRIV_KEY, DEFAULT_CA
 
 CERT_LOCATION = '/tmp/'
-
+DOCKER_CONF_FILE1 = 'common.sh'
 CONTRAIL_CONF_FILES = {
     'contrail-vrouter-agent': '/etc/contrail/contrail-vrouter-agent.conf',
     'contrail-analytics-api': '/etc/contrail/contrail-analytics-api.conf',
@@ -151,7 +151,7 @@ def copy_certs_on_node(self, node_ip, cert_list, dstdir=CERT_LOCATION, container
 
         for cert in cert_list:
             self.inputs.copy_file_to_server(node_ip, cert, self.cert_location,
-                cert.split('/')[-1], container=container)
+                cert.split('/')[-1], container=container, force=True)
             self.addCleanup(self.delete_cert_file, node_ip,
                 dstdir+cert.split('/')[-1], container)
 
@@ -162,8 +162,9 @@ def create_agent_certs_and_update_on_compute(self, host_ip, subject,
         agent_key, agent_csr, agent_cert = self.create_cert(subject=subject,
             subjectAltName=subjectAltName)
 
+        cntr = CONTRAIL_SERVICE_CONTAINER[service]
         self.copy_certs_on_node(host_ip, [agent_key, agent_cert,
-            self.ca_cert], container=container)
+            self.ca_cert], container=cntr)
 
         self.update_config_file_and_restart_service(host_ip,
             CONTRAIL_CONF_FILES[service], ssl_enable, agent_key,
@@ -172,17 +173,18 @@ def create_agent_certs_and_update_on_compute(self, host_ip, subject,
 
         self.inputs.restart_service(service, [host_ip], container=container,
             verify_service=False)
-        return self.inputs.confirm_service_active(service, host_ip, container,
-            certs_dict={'key': agent_key, 'cert': agent_cert, 'ca': self.ca_cert})
+        return self.inputs.verify_service_state(host_ip, service,
+            tries=30, delay=5, expected_state='active')[0]
 
     def restore_default_config_file(self, conf_file_backup, service_name, node_ip,
-            container=None, verify_in_cleanup=True):
+            container=None, verify_in_cleanup=True, original_conf_file_name=None):
 
-        cmd = "mv %s %s" % (conf_file_backup, CONTRAIL_CONF_FILES[service_name])
+        dst_file = original_conf_file_name or DOCKER_CONF_FILE1
+        cmd = 'docker cp %s %s:/%s;rm -f %s' % (conf_file_backup, container,
+            dst_file, conf_file_backup)
         output = self.inputs.run_cmd_on_server(
             node_ip,
-            cmd,
-            container=container)
+            cmd)
 
         self.inputs.introspect_insecure = self.introspect_insecure_old
         self.inputs.restart_service(service_name, [node_ip], container=container,
@@ -210,41 +212,35 @@ def update_contrail_conf(self, service, operation, section, option, value,
 
     def update_config_file_and_restart_service(self, node_ip, conf_file, ssl_enable, keyfile,
             certfile, ca_certfile, service_name, container_name=None,
-            verify_service=True, verify_in_cleanup=True):
+            verify_service=True, verify_in_cleanup=True, tries=30, delay=5):
         '''
         set the introspect ssl configurations and restart the service
         '''
 
         self.logger.info('Set introspect ssl configs in node %s' % (node_ip))
-
-        #Take backup of original conf file to revert back later
-        conf_file_backup = CERT_LOCATION + get_random_name(conf_file.split('/')[-1])
-        cmd = 'cp %s %s' % (conf_file, conf_file_backup)
+        conf_file = DOCKER_CONF_FILE1
+        #Take backup of original common.sh file to revert back later
+        conf_file_backup = CERT_LOCATION + get_random_name(container_name+conf_file)
+        cmd = 'docker cp %s:%s %s' % (container_name, conf_file, conf_file_backup)
         status = self.inputs.run_cmd_on_server(node_ip, cmd, container=container_name)
 
-        oper = 'set'
-        section = 'SANDESH'
-        self.update_contrail_conf(service_name, oper, section,
-            'introspect_ssl_enable', ssl_enable, node_ip, container_name)
-        self.update_contrail_conf(service_name, oper, section,
-            'sandesh_keyfile', keyfile, node_ip, container_name)
-        self.update_contrail_conf(service_name, oper, section,
-            'sandesh_certfile', certfile, node_ip, container_name)
-        self.update_contrail_conf(service_name, oper, section,
-            'sandesh_ca_cert', ca_certfile, node_ip, container_name)
+        self.add_knob_to_container(node_ip, container_name,
+            level=None, knob=[
+            'INTROSPECT_SSL_ENABLE=%s' % (ssl_enable),
+            'SANDESH_KEYFILE=%s' % (keyfile),
+            'SANDESH_CERTFILE=%s' % (certfile),
+            'SANDESH_CA_CERTFILE=%s' % (ca_certfile)],
+            file_name=DOCKER_CONF_FILE1, restart_container=verify_service)
 
         self.addCleanup(
             self.restore_default_config_file,
             conf_file_backup, service_name, node_ip, container_name,
-            verify_in_cleanup)
+            verify_in_cleanup, original_conf_file_name=conf_file)
 
         if verify_service:
-            self.inputs.restart_service(service_name, [node_ip],
-                container=container_name, verify_service=False)
-            return self.inputs.confirm_service_active(service_name, node_ip,
-                container_name, certs_dict={'key': keyfile,
-                                            'cert': certfile,
-                                            'ca': ca_certfile})
+            return self.inputs.verify_service_state(node_ip, service_name,
+                tries=tries, delay=delay, expected_state='active',
+                keyfile=keyfile, certfile=certfile, cacert=ca_certfile)[0]
 
     def get_introspect_for_service(self, service, host_ip):
         if service == 'contrail-svc-monitor':

common/introspect/openssl.cnf

@@ -0,0 +1,8 @@
+[req]
+default_bits = 2048
+prompt = no
+default_md = sha256
+default_days = 375
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+commonName = Common Name(eg, Host/Domain/Node name)

common/vrouter/base_drop_stats.py

@@ -25,12 +25,16 @@ def print_drop_stats_dict(self, d):
             self.logger.info("%s : %s"  % (k, v))
     # end print_drop_stats_dict
 
+    @retry(delay=2, tries=15)
     def get_drop_stats_dict(self, compute, fq_name=None, module='vif'):
         if module == 'vrouter':
             drop_stats_list = self.agent_inspect_h[compute].get_agent_vrouter_drop_stats()
         else:
             drop_stats_list = self.agent_inspect_h[compute].get_agent_vm_interface_drop_stats(fq_name)
-        return drop_stats_list
+
+        if drop_stats_list is not None:
+            return True, drop_stats_list
+        return False, None
 
     def verify_flow_action_drop_stats(self, drop_type='ds_flow_action_drop'):
         result = True
@@ -55,18 +59,18 @@ def verify_flow_action_drop_stats(self, drop_type='ds_flow_action_drop'):
         ip_addr = intf_details[0]['ip_addr']
         fq_name = intf_details[0]['config_name']
 
-        vif_dict_before = self.get_drop_stats_dict(compute0, fq_name)
+        vif_dict_before = self.get_drop_stats_dict(compute0, fq_name)[1]
         self.print_drop_stats_dict(vif_dict_before)
 
-        vrouter_dict_before = self.get_drop_stats_dict(compute0, module='vrouter')
+        vrouter_dict_before = self.get_drop_stats_dict(compute0, module='vrouter')[1]
         self.print_drop_stats_dict(vrouter_dict_before)
 
         assert not vm1_fixture.ping_to_ip(vm2_ip, count=ping_count)
 
         count = 0
         while True:
-            vif_dict_after = self.get_drop_stats_dict(compute0, fq_name)
-            vrouter_dict_after = self.get_drop_stats_dict(compute0, module='vrouter')
+            vif_dict_after = self.get_drop_stats_dict(compute0, fq_name)[1]
+            vrouter_dict_after = self.get_drop_stats_dict(compute0, module='vrouter')[1]
             vif_stats = self.verify_dropstats_of_type(drop_type, vif_dict_before,
                 vif_dict_after, ping_count)
             vrouter_stats = self.verify_dropstats_of_type(drop_type, vrouter_dict_before,