Merge "Remove hardcoded references to overcloud credentials"

cloud-bulldozer · Aug 3, 2017 · 3cfdf3b · 3cfdf3b
2 parents 4c34cea + 8d11344
commit 3cfdf3b
Show file tree

Hide file tree

Showing 21 changed files with 51 additions and 27 deletions.
diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2
@@ -2,6 +2,7 @@ browbeat:
   results : results/
   rerun: 3
   cloud_name: {{ browbeat_cloud_name }}
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: {{ elastic_enabled }}
   host: {{ elastic_host }}
@@ -42,7 +43,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - netcreate-boot: rally/rally-plugins/netcreate-boot
     - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping

diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2
@@ -4,6 +4,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: {{ browbeat_cloud_name }}
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: {{ elastic_enabled }}
   host: {{ elastic_host }}
@@ -44,7 +45,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - netcreate-boot: rally/rally-plugins/netcreate-boot
     - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping

diff --git a/browbeat-complete.yaml b/browbeat-complete.yaml
@@ -3,6 +3,7 @@ browbeat:
   results : results/
   rerun: 3
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   regather: false
@@ -43,7 +44,7 @@ perfkit:
   enabled: true
   sleep_before: 0
   sleep_after: 0
-  venv: /home/stack/perfkit-venv/bin/activate
+  venv: /home/stack/perfkit-venv
   default:
     image: centos7
     machine_type: m1.small
@@ -62,7 +63,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - netcreate-boot: rally/rally-plugins/netcreate-boot
     - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping

diff --git a/browbeat-config.yaml b/browbeat-config.yaml
@@ -3,6 +3,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: true
   host: 1.1.1.1
@@ -43,7 +44,7 @@ perfkit:
   enabled: false
   sleep_before: 0
   sleep_after: 0
-  venv: /home/stack/perfkit-venv/bin/activate
+  venv: /home/stack/perfkit-venv
   default:
     image: centos7
     machine_type: m1.small
@@ -95,7 +96,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - netcreate-boot: rally/rally-plugins/netcreate-boot
     - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping

diff --git a/conf/browbeat-glance-complete.yaml b/conf/browbeat-glance-complete.yaml
@@ -3,6 +3,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   regather: false
@@ -43,7 +44,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - glance-create-boot-delete: rally/rally-plugins/glance-create-boot-delete
   benchmarks:

diff --git a/conf/browbeat-keystone-complete.yaml b/conf/browbeat-keystone-complete.yaml
@@ -4,6 +4,7 @@ browbeat:
   results : results/
   rerun: 3
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   regather: false
@@ -45,7 +46,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   benchmarks:
     - name: authenticate
       enabled: true

diff --git a/conf/browbeat-keystone-minimal.yaml b/conf/browbeat-keystone-minimal.yaml
@@ -4,6 +4,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   regather: false
@@ -45,7 +46,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   benchmarks:
     - name: authenticate
       enabled: true

diff --git a/conf/ceilometer-minimal.yaml b/conf/ceilometer-minimal.yaml
@@ -3,6 +3,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   regather: false
@@ -43,7 +44,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - validate-ceilometer: rally/rally-plugins/validate-ceilometer
     - nova-ceilometer: rally/rally-plugins/nova-ceilometer

diff --git a/conf/ceilometer-persist-response-timing.yaml b/conf/ceilometer-persist-response-timing.yaml
@@ -6,6 +6,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   regather: false
@@ -46,7 +47,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - nova: rally/rally-plugins/nova
   benchmarks:

diff --git a/conf/gnocchi-minimal.yaml b/conf/gnocchi-minimal.yaml
@@ -4,6 +4,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   host: 1.1.1.1
@@ -44,7 +45,7 @@ rally:
   enabled: true
   sleep_before: 0
   sleep_after: 0
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - gnocchi: rally/rally-plugins/gnocchi
   benchmarks:

diff --git a/conf/perfkit-benchmarks.yaml b/conf/perfkit-benchmarks.yaml
@@ -8,6 +8,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   regather: false
@@ -48,7 +49,7 @@ perfkit:
   enabled: true
   sleep_before: 0
   sleep_after: 0
-  venv: /home/stack/perfkit-venv/bin/activate
+  venv: /home/stack/perfkit-venv
   default:
     image: centos7
     machine_type: m1.small

diff --git a/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml
@@ -10,6 +10,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   host: 1.1.1.1
@@ -50,7 +51,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - nova: rally/rally-plugins/nova
     - neutron: rally/rally-plugins/neutron

diff --git a/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml
@@ -10,6 +10,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   host: 1.1.1.1
@@ -50,7 +51,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - nova: rally/rally-plugins/nova
     - neutron: rally/rally-plugins/neutron

diff --git a/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml
@@ -10,6 +10,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   host: 1.1.1.1
@@ -50,7 +51,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - nova: rally/rally-plugins/nova
     - neutron: rally/rally-plugins/neutron

diff --git a/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml
@@ -10,6 +10,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   host: 1.1.1.1
@@ -50,7 +51,7 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - nova: rally/rally-plugins/nova
     - neutron: rally/rally-plugins/neutron

diff --git a/conf/telemetry-nova-persist-instances-with-network.yaml b/conf/telemetry-nova-persist-instances-with-network.yaml
@@ -8,6 +8,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   host: 1.1.1.1
@@ -50,7 +51,7 @@ rally:
   # Sleeps 30 minutes after each iteration to allow for system resources to degrade or
   # display stepped degrading (If the system is hitting saturation or high utilization)
   sleep_after: 1800
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - nova: rally/rally-plugins/nova
     - neutron: rally/rally-plugins/neutron

diff --git a/conf/telemetry-nova-persist-instances.yaml b/conf/telemetry-nova-persist-instances.yaml
@@ -7,6 +7,7 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
+  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   regather: false
@@ -50,7 +51,7 @@ rally:
   # Sleeps 30 minutes after each iteration to allow for system resources to degrade or
   # display stepped degrading (If the system is hitting saturation or high utilization)
   sleep_after: 1800
-  venv: /home/stack/rally-venv/bin/activate
+  venv: /home/stack/rally-venv
   plugins:
     - nova: rally/rally-plugins/nova
   benchmarks:

diff --git a/lib/PerfKit.py b/lib/PerfKit.py
@@ -89,7 +89,7 @@ def index_results(self, sucessful_run, result_dir, test_name, browbeat_rerun, be
                     as perfkit_results_json:
                 for result_count, json_result in enumerate(perfkit_results_json):
                     complete_result_json = {'browbeat_scenario': benchmark_config}
-                    complete_result_json['results'] = {'unit':{}, 'value': {}}
+                    complete_result_json['results'] = {'unit': {}, 'value': {}}
                     single_result = self.elastic.load_json(json_result.strip())
                     complete_result_json['browbeat_rerun'] = browbeat_rerun
                     complete_result_json['timestamp'] = str(es_ts).replace(" ", "T")
@@ -125,9 +125,11 @@ def run_benchmark(self, benchmark_config, result_dir, test_name, cloud_type="Ope
         # Build command to run
         if 'enabled' in benchmark_config:
             del benchmark_config['enabled']
-        cmd = ("source /home/stack/overcloudrc; source {0}; "
+        cmd = ("source {0}/bin/activate; source {1} "
                "/home/stack/perfkit-venv/PerfKitBenchmarker/pkb.py "
-               "--cloud={1} --run_uri=browbeat".format(self.config['perfkit']['venv'], cloud_type))
+               "--cloud={2} --run_uri=browbeat".format(
+                   self.config['perfkit']['venv'],
+                   self.config['browbeat']['overcloud_credentials'], cloud_type))
         for parameter, value in benchmark_config.iteritems():
             if not parameter == 'name':
                 self.logger.debug(

diff --git a/lib/Rally.py b/lib/Rally.py
@@ -62,7 +62,7 @@ def run_scenario(self, task_file, scenario_args, result_dir, test_name, benchmar
         plugin_string = ""
         if len(plugins) > 0:
             plugin_string = "--plugin-paths {}".format(",".join(plugins))
-        cmd = "source {}; ".format(self.config['rally']['venv'])
+        cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
         cmd += "rally {} task start {} --task-args \'{}\' 2>&1 | tee {}.log".format(
             plugin_string, task_file, task_args, test_name)
         from_time = time.time()
@@ -108,18 +108,18 @@ def _get_details(self):
 
     def gen_scenario_html(self, task_ids, test_name):
         all_task_ids = ' '.join(task_ids)
-        cmd = "source {}; ".format(self.config['rally']['venv'])
+        cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
         cmd += "rally task report --task {} --out {}.html".format(
             all_task_ids, test_name)
         return self.tools.run_cmd(cmd)['stdout']
 
     def gen_scenario_json(self, task_id):
-        cmd = "source {}; ".format(self.config['rally']['venv'])
+        cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
         cmd += "rally task results {}".format(task_id)
         return self.tools.run_cmd(cmd)['stdout']
 
     def gen_scenario_json_file(self, task_id, test_name):
-        cmd = "source {}; ".format(self.config['rally']['venv'])
+        cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
         cmd += "rally task results {} > {}.json".format(task_id, test_name)
         return self.tools.run_cmd(cmd)['stdout']
 

diff --git a/lib/Shaker.py b/lib/Shaker.py
@@ -357,7 +357,8 @@ def run_scenario(self, scenario, result_dir, test_name, filename,
         self.logger.info(
             "The uuid for this shaker scenario is {}".format(shaker_uuid))
         cmd_env = (
-            "source {}/bin/activate; source /home/stack/overcloudrc").format(venv)
+            "source {0}/bin/activate; source {1}").format(venv, self.config['browbeat']
+                                                          ['overcloud_credentials'])
         if 'external' in filename and 'external_host' in self.config['shaker']:
             external_host = self.config['shaker']['external_host']
             cmd_shaker = (

diff --git a/lib/validate.yaml b/lib/validate.yaml
@@ -15,6 +15,10 @@ mapping:
       cloud_name:
         type: str
         required: True
+      overcloud_credentials:
+        type: str
+        required: True
+
 
   elasticsearch:
     required: True