Merge "Auto venv via path module"

.gitignore

@@ -1,11 +1,11 @@
 *.py[cod]
+.tox
 
 # Virtual Environments:
 .browbeat-venv
 .perfkit-venv
 .rally-venv
 .shaker-venv
-.yoda-venv
 
 # OpenStack Environment Files
 stackrc

ansible/install/group_vars/all.yml

@@ -25,25 +25,22 @@ overcloudrc: "{{home_dir}}/overcloudrc"
 # overcloud_ca_path: /etc/pki/ca-trust/source/anchors/overcloud.crt.pem
 
 # The default Browbeat venv
-browbeat_venv: "{{home_dir}}/browbeat-venv"
+browbeat_venv: "{{browbeat_path}}/.browbeat-venv"
 
 # The default Rally venv
-rally_venv: "{{home_dir}}/rally-venv"
+rally_venv: "{{browbeat_path}}/.rally-venv"
 
 # Rally version to install
 rally_version: 0.8.1
 
 # The default Shaker venv
-shaker_venv: "{{home_dir}}/shaker-venv"
+shaker_venv: "{{browbeat_path}}/.shaker-venv"
 
 # Shaker version to Install
 shaker_version: 1.1.0
 
-# The default YODA venv
-yoda_venv: /home/stack/yoda-venv
-
 # PerfKitBenchmarker Settings
-perfkit_venv: "{{home_dir}}/perfkit-venv"
+perfkit_venv: "{{browbeat_path}}/.perfkit-venv"
 perfkit_version: v1.12.0
 
 # Configuration items to adjust browbeat results served through httpd

ansible/oooq/roles/browbeat-run/tasks/main.yml

@@ -2,6 +2,6 @@
 
 - name: Run Browbeat
   shell:
-    "source {{ ansible_env.HOME }}/browbeat-venv/bin/activate; \
+    "source {{ ansible_env.HOME }}/browbeat/.browbeat-venv/bin/activate; \
      cd {{ ansible_env.HOME }}/browbeat/; \
      python browbeat.py all"

ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2

@@ -2,7 +2,6 @@ browbeat:
   results : results/
   rerun: 3
   cloud_name: {{ browbeat_cloud_name }}
-  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: {{ elastic_enabled }}
   host: {{ elastic_host }}

ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2

@@ -1,10 +1,9 @@
-# Tests to be compleated for the install-and-check.sh script minimal and short workloads are performed
+# Tests to be completed for the install-and-check.sh script minimal and short workloads are performed
 #  to confirm functionality.
 browbeat:
   results : results/
   rerun: 1
   cloud_name: {{ browbeat_cloud_name }}
-  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: {{ elastic_enabled }}
   host: {{ elastic_host }}

ansible/oooq/roles/template-configs/templates/browbeat-yoda-ci.yaml.j2

@@ -4,7 +4,6 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: {{ browbeat_cloud_name }}
-  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: {{ elastic_enabled }}
   host: {{ elastic_host }}

browbeat-complete.yaml

@@ -3,7 +3,6 @@ browbeat:
   results : results/
   rerun: 3
   cloud_name: openstack
-  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: false
   regather: false
@@ -40,7 +39,6 @@ perfkit:
   enabled: true
   sleep_before: 0
   sleep_after: 0
-  venv: /home/stack/perfkit-venv
   default:
     image: centos7
     machine_type: m1.small
@@ -59,7 +57,6 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv
   plugins:
     - netcreate-boot: rally/rally-plugins/netcreate-boot
     - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping
@@ -275,7 +272,6 @@ shaker:
   join_timeout: 600
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/shaker-venv
   shaker_region: regionOne
   external_host: 2.2.2.2
   scenarios:
@@ -371,7 +367,6 @@ yoda:
   enabled: false
   instackenv: "/home/stack/instackenv.json"
   stackrc: "/home/stack/stackrc"
-  venv: "/home/stack/yoda-venv/bin/activate"
   benchmarks:
     - name: scale-deploy
       type: overcloud

browbeat-config.yaml

@@ -3,7 +3,6 @@ browbeat:
   results : results/
   rerun: 1
   cloud_name: openstack
-  overcloud_credentials: /home/stack/overcloudrc
 elasticsearch:
   enabled: true
   host: 1.1.1.1
@@ -40,7 +39,6 @@ perfkit:
   enabled: false
   sleep_before: 0
   sleep_after: 0
-  venv: /home/stack/perfkit-venv
   default:
     image: centos7
     machine_type: m1.small
@@ -63,7 +61,6 @@ shaker:
   join_timeout: 600
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/shaker-venv
   shaker_region: regionOne
   external_host: 2.2.2.2
   scenarios:
@@ -92,7 +89,6 @@ rally:
   enabled: true
   sleep_before: 5
   sleep_after: 5
-  venv: /home/stack/rally-venv
   plugins:
     - netcreate-boot: rally/rally-plugins/netcreate-boot
     - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping

browbeat.py

@@ -13,18 +13,15 @@
 
 import argparse
 import datetime
-import browbeat.elastic
-import browbeat.perfkit
-import browbeat.rally
-import browbeat.shaker
-import browbeat.yoda
-import browbeat.workloadbase
-import browbeat.tools
 import logging
 import os
 import sys
 import time
 
+import browbeat.elastic
+import browbeat.tools
+import browbeat.workloadbase
+
 _workload_opts = ['perfkit', 'rally', 'shaker', 'yoda']
 _config_file = 'browbeat-config.yaml'
 debug_log_file = 'log/debug.log'

browbeat/elastic.py

@@ -11,16 +11,17 @@
 #   limitations under the License.
 
 from collections import deque
-import elasticsearch
-from elasticsearch import helpers
-import logging
-import json
 import datetime
-import uuid
-import sys
-import time
+import json
+import logging
 import os
 import re
+import sys
+import time
+import uuid
+
+import elasticsearch
+from elasticsearch import helpers
 
 browbeat_uuid = uuid.uuid4()
 

browbeat/path.py

@@ -0,0 +1,69 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import logging
+import os
+
+
+_logger = logging.getLogger('browbeat.path')
+
+# Browbeat's project modules path, typically /home/stack/browbeat/browbeat
+browbeat_project_path = os.path.dirname(os.path.realpath(__file__))
+
+# Path to Browbeat, typically /home/stack/browbeat
+browbeat_path = os.path.abspath(os.path.join(browbeat_project_path, os.pardir))
+
+# Base Path, typically /home/stack
+base_path = os.path.abspath(os.path.join(browbeat_path, os.pardir))
+
+conf_path = os.path.join(browbeat_path, 'conf')
+
+log_path = os.path.join(browbeat_path, 'log')
+
+results_path = os.path.join(browbeat_path, 'results')
+
+def get_overcloudrc():
+    """Check the several expected locations for the overcloudrc file:
+    * $base_path/overcloudrc
+    * $browbeat_path/overcloudrc
+    """
+    paths = [
+        os.path.join(base_path, 'overcloudrc'),
+        os.path.join(browbeat_path, 'overcloudrc')
+    ]
+    for overcloudrc_file in paths:
+        if os.path.exists(overcloudrc_file):
+            return overcloudrc_file
+        else:
+            _logger.debug("overcloudrc not found in {}".format(overcloudrc_file))
+    _logger.error('overcloudrc file can not be found')
+
+def get_workload_venv(workload, path_activate):
+    """Check the several expected locations for a workload's venv and return the existing venv:
+    * $base_path/$workload-venv
+    * $base_path/.$workload-venv
+    * $browbeat_path/.$workload-venv
+    """
+    paths = [
+        os.path.join(base_path, '{}-venv'.format(workload)),
+        os.path.join(base_path, '.{}-venv'.format(workload)),
+        os.path.join(browbeat_path, '.{}-venv'.format(workload))
+    ]
+    for workload_venv_path in paths:
+        if os.path.isdir(workload_venv_path):
+            if path_activate and os.path.exists(os.path.join(workload_venv_path, 'bin/activate')):
+                return os.path.join(workload_venv_path, 'bin/activate')
+            else:
+                return workload_venv_path
+        else:
+            _logger.debug("{} not installed in {}".format(workload, workload_venv_path))
+    _logger.error('{} does not appear to be installed correctly'.format(workload))

browbeat/perfkit.py

@@ -11,24 +11,27 @@
 #   limitations under the License.
 
 import ast
-import connmon
 import datetime
-import elastic
 import glob
-import grafana
 import logging
 import os
 import shutil
 import subprocess
 import time
+
+import connmon
+import elastic
+import grafana
+from path import get_overcloudrc
+from path import get_workload_venv
 import tools
 import workloadbase
 
-
 class PerfKit(workloadbase.WorkloadBase):
 
     def __init__(self, config):
         self.logger = logging.getLogger('browbeat.perfkit')
+        self.overcloudrc = get_overcloudrc()
         self.config = config
         self.error_count = 0
         self.tools = tools.Tools(self.config)
@@ -125,11 +128,12 @@ def run_benchmark(self, benchmark_config, result_dir, test_name, cloud_type="Ope
         # Build command to run
         if 'enabled' in benchmark_config:
             del benchmark_config['enabled']
-        cmd = ("source {0}/bin/activate; source {1} "
-               "/home/stack/perfkit-venv/PerfKitBenchmarker/pkb.py "
-               "--cloud={2} --run_uri=browbeat".format(
-                   self.config['perfkit']['venv'],
-                   self.config['browbeat']['overcloud_credentials'], cloud_type))
+        cmd = ("source {0}; source {1}; "
+               "{2}/PerfKitBenchmarker/pkb.py "
+               "--cloud={3} --run_uri=browbeat".format(
+                   get_workload_venv('perfkit', True),
+                   self.overcloudrc,
+                   get_workload_venv('perfkit', False), cloud_type))
         for parameter, value in benchmark_config.iteritems():
             if not parameter == 'name':
                 self.logger.debug(

browbeat/rally.py

@@ -11,19 +11,21 @@
 #   limitations under the License.
 
 import collections
-import connmon
 import datetime
-import elastic
 import glob
-import grafana
+import json
 import logging
 import os
 import re
 import shutil
 import time
-import tools
+
+import connmon
+import elastic
+import grafana
+from path import get_workload_venv
 import workloadbase
-import json
+import tools
 
 
 class Rally(workloadbase.WorkloadBase):
@@ -61,7 +63,7 @@ def run_scenario(self, task_file, scenario_args, result_dir, test_name, benchmar
         plugin_string = ""
         if len(plugins) > 0:
             plugin_string = "--plugin-paths {}".format(",".join(plugins))
-        cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
+        cmd = "source {}; ".format(get_workload_venv('rally', True))
         cmd += "rally {} task start {} --task-args \'{}\' 2>&1 | tee {}.log".format(
             plugin_string, task_file, task_args, test_name)
         from_time = time.time()
@@ -104,18 +106,18 @@ def _get_details(self):
 
     def gen_scenario_html(self, task_ids, test_name):
         all_task_ids = ' '.join(task_ids)
-        cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
+        cmd = "source {}; ".format(get_workload_venv('rally', True))
         cmd += "rally task report --task {} --out {}.html".format(
             all_task_ids, test_name)
         return self.tools.run_cmd(cmd)['stdout']
 
     def gen_scenario_json(self, task_id):
-        cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
+        cmd = "source {}; ".format(get_workload_venv('rally', True))
         cmd += "rally task results {}".format(task_id)
         return self.tools.run_cmd(cmd)['stdout']
 
     def gen_scenario_json_file(self, task_id, test_name):
-        cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
+        cmd = "source {}; ".format(get_workload_venv('rally', True))
         cmd += "rally task results {} > {}.json".format(task_id, test_name)
         return self.tools.run_cmd(cmd)['stdout']