Permalink
Browse files

Auto venv via path module

* Add path module to handle paths
  * Finds the venv
  * Finds the overcloudrc
* Clean-up/refactor imports

Change-Id: I24c269f2073a28ae0bee230cb9911dc89249f79f
  • Loading branch information...
akrzos committed Sep 29, 2017
1 parent d00d2c4 commit aa1b7651d38ea05d758d4784fd30fc655b05c6b7
Showing with 148 additions and 141 deletions.
  1. +1 −1 .gitignore
  2. +4 −7 ansible/install/group_vars/all.yml
  3. +1 −1 ansible/oooq/roles/browbeat-run/tasks/main.yml
  4. +0 −1 ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2
  5. +1 −2 ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2
  6. +0 −1 ansible/oooq/roles/template-configs/templates/browbeat-yoda-ci.yaml.j2
  7. +0 −5 browbeat-complete.yaml
  8. +0 −4 browbeat-config.yaml
  9. +4 −7 browbeat.py
  10. +8 −7 browbeat/elastic.py
  11. +69 −0 browbeat/path.py
  12. +13 −9 browbeat/perfkit.py
  13. +11 −9 browbeat/rally.py
  14. +12 −9 browbeat/shaker.py
  15. +7 −5 browbeat/tools.py
  16. +0 −12 browbeat/validate.yaml
  17. +10 −7 browbeat/yoda.py
  18. +0 −2 conf/browbeat-glance-complete.yaml
  19. +0 −2 conf/browbeat-keystone-complete.yaml
  20. +0 −2 conf/browbeat-keystone-minimal.yaml
  21. +0 −1 conf/browbeat-workloads.yml
  22. +0 −2 conf/ceilometer-minimal.yaml
  23. +0 −2 conf/ceilometer-persist-response-timing.yaml
  24. +0 −2 conf/gnocchi-minimal.yaml
  25. +0 −2 conf/perfkit-benchmarks.yaml
  26. +0 −2 conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml
  27. +0 −2 conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml
  28. +0 −2 conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml
  29. +0 −2 conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml
  30. +0 −2 conf/telemetry-nova-persist-instances-with-network.yaml
  31. +0 −2 conf/telemetry-nova-persist-instances.yaml
  32. +0 −2 doc/source/developing.rst
  33. +1 −5 doc/source/installation.rst
  34. +5 −19 doc/source/usage.rst
  35. +1 −1 tox.ini
View
@@ -1,11 +1,11 @@
*.py[cod]
.tox
# Virtual Environments:
.browbeat-venv
.perfkit-venv
.rally-venv
.shaker-venv
.yoda-venv
# OpenStack Environment Files
stackrc
@@ -25,25 +25,22 @@ overcloudrc: "{{home_dir}}/overcloudrc"
# overcloud_ca_path: /etc/pki/ca-trust/source/anchors/overcloud.crt.pem
# The default Browbeat venv
browbeat_venv: "{{home_dir}}/browbeat-venv"
browbeat_venv: "{{browbeat_path}}/.browbeat-venv"
# The default Rally venv
rally_venv: "{{home_dir}}/rally-venv"
rally_venv: "{{browbeat_path}}/.rally-venv"
# Rally version to install
rally_version: 0.8.1
# The default Shaker venv
shaker_venv: "{{home_dir}}/shaker-venv"
shaker_venv: "{{browbeat_path}}/.shaker-venv"
# Shaker version to Install
shaker_version: 1.1.0
# The default YODA venv
yoda_venv: /home/stack/yoda-venv
# PerfKitBenchmarker Settings
perfkit_venv: "{{home_dir}}/perfkit-venv"
perfkit_venv: "{{browbeat_path}}/.perfkit-venv"
perfkit_version: v1.12.0
# Configuration items to adjust browbeat results served through httpd
@@ -2,6 +2,6 @@
- name: Run Browbeat
shell:
"source {{ ansible_env.HOME }}/browbeat-venv/bin/activate; \
"source {{ ansible_env.HOME }}/browbeat/.browbeat-venv/bin/activate; \
cd {{ ansible_env.HOME }}/browbeat/; \
python browbeat.py all"
@@ -2,7 +2,6 @@ browbeat:
results : results/
rerun: 3
cloud_name: {{ browbeat_cloud_name }}
overcloud_credentials: /home/stack/overcloudrc
elasticsearch:
enabled: {{ elastic_enabled }}
host: {{ elastic_host }}
@@ -1,10 +1,9 @@
# Tests to be compleated for the install-and-check.sh script minimal and short workloads are performed
# Tests to be completed for the install-and-check.sh script minimal and short workloads are performed
# to confirm functionality.
browbeat:
results : results/
rerun: 1
cloud_name: {{ browbeat_cloud_name }}
overcloud_credentials: /home/stack/overcloudrc
elasticsearch:
enabled: {{ elastic_enabled }}
host: {{ elastic_host }}
@@ -4,7 +4,6 @@ browbeat:
results : results/
rerun: 1
cloud_name: {{ browbeat_cloud_name }}
overcloud_credentials: /home/stack/overcloudrc
elasticsearch:
enabled: {{ elastic_enabled }}
host: {{ elastic_host }}
View
@@ -3,7 +3,6 @@ browbeat:
results : results/
rerun: 3
cloud_name: openstack
overcloud_credentials: /home/stack/overcloudrc
elasticsearch:
enabled: false
regather: false
@@ -40,7 +39,6 @@ perfkit:
enabled: true
sleep_before: 0
sleep_after: 0
venv: /home/stack/perfkit-venv
default:
image: centos7
machine_type: m1.small
@@ -59,7 +57,6 @@ rally:
enabled: true
sleep_before: 5
sleep_after: 5
venv: /home/stack/rally-venv
plugins:
- netcreate-boot: rally/rally-plugins/netcreate-boot
- netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping
@@ -275,7 +272,6 @@ shaker:
join_timeout: 600
sleep_before: 5
sleep_after: 5
venv: /home/stack/shaker-venv
shaker_region: regionOne
external_host: 2.2.2.2
scenarios:
@@ -371,7 +367,6 @@ yoda:
enabled: false
instackenv: "/home/stack/instackenv.json"
stackrc: "/home/stack/stackrc"
venv: "/home/stack/yoda-venv/bin/activate"
benchmarks:
- name: scale-deploy
type: overcloud
View
@@ -3,7 +3,6 @@ browbeat:
results : results/
rerun: 1
cloud_name: openstack
overcloud_credentials: /home/stack/overcloudrc
elasticsearch:
enabled: true
host: 1.1.1.1
@@ -40,7 +39,6 @@ perfkit:
enabled: false
sleep_before: 0
sleep_after: 0
venv: /home/stack/perfkit-venv
default:
image: centos7
machine_type: m1.small
@@ -63,7 +61,6 @@ shaker:
join_timeout: 600
sleep_before: 5
sleep_after: 5
venv: /home/stack/shaker-venv
shaker_region: regionOne
external_host: 2.2.2.2
scenarios:
@@ -92,7 +89,6 @@ rally:
enabled: true
sleep_before: 5
sleep_after: 5
venv: /home/stack/rally-venv
plugins:
- netcreate-boot: rally/rally-plugins/netcreate-boot
- netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping
View
@@ -13,18 +13,15 @@
import argparse
import datetime
import browbeat.elastic
import browbeat.perfkit
import browbeat.rally
import browbeat.shaker
import browbeat.yoda
import browbeat.workloadbase
import browbeat.tools
import logging
import os
import sys
import time
import browbeat.elastic
import browbeat.tools
import browbeat.workloadbase
_workload_opts = ['perfkit', 'rally', 'shaker', 'yoda']
_config_file = 'browbeat-config.yaml'
debug_log_file = 'log/debug.log'
View
@@ -11,16 +11,17 @@
# limitations under the License.
from collections import deque
import elasticsearch
from elasticsearch import helpers
import logging
import json
import datetime
import uuid
import sys
import time
import json
import logging
import os
import re
import sys
import time
import uuid
import elasticsearch
from elasticsearch import helpers
browbeat_uuid = uuid.uuid4()
View
@@ -0,0 +1,69 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
_logger = logging.getLogger('browbeat.path')
# Browbeat's project modules path, typically /home/stack/browbeat/browbeat
browbeat_project_path = os.path.dirname(os.path.realpath(__file__))
# Path to Browbeat, typically /home/stack/browbeat
browbeat_path = os.path.abspath(os.path.join(browbeat_project_path, os.pardir))
# Base Path, typically /home/stack
base_path = os.path.abspath(os.path.join(browbeat_path, os.pardir))
conf_path = os.path.join(browbeat_path, 'conf')
log_path = os.path.join(browbeat_path, 'log')
results_path = os.path.join(browbeat_path, 'results')
def get_overcloudrc():
"""Check the several expected locations for the overcloudrc file:
* $base_path/overcloudrc
* $browbeat_path/overcloudrc
"""
paths = [
os.path.join(base_path, 'overcloudrc'),
os.path.join(browbeat_path, 'overcloudrc')
]
for overcloudrc_file in paths:
if os.path.exists(overcloudrc_file):
return overcloudrc_file
else:
_logger.debug("overcloudrc not found in {}".format(overcloudrc_file))
_logger.error('overcloudrc file can not be found')
def get_workload_venv(workload, path_activate):
"""Check the several expected locations for a workload's venv and return the existing venv:
* $base_path/$workload-venv
* $base_path/.$workload-venv
* $browbeat_path/.$workload-venv
"""
paths = [
os.path.join(base_path, '{}-venv'.format(workload)),
os.path.join(base_path, '.{}-venv'.format(workload)),
os.path.join(browbeat_path, '.{}-venv'.format(workload))
]
for workload_venv_path in paths:
if os.path.isdir(workload_venv_path):
if path_activate and os.path.exists(os.path.join(workload_venv_path, 'bin/activate')):
return os.path.join(workload_venv_path, 'bin/activate')
else:
return workload_venv_path
else:
_logger.debug("{} not installed in {}".format(workload, workload_venv_path))
_logger.error('{} does not appear to be installed correctly'.format(workload))
View
@@ -11,24 +11,27 @@
# limitations under the License.
import ast
import connmon
import datetime
import elastic
import glob
import grafana
import logging
import os
import shutil
import subprocess
import time
import connmon
import elastic
import grafana
from path import get_overcloudrc
from path import get_workload_venv
import tools
import workloadbase
class PerfKit(workloadbase.WorkloadBase):
def __init__(self, config):
self.logger = logging.getLogger('browbeat.perfkit')
self.overcloudrc = get_overcloudrc()
self.config = config
self.error_count = 0
self.tools = tools.Tools(self.config)
@@ -125,11 +128,12 @@ def run_benchmark(self, benchmark_config, result_dir, test_name, cloud_type="Ope
# Build command to run
if 'enabled' in benchmark_config:
del benchmark_config['enabled']
cmd = ("source {0}/bin/activate; source {1} "
"/home/stack/perfkit-venv/PerfKitBenchmarker/pkb.py "
"--cloud={2} --run_uri=browbeat".format(
self.config['perfkit']['venv'],
self.config['browbeat']['overcloud_credentials'], cloud_type))
cmd = ("source {0}; source {1}; "
"{2}/PerfKitBenchmarker/pkb.py "
"--cloud={3} --run_uri=browbeat".format(
get_workload_venv('perfkit', True),
self.overcloudrc,
get_workload_venv('perfkit', False), cloud_type))
for parameter, value in benchmark_config.iteritems():
if not parameter == 'name':
self.logger.debug(
View
@@ -11,19 +11,21 @@
# limitations under the License.
import collections
import connmon
import datetime
import elastic
import glob
import grafana
import json
import logging
import os
import re
import shutil
import time
import tools
import connmon
import elastic
import grafana
from path import get_workload_venv
import workloadbase
import json
import tools
class Rally(workloadbase.WorkloadBase):
@@ -61,7 +63,7 @@ def run_scenario(self, task_file, scenario_args, result_dir, test_name, benchmar
plugin_string = ""
if len(plugins) > 0:
plugin_string = "--plugin-paths {}".format(",".join(plugins))
cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
cmd = "source {}; ".format(get_workload_venv('rally', True))
cmd += "rally {} task start {} --task-args \'{}\' 2>&1 | tee {}.log".format(
plugin_string, task_file, task_args, test_name)
from_time = time.time()
@@ -104,18 +106,18 @@ def _get_details(self):
def gen_scenario_html(self, task_ids, test_name):
all_task_ids = ' '.join(task_ids)
cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
cmd = "source {}; ".format(get_workload_venv('rally', True))
cmd += "rally task report --task {} --out {}.html".format(
all_task_ids, test_name)
return self.tools.run_cmd(cmd)['stdout']
def gen_scenario_json(self, task_id):
cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
cmd = "source {}; ".format(get_workload_venv('rally', True))
cmd += "rally task results {}".format(task_id)
return self.tools.run_cmd(cmd)['stdout']
def gen_scenario_json_file(self, task_id, test_name):
cmd = "source {}/bin/activate; ".format(self.config['rally']['venv'])
cmd = "source {}; ".format(get_workload_venv('rally', True))
cmd += "rally task results {} > {}.json".format(task_id, test_name)
return self.tools.run_cmd(cmd)['stdout']
Oops, something went wrong.

0 comments on commit aa1b765

Please sign in to comment.