Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Support Kytoon and add new XenServer runner.

Add support for latest openstack_vpc changes (Kytoon) and
a new Puppet XenServer runner.
  • Loading branch information...
commit 8908f2dd1caab71fd5b2f702e877076c5218fdcb 1 parent 3810b31
@dprince authored
View
2  .rvmrc
@@ -1 +1 @@
-rvm 1.8.7@smokestack
+rvm 1.9.3@smokestack
View
3  Gemfile
@@ -8,8 +8,7 @@ gem 'rails', '3.1.1'
gem 'popen4'
gem 'mysql'
gem 'resque', '1.19.0'
-gem 'SystemTimer', '~> 1.2.3'
-gem 'chef-vpc-toolkit'
+gem 'kytoon'
# Gems used only for assets and not required
# in production environments by default.
View
14 Gemfile.lock
@@ -2,7 +2,6 @@ GEM
remote: http://rubygems.org/
specs:
Platform (0.4.0)
- SystemTimer (1.2.3)
actionmailer (3.1.1)
actionpack (= 3.1.1)
mail (~> 2.3.0)
@@ -33,11 +32,6 @@ GEM
multi_json (~> 1.0)
arel (2.2.1)
builder (3.0.0)
- chef-vpc-toolkit (2.8.1)
- builder
- json
- rake
- uuidtools
coffee-rails (3.1.1)
coffee-script (>= 2.2.0)
railties (~> 3.1.0)
@@ -51,6 +45,11 @@ GEM
hike (1.2.1)
i18n (0.6.0)
json (1.6.1)
+ kytoon (1.0.1)
+ builder
+ json
+ rake
+ uuidtools
libv8 (3.3.10.4)
mail (2.3.0)
i18n (>= 0.4.0)
@@ -134,9 +133,8 @@ PLATFORMS
ruby
DEPENDENCIES
- SystemTimer (~> 1.2.3)
- chef-vpc-toolkit
coffee-rails (~> 3.1.1)
+ kytoon
mysql
popen4
rails (= 3.1.1)
View
4 README.md
@@ -9,7 +9,7 @@ A web application with a REST based HTTP interface to help smoke test OpenStack.
* Rails 3.1
* Resque: a redis backed job queue
* Job runner templates: The default VPC job runner uses openstack_vpc to spin up groups of servers in the cloud for smoke testing. A job runner for unit tests is included as well. (add more job runners for: PXE, Crowbar, etc.)
-* Configuration management to install and configure everything (Chef/Puppet)
+* Configuration management to install and configure everything (Puppet)
* Packages to install software (currently supports Fedora/Ubuntu)
For more information and examples see the wiki: http://wiki.openstack.org/smokestack
@@ -60,7 +60,7 @@ At this point the web application should be running at http://localhost:3000.
# setup vpc
ssh-keygen # don't use passphrase
- # FIXME - add details on creating .chef_vpc_toolkit.conf for running VPC jobs
+ # FIXME - add details on creating .kytoon.conf for running VPC jobs
# in tab 1: launch some workers
bundle exec rake resque:workers QUEUE="vpc" COUNT=3
View
7 app/models/job.rb
@@ -5,10 +5,9 @@
class Job < ActiveRecord::Base
- CHEF_VPC = "Chef Vpc"
- CHEF_VPC_XEN = "Chef Vpc Xen"
PUPPET_VPC = "Puppet Vpc"
- JOB_TYPES = [CHEF_VPC, CHEF_VPC_XEN, PUPPET_VPC]
+ PUPPET_XEN = "Puppet Xen"
+ JOB_TYPES = [PUPPET_VPC, PUPPET_XEN]
validates_presence_of :job_group_id
belongs_to :job_group
@@ -27,7 +26,7 @@ def handle_after_save
self.job_group.update_status
end
- def self.run_job(job, template_name="chef_vpc_runner.sh.erb", script_text=nil)
+ def self.run_job(job, template_name="puppet_vpc_runner.sh.erb", script_text=nil)
job.update_attributes(:status => "Running", :start_time => Time.now)
View
22 app/models/job_chef_vpc.rb
@@ -1,22 +0,0 @@
-class JobChefVpc < Job
-
- @queue=:vpc
-
- def self.perform(id)
- 5.times do
- begin
- job=JobChefVpc.find(id)
- JobChefVpc.run_job(job)
- break
- rescue ActiveRecord::RecordNotFound
- sleep 5
- end
- end
- end
-
- after_create :handle_after_create
- def handle_after_create
- AsyncExec.run_job(JobChefVpc, self.id)
- end
-
-end
View
22 app/models/job_chef_vpc_xen.rb
@@ -1,22 +0,0 @@
-class JobChefVpcXen < Job
-
- @queue=:xen
-
- def self.perform(id)
- 5.times do
- begin
- job=JobChefVpcXen.find(id)
- self.run_job(job)
- break
- rescue ActiveRecord::RecordNotFound
- sleep 5
- end
- end
- end
-
- after_create :handle_after_create
- def handle_after_create
- AsyncExec.run_job(JobChefVpcXen, self.id)
- end
-
-end
View
260 app/templates/chef_vpc_runner.sh.erb
@@ -1,260 +0,0 @@
-#Global Chef settings
-export CHEF_TIMEOUT=900
-export CHEF_RESTART_ON_FAILURE="true"
-TMP_DIR=$(mktemp -d)
-
-<% job.job_group.smoke_test.test_suites.each do |suite| %>
-RUN_<%= suite.name %>=true
-<% end %>
-
-function copy_hosts {
-# Copy hosts file to each node
-rake ssh bash <<-"EOF_COPY_HOSTS"
-for IP in $(cat /etc/hosts | cut -f 1); do
-[[ "$IP" != "127.0.0.1" ]] && scp /etc/hosts $IP:/etc/hosts
-done
-EOF_COPY_HOSTS
-}
-
-function delete_group {
- XEN_DISCONNECT=$1
-
- #for debugging purposes you can touch this file to hang the group
- rake ssh bash <<-EOF_BASH
- until [ ! -f /tmp/do_not_delete ]; do
- sleep 1
- done
- EOF_BASH
-
- [ -n "$XEN_DISCONNECT" ] && rake xen:disconnect
- rake group:delete
-}
-
-function setup {
-
- chef-vpc-toolkit -v || \
- fail "Please install the Chef VPC Toolkit."
-
- cd $TMP_DIR
- if [ -n "$NOVA_URL" ]; then
- get_nova_source_git
- fi
-
- if [ -n "$KEYSTONE_URL" ]; then
- get_keystone_source_git
- fi
-
- if [ -n "$GLANCE_URL" ]; then
- get_glance_source_git
- fi
-
- GIT_ASKPASS=echo \
- git clone <%= ENV['OPENSTACK_VPC_URL'] %> openstack_vpc && \
- cd openstack_vpc || \
- fail "Failed to checkout openstack VPC."
-
- cat > config/chef_installer.yml <<-EOF_CAT
-chef_cookbook_repos: $COOKBOOK_URL
-
-# The Chef JSON configuration contains attributes and recipes for nodes.
-chef_json_file: config/nodes.json
-
-#databags_json_file: config/databags.json
-
-# Chef server name
-chef_server_name: login
-EOF_CAT
-
- cp "$SERVER_GROUP_JSON_CONF" config/server_group.json || \
- fail "Failed to copy server_group.json"
-
- rake chef:validate_json || fail "Invalid JSON config file(s).";
-
-}
-
-function run_job {
-
-trap "{ delete_group; cd /tmp; rm -Rf $TMP_DIR; }" INT TERM EXIT
-if rake group:create && rake group:poll; then
-
- rake nova:build_packages SOURCE_DIR="$TMP_DIR/nova_source" \
- DEB_PACKAGER_URL="$NOVA_DEB_PACKAGER_URL" || \
- #RPM_PACKAGER_URL="$NOVA_RPM_PACKAGER_URL" || \
- fail "Failed to build nova packages."
-
- rake glance:build_packages SOURCE_DIR="$TMP_DIR/glance_source" \
- DEB_PACKAGER_URL="$GLANCE_DEB_PACKAGER_URL" || \
- #RPM_PACKAGER_URL="$GLANCE_RPM_PACKAGER_URL" || \
- fail "Failed to build glance packages."
-
- #rake keystone:build_packages SOURCE_DIR="$TMP_DIR/keystone_source" \
- #DEB_PACKAGER_URL="$KEYSTONE_DEB_PACKAGER_URL" || \
- #RPM_PACKAGER_URL="$KEYSTONE_RPM_PACKAGER_URL" || \
- #fail "Failed to build keystone packages."
-
- # install Chef and the Chef Cookbooks
- copy_hosts
- rake chef:push_repos && rake chef:install || \
- { rake tail_logs; fail "Failed to install Chef server."; }
-
- rake chef_test || fail "Chef Client/Server setup is invalid!"
-
- # poll for Chef clients to finish runnings
- rake chef:poll_clients || \
- { rake tail_logs; fail "Chef client timeout."; }
-
- # run Torpedo API tests (Uses the Ruby openstack-compute gem)
- if [ -n "$RUN_TORPEDO" ]; then
- rake torpedo SERVER_NAME=nova1 || \
- { rake tail_logs; fail "Hit by Torpedo."; }
- fi
-
- # run OS API v1.0 tests (Uses the Ruby openstack-compute gem)
- if [ -n "$RUN_RUBY_OSAPI_V10" ]; then
- rake nova:ruby_osapi_tests SERVER_NAME=nova1 || \
- { rake tail_logs; fail "OS API tests failed."; }
- fi
-
- # run OS API v1.1 tests (Uses the Ruby openstack-compute gem)
- if [ -n "$RUN_RUBY_OSAPI_V11" ]; then
- rake nova:ruby_osapi_v11_tests SERVER_NAME=nova1 || \
- { rake tail_logs; fail "OS API tests failed."; }
- fi
-
- # run smoke tests
- if [ -n "$RUN_NOVA_SMOKE_TESTS" ]; then
- rake nova:smoke_tests SERVER_NAME=nova1 || \
- { rake tail_logs; fail "Smoke tests failed."; }
- fi
-
- # run stacktester
- if [ -n "$RUN_STACK_TESTER" ]; then
- rake nova:stacktester SERVER_NAME=nova1 || \
- { rake tail_logs; fail "Stacktester failed."; }
- fi
-
-else
- rake tail_logs
- fail "Failed to create server group."
-fi
-
-}
-
-function run_xen_hybrid_job {
-
-[ -n "$SERVER_IP_LIST" ] || fail "Xen jobs require a SERVER_IP_LIST env variable."
-
-trap "{ delete_group 'xen_discon'; cd /tmp; rm -Rf $TMP_DIR; }" INT TERM EXIT
-if rake group:create && rake group:poll; then
-
- rake nova:build_packages SOURCE_DIR="$TMP_DIR/nova_source" \
- DEB_PACKAGER_URL="$NOVA_DEB_PACKAGER_URL" || \
- #RPM_PACKAGER_URL="$NOVA_RPM_PACKAGER_URL" || \
- fail "Failed to build nova packages."
-
- rake nova:build_rpms SOURCE_DIR="$TMP_DIR/nova_source" || \
- fail "Failed to build Xen plugin RPMs."
-
- rake glance:build_packages SOURCE_DIR="$TMP_DIR/glance_source" \
- DEB_PACKAGER_URL="$GLANCE_DEB_PACKAGER_URL" || \
- #RPM_PACKAGER_URL="$GLANCE_RPM_PACKAGER_URL" || \
- fail "Failed to build glance packages."
-
- #rake keystone:build_packages SOURCE_DIR="$TMP_DIR/keystone_source" \
- #DEB_PACKAGER_URL="$KEYSTONE_DEB_PACKAGER_URL" || \
- #RPM_PACKAGER_URL="$KEYSTONE_RPM_PACKAGER_URL" || \
- #fail "Failed to build keystone packages."
-
- # install Chef and the Chef Cookbooks
- copy_hosts
- rake chef:push_repos && rake chef:install || \
- { rake tail_logs; fail "Failed to install Chef server."; }
-
- rake chef_test || fail "Chef Client/Server setup is invalid!"
-
- # Bootstrap all the Xen Servers
- local COUNT=0
- for SERVER_IP in $SERVER_IP_LIST; do
- COUNT=$(($COUNT + 1))
- local SERVER_NAME="xen$COUNT"
- rake xen:bootstrap XENSERVER_IP=$SERVER_IP SERVER_NAME=$SERVER_NAME || \
- fail "Failed to connect Xen Server: $SERVER_NAME to VPN group."
- sleep 10
- copy_hosts
- rake chef:install SERVER_NAME=$SERVER_NAME || \
- fail "Failed to install Chef on $SERVER_NAME."
- done
-
- #FIXME: this can come out when compute1.vpc gets a proper domain name
- # Ideally we could just use rake:poll_clients (to poll them all)
- rake chef:poll_clients SERVER_NAME=nova1 || \
- { rake tail_logs SERVER_NAME=nova1; fail "Chef client timeout: nova1"; }
- rake chef:poll_clients SERVER_NAME=glance1 || \
- { rake tail_logs SERVER_NAME=glance1; fail "Chef client timeout: glance1"; }
-
- # Poll Xen Servers to finish Chef'ing
- COUNT=0
- for SERVER_IP in $SERVER_IP_LIST; do
- COUNT=$(($COUNT + 1))
- local SERVER_NAME="xen$COUNT"
- local COMPUTE_NAME="compute$COUNT"
- rake chef:poll_clients SERVER_NAME=$SERVER_NAME || \
- { rake tail_logs SERVER_NAME=$SERVER_NAME; fail "Chef client timeout: $SERVER_NAME"; }
- rake ssh bash <<-EOF_BASH
- if ! grep -c "$COMPUTE_NAME.vpc" /etc/hosts &> /dev/null; then
- echo "172.19.0.10$COUNT $COMPUTE_NAME.vpc $COMPUTE_NAME" >> /etc/hosts
- fi
- EOF_BASH
- copy_hosts
- rake chef:install SERVER_NAME=$COMPUTE_NAME || \
- fail "Failed to install Chef on $COMPUTE_NAME."
- done
-
- # Poll Compute nodes (dom0) to finish Chef'ing
- COUNT=0
- for SERVER_IP in $SERVER_IP_LIST; do
- COUNT=$(($COUNT + 1))
- # NOTE: we put .vpc on these because the agent doesn't seem to
- # set the domain name correctly
- local COMPUTE_NAME="compute$COUNT.vpc"
- rake chef:poll_clients SERVER_NAME=$COMPUTE_NAME || \
- { rake tail_logs SERVER_NAME=$COMPUTE_NAME; fail "Chef client timeout: $COMPUTE_NAME"; }
- done
-
- # run Torpedo API tests (Uses the Ruby openstack-compute gem)
- if [ -n "$RUN_TORPEDO" ]; then
- rake torpedo SERVER_NAME=nova1 MODE=xen || \
- { rake tail_logs; fail "Hit by Torpedo."; }
- fi
-
- # run OS API v1.0 tests (Uses the Ruby openstack-compute gem)
- if [ -n "$RUN_RUBY_OSAPI_V10" ]; then
- rake nova:ruby_osapi_tests SERVER_NAME=nova1 MODE=xen || \
- { rake tail_logs; fail "OS API tests failed."; }
- fi
-
- # run OS API v1.1 tests (Uses the Ruby openstack-compute gem)
- if [ -n "$RUN_RUBY_OSAPI_V11" ]; then
- rake nova:ruby_osapi_v11_tests SERVER_NAME=nova1 MODE=xen || \
- { rake tail_logs; fail "OS API tests failed."; }
- fi
-
- # run stacktester
- if [ -n "$RUN_STACK_TESTER" ]; then
- rake nova:stacktester SERVER_NAME=nova1 || \
- { rake tail_logs; fail "Stacktester failed."; }
- fi
-
-else
- rake tail_logs
- fail "Failed to create server group."
-fi
-
-}
-
-setup
-<% if job.type == 'JobChefVpcXen' %>
-run_xen_hybrid_job
-<% else %>
-run_job
-<% end %>
View
10 app/templates/puppet_vpc_runner.sh.erb
@@ -1,6 +1,8 @@
-#Global Chef settings
TMP_DIR=$(mktemp -d)
+# Set default Kytoon group type to 'cloud_server_vpc'
+export GROUP_TYPE=cloud_server_vpc
+
<% job.job_group.smoke_test.test_suites.each do |suite| %>
RUN_<%= suite.name %>=true
<% end %>
@@ -43,7 +45,7 @@ function run_job {
trap "{ delete_group; cd /tmp; rm -Rf $TMP_DIR; }" INT TERM EXIT
start_metric "Create Server Group"
-if rake group:create && rake group:poll; then
+if rake group:create; then
stop_metric
local NOVA_MERGE_MASTER=""
@@ -159,7 +161,9 @@ stop_metric
cat config.yml
mv config.yml config/puppet-configs/smokestack/config.yml
cp -a config/puppet-configs/smokestack /tmp/smokestack-config
-
+
+ rake fedora:create_rpm_repo
+
start_metric "Puppet Install"
rake puppet:install SOURCE_URL="$COOKBOOK_URL" PUPPET_CONFIG="smokestack" --trace
stop_metric
View
2  app/views/about/index.html.erb
@@ -40,7 +40,7 @@ A web application with a REST based HTTP interface to help smoke test the OpenSt
<li>Resque: a redis backed job queue</li>
<li>A templated job runner. The default VPC job runner uses <a href="https://github.com/dprince/openstack_vpc">openstack_vpc</a> to spin up groups of servers in the cloud for smoke testing.</li>
<li>Packages to install software (currently supports Fedora/Ubuntu)</li>
-<li>Configuration management to install and configure everything (Chef/Puppet)</li>
+<li>Configuration management to install and configure everything (Puppet)</li>
</ul>
For more information and examples see the <a href="http://wiki.openstack.org/smokestack"/>wiki</a>.
View
2  app/views/config_templates/show.html.erb
@@ -9,7 +9,7 @@
</p>
<p>
- <b>Chef Cookbook Repo URL:</b>
+ <b>Cookbook Repo URL:</b>
<%= @config_template.cookbook_repo_url %>
</p>
View
2  config/application.rb
@@ -1,6 +1,6 @@
require File.expand_path('../boot', __FILE__)
SMOKESTACK_VERSION="dev"
-CHEF_VPC_PROJECT = "#{File.dirname(__FILE__)}" unless defined?(CHEF_VPC_PROJECT)
+KYTOON_PROJECT = "#{File.dirname(__FILE__)}" unless defined?(KYTOON_PROJECT)
require 'rails/all'
View
3  config/environments/development.rb
@@ -33,7 +33,8 @@
ENV['NOVACLIENT_GIT_MASTER'] = "git://github.com/openstack/python-novaclient.git"
ENV['GLANCE_GIT_MASTER'] = "git://github.com/openstack/glance.git"
ENV['KEYSTONE_GIT_MASTER'] = "git://github.com/openstack/keystone.git"
-ENV['KEYSTONECLIENT_GIT_MASTER'] = "git://github.com/openstack/python-keystoneclient.git"
+#ENV['KEYSTONECLIENT_GIT_MASTER'] = "git://github.com/openstack/python-keystoneclient.git"
+ENV['KEYSTONECLIENT_GIT_MASTER'] = "git://github.com/dprince/python-keystoneclient.git"
ENV['SWIFT_GIT_MASTER'] = "git://github.com/openstack/swift.git"
ENV['SWIFTCLIENT_GIT_MASTER'] = "git://github.com/openstack/python-swiftclient.git"
View
1  db/schema.rb
@@ -1,3 +1,4 @@
+# encoding: UTF-8
# This file is auto-generated from the current state of the database. Instead
# of editing this file, please use the migrations feature of Active Record to
# incrementally modify your database, and then regenerate this schema definition.
View
2  test/fixtures/config_templates.yml
@@ -2,7 +2,7 @@
libvirt_psql:
name: Libvirt Postgres
- job_type: Chef Vpc
+ job_type: Puppet Vpc
description: Libvirt QEME w/ Postgres
cookbook_repo_url: "http://a.b.c/blah"
server_group_json: {}
View
1  test/fixtures/jobs.yml
@@ -11,7 +11,6 @@ one:
two:
status: Success
stdout: |
- Chef VPC Toolkit Version: 2.5.1
Cloning into openstack_vpc..
NOVA_REVISION=102
FAILURE_MSG=This is only a test.
View
4 test/functional/smoke_tests_controller_test.rb
@@ -111,7 +111,7 @@ class SmokeTestsControllerTest < ActionController::TestCase
assert_difference('Job.count', 2) do
post :run_jobs, :id => @smoke_test.id
end
- assert_not_nil AsyncExec.jobs[JobChefVpc]
+ assert_not_nil AsyncExec.jobs[JobPuppetVpc]
end
test "should not run job" do
@@ -119,7 +119,7 @@ class SmokeTestsControllerTest < ActionController::TestCase
assert_no_difference('Job.count') do
post :run_jobs, :id => @smoke_test.id
end
- assert_nil AsyncExec.jobs[JobChefVpc]
+ assert_nil AsyncExec.jobs[JobPuppetVpc]
end
private
Please sign in to comment.
Something went wrong with that request. Please try again.