Merge branch 'master' of https://github.com/crowbar/crowbar

README.md

@@ -0,0 +1,49 @@
+# Welcome to the Crowbar project [![Build Status](https://travis-ci.org/crowbar/travis-ci-crowbar_framework.png?branch=master)](https://travis-ci.org/crowbar/travis-ci-crowbar_framework)
+
+The code and documentation are distributed under the [Apache 2
+license](http://www.apache.org/licenses/LICENSE-2.0.html). Contributions back
+to the source are encouraged.
+
+Crowbar was developed by the [Dell CloudEdge Solutions Team]
+(http://dell.com/openstack) as an [OpenStack](http://OpenStack.org) installer,
+but has evolved as a much broader function tool. It is a wrapper for Opscode
+[Chef](http://opscode.com) Server.
+
+Much of the design information about Crowbar has been published on Rob
+Hirschfeld's [blog](http://robhirschfeld.com). This wiki maintains information
+about the operation of Crowbar. Documentation for users and deployment will be
+committed into the Git repo. Intermediate updates are likely to surface in the
+wiki.
+
+Please consult the release notes for known issues.
+
+To build your own copy of Crowbar, consult the [readme]
+(https://github.com/crowbar/crowbar/blob/master/README.build). Some important
+notes about the build process:
+
+* Build process has been tested on Ubuntu 12.04 and CentOS.
+
+With that said, from a fresh install off the crowbar-dev.iso, the next steps
+are:
+
+1. Boot using the ISO and it will setup Ubuntu 12.04 and stage Crowbar for
+   install.
+1. Log in as `crowbar/crowbar` (Ubuntu) or `root/crowbar` (Redhat).
+1. `sudo su - root`.
+1. `cd /tftpboot/ubuntu_dvd/extra` (Ubuntu) or `cd /tftpboot/redhat_dvd/extra`
+   (Redhat).
+1. `./install admin.crowbar.org` (or whatever FQDN you want your admin node to
+   have).
+
+Note: If the install did not work, you can try `sudo chef-client` and that may
+fix issues by repeating the last part of the installation.
+
+The rest of Crowbar will then install. Unless you change the networking
+defaults, you can access the Crowbar server:
+
+* Crowbar UI on http://192.168.124.10:3000.  (crowbar/crowbar)
+* Chef UI on http://192.168.124.10:4040.  (admin/password)
+* Nagios on http://192.168.124.10/nagios3.  (nagiosadmin/password)
+* Ganglia on http://192.168.124.10/ganglia.  (nagiosadmin/password)
+
+We are working to make this wiki complete, please feel free to update content.

dev

@@ -3308,7 +3308,11 @@ reinstall_barclamps() {
         ./barclamp_install.rb --force --no-files --no-chef -b \
             "$CROWBAR_TEST_DIR" "${barclamp_list[@]}") || \
             die "Failed to install some barclamps."
-    sudo chown -R "$USER.$USER" "${CROWBAR_TEST_DIR}"
+
+    local group=$USER
+    [ -f /etc/SuSE-release ] && group=users
+    sudo chown -R "$USER.$group" "${CROWBAR_TEST_DIR}"
+
     find "$CROWBAR_TEST_DIR" -type d -exec chmod a+x '{}' ';'
     # Clean up the filelists.
     rm barclamps/*.txt
@@ -3325,14 +3329,28 @@ unit_tests_need_reset() {
 
 setup_unit_tests() (
     local gem f bc barclamps=()
-    local gem_offline=true update_gem_cache=false
-    unit_prereqs_installed || exit 1
+    local gem_offline=true update_gem_cache=false use_gem_cache=true
     for f in "$@"; do
         case $f in
             --update-gem-cache) update_gem_cache=true;;
+            --no-gem-cache)     use_gem_cache=false;;
             *) barclamps+=("$f");;
         esac
     done
+    if [[ $use_gem_cache = true ]]; then
+        unit_prereqs_installed || exit 1
+    else
+        local rvm=/usr/local/rvm/scripts/rvm
+        [ -f $rvm ] && source $rvm
+        if command -v bundle >/dev/null 2>&1; then
+            debug "** Installing minimal dev setup dependencies..."
+            bundle install --gemfile=dev-setup/Gemfile
+            [ $? -eq 0 ] || die "Failed to install gem dependencies"
+            debug
+        else
+            die "Please make sure bundler is installed. Eg. 'sudo gem install bundler'"
+        fi
+    fi
     rm -rf "$CROWBAR_TEST_DIR"
     mkdir "$CROWBAR_TEST_DIR"
     reinstall_barclamps "${barclamps[@]}" || exit 1
@@ -3352,20 +3370,30 @@ setup_unit_tests() (
         done
     fi
     unset update_gem_cache
-    maybe_checkout_build_cache_branch
-    [[ -d $CROWBAR_TEST_DIR/gemsite ]] && \
-        rm -rf "$CROWBAR_TEST_DIR/gemsite/gems"
-    mkdir -p "$CROWBAR_TEST_DIR/gemsite/gems"
-    find "$CACHE_DIR/barclamps" -type f -name '*.gem' \
-        -exec cp '{}' "$CROWBAR_TEST_DIR/gemsite/gems" ';'
-    (   cd "$CROWBAR_TEST_DIR/gemsite/"
-        gem generate_index)
-    sed -ie "s@^source .*@source \"file://$CROWBAR_TEST_DIR/gemsite\"@" "Gemfile"
-    bundle install --path "$CROWBAR_TEST_DIR/gems" || \
-        die "Not all required gems are present." \
-        "Please run again with --update-gem-cache." \
-        "If that does not work, then you will need to add the missing gems to:" \
-        "$CROWBAR_DIR/barclamps/crowbar/crowbar.yml"
+
+    if [[ $use_gem_cache = true ]]; then
+        maybe_checkout_build_cache_branch
+        [[ -d $CROWBAR_TEST_DIR/gemsite ]] && \
+            rm -rf "$CROWBAR_TEST_DIR/gemsite/gems"
+        mkdir -p "$CROWBAR_TEST_DIR/gemsite/gems"
+        find "$CACHE_DIR/barclamps" -type f -name '*.gem' \
+            -exec cp '{}' "$CROWBAR_TEST_DIR/gemsite/gems" ';'
+        (   cd "$CROWBAR_TEST_DIR/gemsite/"
+            gem generate_index)
+        sed -ie "s@^source .*@source \"file://$CROWBAR_TEST_DIR/gemsite\"@" "Gemfile"
+
+        bundle install --path "$CROWBAR_TEST_DIR/gems" || \
+            die "Not all required gems are present." \
+            "Please run again with --update-gem-cache." \
+            "If that does not work, then you will need to add the missing gems to:" \
+            "$CROWBAR_DIR/barclamps/crowbar/crowbar.yml"
+    else
+        sed -ie "s@^source .*@source \"http://rubygems.org\"@" "Gemfile"
+        echo "NOTE: Using the --no-gem-cache option means that the gems used to"\
+        "run the unit tests may not be the same as the ones used to build and"\
+        "install Crowbar, so results may differ."
+    fi
+
     cd "$CROWBAR_TEST_DIR/crowbar_framework/BDD"
     sudo "$CROWBAR_TEST_DIR/crowbar_framework/BDD/linux_install.sh"
     ./linux_compile.sh
@@ -3402,16 +3430,17 @@ run_unit_tests() (
     local barclamps=($(barclamps_needing_reinstall "$@"))
     local final_result=pass
     unit_tests_need_reset && clear_unit_tests
+    rm -f "$CROWBAR_TEST_DIR/crowbar_framework/Gemfile" # Use the parent directory gemfile
     reload_unit_tests "$@" || exit 1
     # Run unit tests
     cd "$CROWBAR_TEST_DIR/crowbar_framework"
     rm -rf coverage
     mkdir coverage
     bundle exec rcov --aggregate coverage/testing.data -Ispec:lib \
-        --exclude /gems/,/Library/,/usr/,lib/tasks,.bundle,config,/lib/rspec/,/lib/rspec-,spec \
+        --exclude /gems/,/Library/,/usr/,lib/tasks,.bundle,/config/,/lib/rspec/,/lib/rspec-,spec \
         spec/*/*.rb || final_result=fail
     bundle exec rcov --aggregate coverage/testing.data -Itest:lib \
-        --exclude /gems/,/Library/,/usr/,lib/tasks,.bundle,config,/lib/rspec/,/lib/rspec-,spec \
+        --exclude /gems/,/Library/,/usr/,lib/tasks,.bundle,/config/,/lib/rspec/,/lib/rspec-,spec \
         test/unit/*.rb || final_result=fail
 
     # Start rails

dev-setup/Gemfile

@@ -0,0 +1,19 @@
+# Copyright 2013, SUSE
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Minimal set of gems required to run `./dev setup-unit-tests`.
+
+source 'https://rubygems.org'
+
+gem 'kwalify'
+gem 'builder'

dev-setup/README-Ubuntu.md

@@ -0,0 +1,173 @@
+# Crowbar Dev environment based on Ubuntu
+
+## Setting up the virtual machine (VM)
+
+Currently only Ubuntu 12.04 LTS is supported, though the instructions here
+should also work with other versions.
+
+The steps here describe how to setup the VM from the command line. You can use
+[virt-manager](http://virt-manager.org) if you prefer a graphical user
+interface. Do submit your relevant virt-manager configs if you have some!
+
+The steps here assume that your KVM host is also the desktop that you are
+working from. If not, adapt the commands accordingly.
+
+Installation steps:
+
+1. Download Ubuntu 12.04 LTS 64 bit (`ubuntu-12.04.1-server-amd64.iso`) from
+   http://www.ubuntu.com/download/server. For example, run the following
+   commands within the Crowbar git checkout on the qemu-kvm host:
+
+   ````
+   cd dev-setup/qemu-kvm
+   aria2c http://releases.ubuntu.com/precise/ubuntu-12.04.1-server-amd64.iso.torrent
+   ````
+
+1. Create a blank disk image that is at least 20 GB, eg:
+
+   ````
+   qemu-img create -f qcow2 -o preallocation=metadata ubuntu-12.04.qcow2 20G
+   ````
+
+1. Start a VM with the desired network (private network with NAT), with the ISO
+   and disk attached. For example:
+
+   ````
+   sudo qemu-kvm -m 2G -daemonize -vnc :10 -cdrom ubuntu-12.04.1-server-amd64.iso \
+                 -net nic,model=virtio,macaddr=DE:AD:BE:EF:30:22 \
+                 -net tap,script=qemu-ifup \
+                 -drive file=ubuntu-12.04.qcow2,cache=none,if=virtio
+   ````
+
+   Note that `script=qemu-ifup` points to the script at `qemu-kvm/qemu-ifup`,
+   so make sure you are running the above command in the same directory, or
+   modify it accordingly.
+
+1. Connect to the VM via VNC and install the system:
+
+   ````
+   vncviewer :10
+   ````
+
+   The installer will attempt to auto-configure the network with DHCP, which
+   you can cancel and jump to manual configuration instead with the following
+   settings:
+
+   ````
+   IP address: 192.168.124.10
+   Netmask:    255.255.255.0
+   Gateway:    192.168.124.1
+   ````
+
+   Use the same name server (DNS) address as your host, which you can find out
+   on Linux systems by running `grep nameserver /etc/resolv.conf` on the host.
+   For example, within the SUSE intranet it is `10.120.2.88`. If the host is not
+   running in any internal or corporate network, you can use `8.8.8.8`.
+
+   The hostname and domain names can be left at the defaults. The apt-get proxy
+   can also be left blank.
+
+1. Once installation is complete, you can shutdown the VM (`sudo poweroff`) and
+   subsequently start it in the same way, minus the `-cdrom ...` option. Or
+   use the [qemu-kvm/start-vm](https://github.com/crowbar/crowbar/blob/master/dev-setup/qemu-kvm/start-vm))
+   helper script.
+
+## Setting up the development environment
+
+You should now have a working VM that you can SSH into from the qemu-kvm host.
+For example:
+
+    ssh 192.168.124.10
+
+The VM should also be able to access the external network. We can now start
+with setting up the Crowbar development environment.
+
+1. Install the basic Crowbar dev tool dependencies:
+
+   ````
+   sudo apt-get install git rubygems
+   sudo gem install json --no-ri --no-rdoc
+   ````
+
+1. Copy your .gitconfig and other configuration files to the VM, eg:
+
+   ````
+   scp -r <your-usual-dev-host>:.{gitconfig,vimrc,vim,profile,ssh} .
+   ````
+
+1. Check out the Crowbar git repo and run the dev tool:
+
+   ````
+   git clone git://github.com/crowbar/crowbar.git
+   cd crowbar
+   ./dev setup
+   ````
+
+   The `./dev setup` script will ask for your Github username and password. It
+   will fork the Crowbar and corresponding barclamp repositories to your
+   account and clone them into `crowbar/barclamps/`. See [dev-and-workflow]
+   (https://github.com/crowbar/crowbar/blob/master/README.dev-and-workflow)
+   and [dev-and-code-review]
+   (https://github.com/crowbar/crowbar/blob/master/README.dev-and-code-review)
+   for details. This will take a while so get some coffee.
+
+1. Install more dependencies for setting up and running the tests:
+
+   ````
+   sudo apt-get install libsqlite3-dev
+   sudo gem install markdown kwalify rake bundler rcov rspec --no-ri --no-rdoc
+   sudo gem install rails -v 3.2.10 --no-ri --no-rdoc
+   ````
+
+1. Running the dev tool to setup the Crowbar run-time environment at
+   `/tmp/crowbar-dev-test`:
+
+   ````
+   ./dev setup-unit-tests
+   ````
+
+1. Running the unit and Rspec tests:
+
+   ````
+   cd /tmp/crowbar-dev-test/crowbar_framework
+   bundle exec rake db:drop db:migrate db:fixtures:dump test:units spec
+   ````
+
+1. Running the BDD tests:
+
+   ````
+   cd /tmp/crowbar-dev-test/crowbar_framework/BDD
+   ./linux_compile.sh
+   ./linux_run.sh
+   ````
+
+   Note that the BDD tests require a running instance of Crowbar, which is
+   started by `linux_run.sh`. If it fails with an error message like:
+
+   ````
+   ERROR: step run found error:{badmatch,{error,econnrefused}}
+   ...
+   ````
+
+   This means that the Crowbar server is not running and can usually be fixed
+   by running `linux_run.sh` again. Refer to the [BDD dev guide]
+   (https://github.com/crowbar/barclamp-crowbar/blob/master/crowbar_framework/doc/default/crowbar/devguide/testing/bdd.md)
+   for more details.
+
+1. Starting the Crowbar web interface:
+
+   ````
+   cd /tmp/crowbar-dev-test/crowbar_framework
+   bundle exec rails s puma
+   ````
+
+   You will want to keep this terminal open to see the Rails logs, which will
+   come in very handy during development and debugging. The server can be
+   terminated with `Ctrl-c`.
+
+   The Crowbar web interface should now be accessible from your host web
+   browser, eg. at `http://192.168.124.10:3000`.
+
+Happy hacking! We will be updating this document regularly as we expand the
+supported distros and evolve the code base. Pull requests are very much
+appreciated!