remove submodules and use native ansible-y bits

.gitignore

@@ -54,6 +54,8 @@ docs/_build/
 target/
 
 # generated files
+rpcd/playbooks/ansible.cfg
+openstack-ansible
 rpcd/playbooks/roles/ceph-common
 rpcd/playbooks/roles/ceph-mon
 rpcd/playbooks/roles/ceph-osd

.travis.yml

@@ -3,6 +3,8 @@ language: python
 before_script:
 - pip install tox
 - pip2 install --force-reinstall 'ansible===1.9.4'
+- sed -i 's|/opt|/home/travis/build/rcbops|g' rpcd/playbooks/repo-fetcher.yml
+- ansible-playbook -i "localhost," -c local rpcd/playbooks/repo-fetcher.yml
 - ansible-galaxy install --role-file=ansible-role-requirements.yml --force --roles-path=/home/travis/build/rcbops/rpc-openstack/rpcd/playbooks/roles
 
 script: tox

requirements.txt

@@ -0,0 +1,7 @@
+Jinja2>=2.6             # ansible
+netaddr>=0.7.12         # playbooks/inventory/dynamic_inventory.py
+paramiko>=1.13.0        # ansible
+pip>=6.0
+PrettyTable>=0.7,<0.8   # scripts/inventory-manage.py
+pycrypto>=2.6           # ansible
+PyYAML>=3.1.0           # ansible

rpcd/playbooks/repo-fetcher.yml

@@ -0,0 +1,67 @@
+---
+- name: lay the repos down
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  user: root
+  tasks:
+    - name: lay the repos down
+      git:
+        dest: "{{ item.value.dest }}"
+        repo: "{{ item.value.repo }}"
+        version: "{{ item.value.version }}"
+      with_dict: "{{repos}}"
+    - name: generate rpcd ansible.cfg
+      copy:
+        dest: /opt/rpc-openstack/rpcd/playbooks/ansible.cfg
+        content: |
+          [defaults]
+          gathering = smart
+          host_key_checking = False
+
+          # Setting forks should be based on your system. The Ansible defaults to 5,
+          # the os-lxc-hosts assumes that you have a system that can support
+          # OpenStack, thus it has been conservatively been set to 15
+          forks = 15
+
+          # SSH timeout
+          timeout = 120
+
+          # Set the path to the folder in openstack-ansible which holds the dynamic
+          # inventory script - new config setting for ansible v1.9 and above
+          inventory = {{ repos.openstack_ansible.dest }}/playbooks/inventory/
+
+          # Set the path to the folder in openstack-ansible which holds the dynamic
+          # inventory script - uncomment if using ansible below v1.9
+          #hostfile = {{ repos.openstack_ansible.dest }}/playbooks/inventory/
+
+          # Set the path to the folder in openstack-ansible which holds the
+          # libraries required
+          library = {{ repos.openstack_ansible.dest }}/playbooks/library/
+
+          # Set the path to the folder in openstack-ansible which holds the roles
+          # that are depended on by the rpc-openstack roles
+          roles_path = {{ repos.openstack_ansible.dest }}/playbooks/roles/
+
+          # Set the path to the folder in openstack-ansible which holds the
+          # lookup plugins required
+          lookup_plugins = {{ repos.openstack_ansible.dest }}/playbooks/plugins/lookups/
+
+          # Set the path to the folder in openstack-ansible which holds the filter
+          # plugins required
+          filter_plugins = {{ repos.openstack_ansible.dest }}/playbooks/plugins/filters/
+
+          # Set the path to the folder in openstack-ansible which holds the action
+          # plugins required
+          action_plugins = {{ repos.openstack_ansible.dest }}/playbooks/plugins/actions/
+
+          [ssh_connection]
+          pipelining = True
+
+
+  vars:
+    repos:
+      openstack_ansible:
+        dest: /opt/rpc-openstack/openstack-ansible
+        repo: https://github.com/openstack/openstack-ansible.git
+        version: e0025962cf09822dce858b5235fce2fdf8a31b94

rpcd/playbooks/roles/beaver/tasks/beaver_install.yml

@@ -44,7 +44,7 @@
     dest: "/etc/init.d/beaver"
     owner: "root"
     group: "root"
-    mode: 755
+    mode: 0755
   notify: Restart beaver
   tags:
     - beaver-install
@@ -56,7 +56,7 @@
     dest: "/etc/logrotate.d/beaver"
     owner: "root"
     group: "root"
-    mode: 644
+    mode: 0644
   tags:
     - beaver-install
     - beaver-logrotate

rpcd/playbooks/roles/logstash/tasks/logstash_post_install.yml

@@ -19,7 +19,7 @@
     content: "manual\n"
     owner: "root"
     group: "root"
-    mode: 644
+    mode: 0644
   tags:
     - logstash-web-disable
     - logstash-post-install

scripts/bootstrap-ansible.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# (c) 2014, Kevin Carter <kevin.carter@rackspace.com>
+
+## Shell Opts ----------------------------------------------------------------
+set -e -u -x
+
+
+## Vars ----------------------------------------------------------------------
+export HTTP_PROXY=${HTTP_PROXY:-""}
+export HTTPS_PROXY=${HTTPS_PROXY:-""}
+export ANSIBLE_GIT_RELEASE=${ANSIBLE_GIT_RELEASE:-"v1.9.4-1"}
+export ANSIBLE_GIT_REPO=${ANSIBLE_GIT_REPO:-"https://github.com/ansible/ansible"}
+export ANSIBLE_ROLE_FILE=${ANSIBLE_ROLE_FILE:-"ansible-role-requirements.yml"}
+export ANSIBLE_WORKING_DIR=${ANSIBLE_WORKING_DIR:-/opt/ansible_${ANSIBLE_GIT_RELEASE}}
+export SSH_DIR=${SSH_DIR:-"/root/.ssh"}
+export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-"noninteractive"}
+
+
+## Functions -----------------------------------------------------------------
+info_block "Checking for required libraries." 2> /dev/null ||
+    source $(dirname ${0})/scripts-library.sh ||
+    source scripts/scripts-library.sh
+
+## Main ----------------------------------------------------------------------
+info_block "Bootstrapping System with Ansible"
+
+# Create the ssh dir if needed
+ssh_key_create
+
+# Install the base packages
+if [[ $HOST_DISTRO =~ ^(Ubuntu|Debian) ]]; then
+    apt-get update && apt-get -y install git python-all python-dev curl autoconf g++ python2.7-dev < /dev/null
+elif [[ $HOST_DISTRO =~ ^(CentOS|Red Hat) ]]; then
+    yum check-update && yum -y install git python2 curl autoconf gcc-c++ python2-devel
+elif [[ $HOST_DISTRO =~ ^Fedora ]]; then
+    dnf -y install git python curl autoconf gcc-c++ python-devel
+fi
+
+# If the working directory exists remove it
+if [ -d "${ANSIBLE_WORKING_DIR}" ];then
+    rm -rf "${ANSIBLE_WORKING_DIR}"
+fi
+
+# Clone down the base ansible source
+git clone "${ANSIBLE_GIT_REPO}" "${ANSIBLE_WORKING_DIR}"
+pushd "${ANSIBLE_WORKING_DIR}"
+    git checkout "${ANSIBLE_GIT_RELEASE}"
+    git submodule update --init --recursive
+popd
+
+
+# Install pip
+get_pip
+
+# Ensure we use the HTTPS/HTTP proxy with pip if it is specified
+PIP_OPTS=""
+if [ -n "$HTTPS_PROXY" ]; then
+  PIP_OPTS="--proxy $HTTPS_PROXY"
+elif [ -n "$HTTP_PROXY" ]; then
+  PIP_OPTS="--proxy $HTTP_PROXY"
+fi
+
+# Install requirements if there are any
+if [ -f "requirements.txt" ];then
+  pip2 install $PIP_OPTS -r requirements.txt || pip install $PIP_OPTS -r requirements.txt
+fi
+
+# Install ansible
+pip2 install $PIP_OPTS "${ANSIBLE_WORKING_DIR}" || pip install $PIP_OPTS "${ANSIBLE_WORKING_DIR}"
+
+# Update dependent roles
+if [ -f "${ANSIBLE_ROLE_FILE}" ];then
+    # Pull all required roles.
+    ansible-galaxy install --role-file=${ANSIBLE_ROLE_FILE} \
+                           --ignore-errors \
+                           --force
+fi
+
+# Create openstack ansible wrapper tool
+cat > /usr/local/bin/openstack-ansible <<EOF
+#!/usr/bin/env bash
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# (c) 2014, Kevin Carter <kevin.carter@rackspace.com>
+
+# OpenStack wrapper tool to ease the use of ansible with multiple variable files.
+
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}"
+
+function info() {
+    echo -e "\e[0;35m\${@}\e[0m"
+}
+
+# Discover the variable files.
+VAR1="\$(for i in \$(ls /etc/openstack_deploy/user_*.yml); do echo -ne "-e @\$i "; done)"
+
+# Provide information on the discovered variables.
+info "Variable files: \"\${VAR1}\""
+
+# Run the ansible playbook command.
+\$(which ansible-playbook) \${VAR1} \$@
+EOF
+
+# Ensure wrapper tool is executable
+chmod +x /usr/local/bin/openstack-ansible
+
+echo "openstack-ansible script created."
+echo "System is bootstrapped and ready for use."

scripts/deploy.sh

@@ -28,14 +28,17 @@ function run_ansible {
 }
 
 # begin the bootstrap process
-cd ${OA_DIR}
+cd "${RPCD_DIR}/.."
 
 # bootstrap ansible and install galaxy roles (needed whether AIO or multinode)
 which openstack-ansible || ./scripts/bootstrap-ansible.sh
+ansible-playbook -i 'localhost,' /opt/rpc-openstack/rpcd/playbooks/repo-fetcher.yml
 ansible-galaxy install --role-file=/opt/rpc-openstack/ansible-role-requirements.yml --force \
                            --roles-path=/opt/rpc-openstack/rpcd/playbooks/roles
+ansible-galaxy install --role-file=/opt/rpc-openstack/openstack-ansible/ansible-role-requirements.yml --force --ignore-errors
 
 # bootstrap the AIO
+cd ${OA_DIR}
 if [[ "${DEPLOY_AIO}" == "yes" ]]; then
   # Determine the largest secondary disk device available for repartitioning
   DATA_DISK_DEVICE=$(lsblk -brndo NAME,TYPE,RO,SIZE | \

scripts/linting.sh

@@ -16,7 +16,11 @@
 ## Shell Opts ----------------------------------------------------------------
 set -euo pipefail
 
-# we need to fetch roles
+# linting is done in travis, update the path
+sed -i 's|/opt|/home/travis/build/rcbops|g' rpcd/playbooks/repo-fetcher.yml
+
+# we need to fetch roles and openstack-ansible
+ansible-playbook -i <(echo '[all]\nlocalhost ansible_connection=local') rpcd/playbooks/repo-fetcher.yml
 ansible-galaxy install --role-file=ansible-role-requirements.yml --force --roles-path=/home/travis/build/rcbops/rpc-openstack/rpcd/playbooks/roles
 
 python -m tox

scripts/os-detection.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Determines the operating system of a host and returns bash variables."""
+import platform
+
+
+template = """
+HOST_DISTRO="{0}"
+HOST_VERSION="{1}"
+HOST_CODENAME="{2}"
+""".format(*platform.linux_distribution())
+
+print(template)