Repository navigation
Home
- Install OZ onto temporary host - ensure oz >= v0.13
yum -y install oz
- Clone alces-symphony (oztemplates for now) repository
git clone https://github.com/alces-software/oztemplates.git
TODO: Wrap up all OZ templates/cfg into alces-symphony repo
-
Modify
oztemplates/config/oz.cfgto suit requirements, mainlybridge_nameandoutput_dir -
Switch to
oztemplates/centos/7.0/and run vm creator script as root.
sudo ./symphony_create.sh
TODO: Wrap up adding Networks and Drives post creation in symphony_create.sh script.
- Define secondary Network for
symphony-directorinalces-director.xml:
<interface type="bridge"> <source bridge="dmz"/> <model type="virtio"/> </interface>
NOTE: This step is not essential - it is possible to perform later if VMs are being moved at a later date before Repos are synced.
- Create and add data disk for
symphony-repoand define inalces-repo.xml - Define in
alces-repo.xml:
<disk device="disk" type="file"> <target bus="virtio" dev="vdb"/> <source file="/opt/vm2/symphony-repo-data.qcow2"/> <driver type="qcow2" name="qemu"/> </disk>
- Create disk in your defined storage pool as set during the initial configuration (possibly
/var/lib/libvirt/images)
qemu-img create -f qcow2 -o preallocation=metadata /var/lib/libvirt/images/alces-repo-data.qcow2 80G
- Define images in virsh from
.xmlfiles - run this foralces-director.xml,alces-repo.xml,alces-monitor.xml.
virsh define alces-director.xml
base image = (centos 7 minimal)
- General setup
systemctl disable NetworkManager
-
Setup network as hostname as per instructions in specific symphony-x
-
Setup firewall
systemctl enable firewalld
systemctl start firewalld
- Checkout symphony home
yum install git
cd /var/lib
git clone https://github.com/ste78/symphony.git
- Puppet Client - everything but symphony
if [ `hostname -s` != "symphony-director" ]; then
/var/lib/symphony/puppet/bin/install_client.sh
fi
#Follow instructions about signing certificates
puppet agent -t --environment=symphony
- packages:
yum -y install vim yum-utils
#SYMPHONY DIRECTOR base image = (centos 7 minimal)
- Configure hostname + networking
echo symphony-director.prv.cluster > /etc/hostname
- Run installer
/var/lib/symphony/director/bin/install_all.sh
- eth0 = primary network
IPV6INIT=no
BOOTPROTO=none
DEVICE=eth0
ONBOOT=yes
IPADDR=10.78.254.1
NETMASK=255.255.0.0
NETWORK=10.78.0.0
ZONE=private
NM_CONTROLLED=no
- eth1 = public network dhcp
IPV6INIT=no
BOOTPROTO=dhcp
DEVICE=eth1
ONBOOT=yes
PEERDNS=no
ZONE=external
NM_CONTROLLED=no
- Firewall
firewall-cmd --zone=external --add-masquerade --permanent
firewall-cmd --new-zone private --permanent
#dhcp
firewall-cmd --zone private --add-service dhcpd --permanent
#tftp
firewall-cmd --zone private --add-service tftp --permanent
#http
firewall-cmd --zone private --add-service http --permanent
#named
firewall-cmd --add-service dns --zone private --permanent
#smtp
firewall-cmd --add-service smtp --zone private --permanent
###OpenVPN
- Install
yum --config http://symphony-repo/configs/centos/7.0/yum.conf --enablerepo=epel install openvpn easy-rsa
- Prepare SSL
mkdir /etc/openvpn/easyrsa
CN=`hostname -f`
cp -pav /usr/share/easy-rsa/2.0/* /etc/openvpn/easyrsa/
sed -i /etc/openvpn/easyrsa/vars \
-e 's|KEY_COUNTRY=.*$|KEY_COUNTRY=\"GB\"|g' \
-e 's|KEY_PROVINCE=.*|KEY_PROVINCE=\"Oxfordshire"|g' \
-e 's|KEY_CITY=.*|KEY_CITY=\"Bicester\"|g' \
-e 's|KEY_ORG=.*|KEY_ORG=\"Alces Software Ltd\"|g' \
-e 's|KEY_EMAIL=.*|KEY_EMAIL=\"ssl@alces-software.com\"|g' \
-e 's|KEY_NAME=.*|KEY_NAME=\"server\"|g' \
-e 's|KEY_OU=.*|KEY_OU=\"server\"|g' \
-e "s|KEY_CN=.*|KEY_CN=\"${CN}\"|g"
cd /etc/openvpn/easyrsa/
source ./vars
./clean-all
./pkitool --initca
./pkitool --server server
./build-dh
- Management vpn
cat << EOF > /etc/openvpn/manage.conf
port 1194
proto tcp
dev tun0
ca /etc/openvpn/easyrsa/keys/ca.crt
cert /etc/openvpn/easyrsa/keys/server.crt
key /etc/openvpn/easyrsa/keys/server.key
dh /etc/openvpn/easyrsa/keys/dh2048.pem
server 10.80.0.0 255.255.255.248
ifconfig-pool-persist ipp.txt
push "route 10.78.0.0 255.255.0.0"
duplicate-cn
keepalive 10 120
comp-lzo
persist-key
persist-tun
status openvpn-status.log
log /var/log/openvpn.log
log-append /var/log/openvpn.log
verb 3
client-cert-not-required
username-as-common-name
plugin /usr/lib64/openvpn/plugins/openvpn-plugin-auth-pam.so openvpn-manage
EOF
FQDN=10.101.0.46
cat << EOF > /etc/openvpn/manage.client.conf
client
dev tun
proto tcp
remote $FQDN 1194
resolv-retry infinite
nobind
persist-key
persist-tun
ca ca.crt
auth-user-pass
ns-cert-type server
comp-lzo
verb 3
EOF
cat << EOF > /etc/openvpn/manage.users
admin
EOF
#Only allow specific users
sed -e "/^account required pam_nologin.so$/iauth required pam_listfile.so onerr=fail item=group sense=allow file=/etc/openvpn/manage.users" /etc/pam.d/login > /etc/pam.d/openvpn-manage
systemctl enable openvpn@manage.service
systemctl start openvpn@manage.service
#Create management vpn user
useradd admin
passwd admin
firewall-cmd --zone trusted --add-interface tun0 --permanent
- Install
yum --config http://symphony-repo/configs/centos/7.0/yum.conf --enablerepo=epel install rsyslog
systemctl enable rsyslog
firewall-cmd --add-port 514/udp --zone private --permanent
firewall-cmd --add-port 514/tcp --zone private --permanent
- Configure
cat << EOF >> /etc/rsyslog.conf
\$template remoteAuth, "/var/log/slave/%HOSTNAME%/%PROGRAMNAME%.log"
\$template remoteMessage, "/var/log/slave/%HOSTNAME%/messages.log"
authpriv.* ?remoteAuth
*.info,mail.none,authpriv.none,cron.none ?remoteMessage
EOF
sed -i -e "s/^#\$ModLoad imudp.*$/\$ModLoad imudp/g" /etc/rsyslog.conf
sed -i -e "s/^#\$UDPServerRun 514.*$/\$UDPServerRun 514/g" /etc/rsyslog.conf
sed -i -e "s/^#\$ModLoad imtcp.*$/\$ModLoad imtcp/g" /etc/rsyslog.conf
sed -i -e "s/^#\$InputTCPServerRun 514.*$/\$InputTCPServerRun 514/g" /etc/rsyslog.conf
#log rotate
cat << EOF > /etc/logrotate.d/rsyslog-remote
/var/log/slave/*/* {
sharedscripts
compress
rotate 2
postrotate
/bin/kill -HUP \`cat /var/run/syslogd.pid 2> /dev/null\` 2> /dev/null || true
/bin/kill -HUP \`cat /var/run/rsyslogd.pid 2> /dev/null\` 2> /dev/null || true
endscript
}
EOF
-
TODO - requires network access for passenger gems (used to be in epel but misisng in 7) switch to different ruby engine or wait for passenger in epel7
-
Install
yum --config http://symphony-repo/configs/centos/7.0/yum.conf --enablerepo=epel --enablerepo=puppet-deps --enablerepo=puppet-base install puppet puppet-server hiera httpd httpd-devel mod_ssl ruby-devel rubygems gcc gcc-c++ libcurl-devel openssl-devel zlib-devel
gem install rack passenger
passenger-install-apache2-module
- Configure puppet
cat << EOF > /etc/puppet/puppet.conf
[main]
vardir = /var/lib/puppet
logdir = /var/log/puppet
rundir = /var/run/puppet
ssldir = \$vardir/ssl
allow_duplicate_certs = true
pluginsync = true
hiera_config=/etc/puppet/hiera.yaml
#dns_alt_names=
[agent]
classfile = \$vardir/classes.txt
localconfig = \$vardir/localconfig
report = true
pluginsync = true
certname = symphony-director
server = symphony-director
listen = false
environment = production
[master]
reports = store
certname = symphony-director
ca = true
environmentpath = \$confdir/environments/
EOF
cat << EOF > /etc/puppet/hiera.yaml
---
:backends:
- yaml
:yaml:
:datadir: /etc/puppet/hieradata/
:hierarchy:
- common
- "%{::clientcert}"
- "%{::environment}"
- site
EOF
#Create default environment
mv /etc/puppet/environments/example_env /etc/puppet/environments/production
#Start and stop puppetmaster to generate initial certs
systemctl start puppetmaster
puppet agent -t
systemctl stop puppetmaster
- Configure http
#GET LASTEST mod_passenger
PASSENGER_MOD=`find /usr/local/share/gems/gems/passenger-* -type f -iname mod_passenger.so -printf '%T@ %p\n'| sort -n | tail -1 | cut -f2- -d" "`
PASSENGER_ROOT=`echo $PASSENGER_MOD | sed -e 's|/buildout/apache2/mod_passenger.so||'`
cat << EOF > /etc/httpd/conf.d/passenger.conf
LoadModule passenger_module $PASSENGER_MOD
<IfModule mod_passenger.c>
PassengerRoot $PASSENGER_ROOT
PassengerRuby /usr/bin/ruby
PassengerEnabled off
</IfModule>
EOF
cat << EOF > /etc/httpd/conf.d/puppet-master.conf
PassengerHighPerformance On
PassengerMaxPoolSize 12
PassengerMaxRequests 1000
PassengerPoolIdleTime 600
Listen 8140
<VirtualHost *:8140>
PassengerEnabled On
SSLEngine On
SSLProtocol All -SSLv2
SSLCipherSuite HIGH:!ADH:RC4+RSA:-MEDIUM:-LOW:-EXP
SSLCertificateFile /var/lib/puppet/ssl/certs/symphony-director.pem
SSLCertificateKeyFile /var/lib/puppet/ssl/private_keys/symphony-director.pem
SSLCertificateChainFile /var/lib/puppet/ssl/ca/ca_crt.pem
SSLCACertificateFile /var/lib/puppet/ssl/ca/ca_crt.pem
SSLCARevocationFile /var/lib/puppet/ssl/ca/ca_crl.pem
SSLCARevocationCheck chain
SSLVerifyClient optional
SSLVerifyDepth 1
SSLOptions +StdEnvVars +ExportCertData
RequestHeader set X-SSL-Subject %{SSL_CLIENT_S_DN}e
RequestHeader set X-Client-DN %{SSL_CLIENT_S_DN}e
RequestHeader set X-Client-Verify %{SSL_CLIENT_VERIFY}e
DocumentRoot /usr/share/puppet/rack/puppetmasterd/public
<Directory /usr/share/puppet/rack/puppetmasterd/>
Options None
AllowOverride None
Order Allow,Deny
Allow from All
</Directory>
ErrorLog /var/log/httpd/puppet-server.example.com_ssl_error.log
CustomLog /var/log/httpd/puppet-server.example.com_ssl_access.log combined
</VirtualHost>
EOF
mkdir -p /usr/share/puppet/rack/puppetmasterd
mkdir /usr/share/puppet/rack/puppetmasterd/public /usr/share/puppet/rack/puppetmasterd/tmp
cp /usr/share/puppet/ext/rack/config.ru /usr/share/puppet/rack/puppetmasterd/
chown puppet:puppet /usr/share/puppet/rack/puppetmasterd/config.ru
systemctl restart httpd
#Open firewall
firewall-cmd --add-port 8140/tcp --zone private --permanent
- Install Symphony templates
/var/lib/symphony/puppet/bin/install_all.sh
- Generate hiera config
/var/lib/symphony/puppet/environments/symphony/modules/symphonydirector/gen_config.sh
- Install
yum --config http://symphony-repo/configs/centos/7.0/yum.conf --enablerepo=epel --enablerepo=cobbler install cobbler httpd dhcp fence-agents tftp xinetd tftp-server cobbler-web bind bind-utils bind-chroot
- Modify config
#Set default password
PASS=`openssl passwd -1` ; sed -i /etc/cobbler/settings -e "s/^default_password_crypted.*/default_password_crypted: \"$PASS\"/g"
#Set master ip
IP=`hostname -i`; sed -i /etc/cobbler/settings -e "s/^server:.*/server: $IP/g"
IP=`hostname -i`; sed -i /etc/cobbler/settings -e "s/^next_server:.*/next_server: $IP/g"
#Enable dhcp management
sed -ri /etc/cobbler/settings -e "s/^#?manage_dhcp.*/manage_dhcp: 1/g"
#Enable DNS
sed -ri /etc/cobbler/settings -e "s/^#?manage_dns.*/manage_dns: 1/g"
sed -ri /etc/cobbler/settings -e "s/^#?manage_forward_zones.*/manage_forward_zones: ['prv.cluster.local']/g"
sed -ri /etc/cobbler/settings -e "s/^#?manage_reverse_zones.*/manage_reverse_zones: ['10.78']/g"
#Enable Puppet management
sed -ri /etc/cobbler/settings \
-e "s/^#?puppet_auto_setup:.*/puppet_auto_setup: 1/g" \
-e "s/^#?sign_puppet_certs_automatically:.*/sign_puppet_certs_automatically: 1/g" \
-e "s/^#?remove_old_puppet_certs_automatically:.*/remove_old_puppet_certs_automatically: 1/g" \
-e "s/^#?puppet_version:.*/puppet_version: 3/g" \
-e "s/^#?puppet_server:.*/puppet_server: symphony-director/g" \
-e "s/^#?puppetca_path:.*/puppetca_path: "/usr/bin/puppet"/g"
#Other
sed -i /etc/cobbler/settings -e "s/^pxe_just_once.*$/pxe_just_once: 1/g"
cat << EOF > /etc/cobbler/dhcp.template
#Configure dhcp template
# ******************************************************************
# Cobbler managed dhcpd.conf file
#
# generated from cobbler dhcp.conf template ($date)
# Do NOT make changes to /etc/dhcpd.conf. Instead, make your changes
# in /etc/cobbler/dhcp.template, as /etc/dhcpd.conf will be
# overwritten.
#
# ******************************************************************
ddns-update-style interim;
allow booting;
allow bootp;
ignore client-updates;
set vendorclass = option vendor-class-identifier;
option pxe-system-type code 93 = unsigned integer 16;
subnet 10.78.0.0 netmask 255.255.0.0 {
option routers 10.78.254.1;
option domain-name-servers 10.78.254.1;
option domain-search "prv.cluster.local","cluster.local";
option subnet-mask 255.255.0.0;
range dynamic-bootp 10.78.0.1 10.78.0.254;
default-lease-time 21600;
max-lease-time 43200;
next-server \$next_server;
class "pxeclients" {
match if substring (option vendor-class-identifier, 0, 9) = "PXEClient";
if option pxe-system-type = 00:02 {
filename "ia64/elilo.efi";
} else if option pxe-system-type = 00:06 {
filename "grub/grub-x86.efi";
} else if option pxe-system-type = 00:07 {
filename "grub/grub-x86_64.efi";
} else {
filename "pxelinux.0";
}
}
}
#for dhcp_tag in \$dhcp_tags.keys():
## group could be subnet if your dhcp tags line up with your subnets
## or really any valid dhcpd.conf construct ... if you only use the
## default dhcp tag in cobbler, the group block can be deleted for a
## flat configuration
# group for Cobbler DHCP tag: \$dhcp_tag
group {
#for mac in \$dhcp_tags[\$dhcp_tag].keys():
#set iface = \$dhcp_tags[\$dhcp_tag][\$mac]
host \$iface.name {
hardware ethernet \$mac;
#if \$iface.ip_address:
fixed-address \$iface.ip_address;
#end if
#if \$iface.hostname:
option host-name "\$iface.hostname";
#end if
#if \$iface.netmask:
option subnet-mask \$iface.netmask;
#end if
#if \$iface.gateway:
option routers \$iface.gateway;
#end if
#if \$iface.enable_gpxe:
if exists user-class and option user-class = "gPXE" {
filename "http://\$cobbler_server/cblr/svc/op/gpxe/system/\$iface.owner";
} else if exists user-class and option user-class = "iPXE" {
filename "http://\$cobbler_server/cblr/svc/op/gpxe/system/\$iface.owner";
} else {
filename "undionly.kpxe";
}
#else
filename "\$iface.filename";
#end if
## Cobbler defaults to \$next_server, but some users
## may like to use \$iface.system.server for proxied setups
next-server \$next_server;
## next-server \$iface.next_server;
}
#end for
}
#end for
EOF
- Set WebUI password for 'admin' user
htdigest /etc/cobbler/users.digest 'Cobbler' admin
- Start services
systemctl start cobblerd.service
systemctl enable cobblerd.service
systemctl start httpd
systemctl enable httpd
systemctl start xinetd
systemctl enable xinetd
systemctl start dhcpd
systemctl enable dhcpd
systemctl enable named
- Get loaders
#TODO requires internet access
cobbler get-loaders
- Install Symphony templates
/var/lib/symphony/cobbler/bin/install_all.sh
- Configure DNS
SITE_DNS=10.101.0.1
cat << EOF > /etc/cobbler/named.template
options {
listen-on port 53 { any; };
directory "/var/named";
dump-file "/var/named/data/cache_dump.db";
statistics-file "/var/named/data/named_stats.txt";
memstatistics-file "/var/named/data/named_mem_stats.txt";
allow-query { any; };
recursion yes;
dnssec-enable no;
dnssec-validation no;
dnssec-lookaside auto;
forward first;
forwarders {
$SITE_DNS;
};
};
logging {
channel default_debug {
file "data/named.run";
severity dynamic;
};
};
#for \$zone in \$forward_zones
zone "\${zone}." {
type master;
file "\$zone";
};
#end for
#for \$zone, \$arpa in \$reverse_zones
zone "\${arpa}." {
type master;
file "\$zone";
};
#end for
EOF
- For each managed domain
cat << EOF > /etc/cobbler/zone_templates/prv.cluster.local
\\\$TTL 300
@ IN SOA \$cobbler_server. nobody.example.com. (
\$serial ; Serial
600 ; Refresh
1800 ; Retry
604800 ; Expire
300 ; TTL
)
IN NS \$cobbler_server.
symphony-director IN A 10.78.254.1
@ IN MX 10 symphony-director
symphony-repo IN A 10.78.254.3
symphony-monitor IN A 10.78.254.4
\$cname_record
\$host_record
EOF
cat << EOF > /etc/cobbler/zone_templates/10.78
\\\$TTL 300
@ IN SOA \$cobbler_server. nobody.example.com. (
\$serial ; Serial
600 ; Refresh
1800 ; Retry
604800 ; Expire
300 ; TTL
)
IN NS \$cobbler_server.
1.254 IN PTR symphony-director.prv.cluster.local.;
3.254 IN PTR symphony-repo.prv.cluster.local.;
4.254 IN PTR symphony-monitor.prv.cluster.local;
\$cname_record
\$host_record
EOF
- Create config
#CENTOS7.0
curl http://symphony-repo/pulp/repos/centos/7.0/os/images/pxeboot/initrd.img > /var/lib/symphony/cobbler/distro_files/CentOS7.0_initrd
curl http://symphony-repo/pulp/repos/centos/7.0/os/images/pxeboot/vmlinuz > /var/lib/symphony/cobbler/distro_files/CentOS7.0_kernel
cobbler distro add --name CentOS7.0 --kernel=/var/lib/symphony/cobbler/distro_files/CentOS7.0_kernel --initrd=/var/lib/symphony/cobbler/distro_files/CentOS7.0_initrd --arch=x86_64 --ksmeta="tree=centos/7.0" --breed redhat --os-version=rhel7 --kopts="syslog=@@server@@"
cobbler sync
#SYMPHONY-REPO (centos 7 minimal vm with large data disk (60g+) mounted on /var/lib/pulp/content/)
- large data disk (30g+) mounted /var/lib/mongodb/
- Configure hostname + networking:
echo symphony-repo.prv.cluster.local > /etc/hostname
- eth0 = primary network
cat << EOF > /etc/sysconfig/network-scripts/ifcfg-eth0
IPV6INIT=no
BOOTPROTO=none
DEVICE=eth0
ONBOOT=yes
IPADDR=10.78.254.3
NETMASK=255.255.0.0
NETWORK=10.78.0.0
GATEWAY=10.78.254.1
DNS=10.78.254.1
ZONE=private
NM_CONTROLLED=no
EOF
- Firewall
firewall-cmd --new-zone private --permanent
firewall-cmd --add-service http --zone private --permanent
firewall-cmd --add-service https --zone private --permanent
firewall-cmd --add-service ssh --zone private --permanent
#qpidd inbound non-https
firewall-cmd --add-port 5672/tcp --zone private --permanent
- Install repos:
curl https://repos.fedorapeople.org/repos/pulp/pulp/rhel-pulp.repo > /etc/yum.repos.d/pulp.repo
#TODO fix 7 != 7server bug in pulp repos
sed -i -e 's/$releasever/7Server/g' /etc/yum.repos.d/pulp.repo
yum -y install https://anorien.csc.warwick.ac.uk/mirrors/epel/7/x86_64/e/epel-release-7-1.noarch.rpm
- Install database
yum -y install mongodb-server
systemctl enable mongod
systemctl start mongod
- Install message broker
yum -y install qpid-cpp-server qpid-cpp-server-store
#TODO Fix bug for augeas with qpidd
ln -snf /etc/qpid/qpidd.conf /etc/qpidd.conf
augtool -s set /files/etc/qpidd.conf/auth no
systemctl enable qpidd
systemctl start qpidd
- Install pulp
yum groupinstall pulp-server-qpid
- Edit /etc/pulp/server.conf
TODO use SSL If you mount up /var/lib/pulp/content from external disk ensure dir is owned by apache:apache
- Init pulp DB:
sudo -u apache pulp-manage-db
- start services
systemctl enable httpd
systemctl start httpd
systemctl enable pulp_workers
systemctl start pulp_workers
systemctl enable pulp_celerybeat
systemctl start pulp_celerybeat
systemctl enable pulp_resource_manager
systemctl start pulp_resource_manager
- Packages
yum groupinstall pulp-admin
edit /etc/pulp/admin/admin.conf: server=symphony-repo.prv.cluster verify_ssl=false
TODO enable proper ssl
- Setup Admin user
pulp-admin login -u admin -p admin
pulp-admin auth user update --login admin --password 'sqrt(s*w)'
#TODO set site password
##Import Repos
###Centos 7
/var/lib/symphony/repo/bin/import_pulp.sh centos7.0
/var/lib/symphony/repo/bin/import_pulp.sh el7other
/var/lib/symphony/repo/bin/import_pulp.sh el7symphony#SYMPHONY-MONITOR (centos 7 minimal)
- Configure hostname + networking:
echo symphony-monitor.prv.cluster.local > /etc/hostname
- eth0 = primary network
cat << EOF > /etc/sysconfig/network-scripts/ifcfg-eth0
IPV6INIT=no
BOOTPROTO=none
DEVICE=eth0
ONBOOT=yes
IPADDR=10.78.254.4
NETMASK=255.255.0.0
NETWORK=10.78.0.0
GATEWAY=10.78.254.1
DNS=10.78.254.1
ZONE=private
NM_CONTROLLED=no
EOF
=== Firewall
firewall-cmd --new-zone private --permanent
firewall-cmd --add-interface eth0 --zone private --permanent
firewall-cmd --add-port 8659/tcp --zone private --permanent
firewall-cmd --add-port 8649/udp --zone private --permanent
firewall-cmd --add-service http --zone private --permanent
firewall-cmd --reload
=== Ganglia
- Install packages
yum-config-manager --enable epel
=== Nagios
- Set Password
htpasswd /etc/nagios/passwd nagiosadmin
Copyright (c) 2008-2015 Alces Software Ltd