Permalink
Browse files

Add demo VM build tool

  • Loading branch information...
1 parent a7bcb28 commit 1f5da4b1e7bb7541f321b87edbfa3fdc6a6e302c @zeha zeha committed Apr 10, 2012
Showing with 432 additions and 3 deletions.
  1. +1 −0 .gitignore
  2. +3 −0 .mrconfig
  3. +47 −2 README.md
  4. +118 −0 build-vm
  5. +1 −1 setup.sh
  6. +109 −0 util/build/hspostinst
  7. +62 −0 util/build/packages
  8. +91 −0 util/build/post-scripts/10-install-stuff
View
@@ -1 +1,2 @@
src
+tmp
View
@@ -40,3 +40,6 @@ checkout = checkout_hs hs-sshgateway
[src/hs-usercontrolpanel]
checkout = checkout_hs hs-usercontrolpanel
+
+[src/app-templates]
+checkout = checkout_hs app-templates
View
@@ -1,8 +1,8 @@
HostingStack build scripts
==========================
-Base requirement: Debian wheezy, amd64
-Preferably as a physical machine. At least 10G+ FREE disk space.
+Base requirement: Debian wheezy, x86-64.
+Preferably as a fast machine, VM is okay. At least 10G+ FREE disk space.
WARNING: At this time most of the repositories are not yet public.
@@ -26,6 +26,51 @@ Building packages
Run `mr run build`.
+Building a demo VM
+------------------
+
+ * Preparations above done.
+ * Ran `mr run build` once, successfully.
+ * Run ./build-vm demo.raw
+ * or, for a VMware VM, run ./build-vm --format vmdk demo.vmdk
+ * Load demo.img or demo.vmdk in a Hypervisor of your choice, and give
+ it some time to boot. 3GB RAM minimum.
+ * After bootup it will create templates, this will take some minutes.
+
+If you have a Debian mirror near you, also set --mirror http://url.to.mirror/debian .
+
+
+
+Demo VM Info
+------------
+
+The default passwords are `CHANGEME`. This applies to user root (for
+SSH), and to the admin account (demo1@hostingstack.org) and the
+normal user (demo2@hostingstack.org).
+
+SSL for applications is currently disabled.
+
+In a production setup, the various services would be split over multiple
+machines.
+
+Important ports:
+
+ * Port 9100, HTTP: Enduser control panel
+ * Port 9000, HTTP: Admin dashboard
+ * Port 80, HTTP: application port
+ * Port 443: HTTP/SSL: application port
+ * Port 2200: Enduser SSH
+ * Port 22: SSH for management (PAM auth)
+
+Not-so-important ports:
+
+ * Port 9200: internal storage
+ * Port 3142: APT cache
+ * Redis, PostgreSQL, MySQL run on their default ports
+ * Port 9090: HSAgent Thrift server
+
+
+
Legalese
--------
View
118 build-vm
@@ -0,0 +1,118 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+require 'getoptlong'
+
+class Builder
+ def initialize
+ @basepath = File.expand_path('../', __FILE__)
+ @vmformat = 'raw'
+ @mirror = 'http://cdn.debian.net/debian'
+ @vzver = "042stab053.5"
+ @outfile = nil
+ end
+
+ def usage
+ puts <<-EOF
+build-vm [OPTION] ... FILE
+
+-h, --help Show this help
+--format [format] Select VM disk format (raw [def], qcow2, vmdk)
+--mirror [url] Debian mirror to use
+ (def: #{@mirror})
+
+FILE: output filename
+EOF
+ end
+
+ def run
+ opts = GetoptLong.new(
+ [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+ [ '--format', GetoptLong::REQUIRED_ARGUMENT ],
+ [ '--mirror', GetoptLong::REQUIRED_ARGUMENT ],
+ )
+
+ opts.each do |opt, arg|
+ case opt
+ when '--help'
+ usage
+ exit 0
+ when '--format'
+ @vmformat = arg
+ when '--mirror'
+ @mirror = arg
+ end
+ end
+
+ outfile = ARGV.shift
+ if outfile.nil? || outfile.empty?
+ usage
+ exit 1
+ end
+ @outfile = File.expand_path(outfile)
+
+ puts "HostingStack/hs-build build-vm"
+ puts " * Using basepath \"#{@basepath}\""
+ puts " * Writing VM image to \"#{@outfile}\""
+
+ prepare_kernels
+ build_vm
+
+ puts " * Done."
+ end
+
+ def system(cmdline)
+ Kernel.system(cmdline)
+ raise "Command \"#{cmdline}\" failed with error code #{$?.exitstatus}" unless $?.exitstatus == 0
+ end
+
+ def prepare_kernels
+ kernels = File.join(@basepath, 'tmp/kernels')
+ puts " * Preparing OpenVZ Kernels ..."
+ FileUtils.mkdir_p kernels
+ def download_vzkernel(path, rpm)
+ return if File.exist?(File.join(path, rpm))
+ system "cd \"#{path}\" && wget http://download.openvz.org/kernel/branches/rhel6-2.6.32/#{@vzver}/#{rpm}"
+ end
+ def make_vzkernel(path, rpm, deb)
+ return if File.exist?(File.join(path, deb))
+ download_vzkernel path, rpm
+ system "cd \"#{path}\" && fakeroot alien -k -d #{rpm}"
+ end
+ make_vzkernel kernels, "vzkernel-2.6.32-#{@vzver}.x86_64.rpm", "vzkernel_2.6.32-#{@vzver}_amd64.deb"
+ make_vzkernel kernels, "vzkernel-devel-2.6.32-#{@vzver}.x86_64.rpm", "vzkernel-devel_2.6.32-#{@vzver}_amd64.deb"
+ end
+
+ def build_vm
+ post_scripts = File.join(@basepath, 'util/build/post-scripts')
+ packages = File.join(@basepath, 'util/build/packages')
+ puts " * Building VM image ..."
+ vmfile = @outfile
+ if @vmformat != 'raw'
+ vmfile = vmfile + '.tmp'
+ end
+
+ cmdline = "sudo HS_BASEPATH=\"#{@basepath}\" grml-debootstrap --vmfile "
+ cmdline << " -r wheezy --arch amd64 --filesystem ext3 --force"
+ cmdline << " --vmsize 4G --hostname demo --password CHANGEME"
+ cmdline << " -t #{vmfile}"
+ cmdline << " --mirror #{@mirror}"
+ cmdline << " --scripts #{post_scripts}"
+ cmdline << " --packages #{packages}"
+ puts " * Running \"#{cmdline}\" ..."
+ system(cmdline)
+
+ if @vmformat != 'raw'
+ puts " * Converting to format #{@vmformat} ..."
+ system("qemu-img convert -p -f raw -O #{@vmformat} #{vmfile} #{@outfile}")
+ File.unlink vmfile
+ end
+ end
+end
+
+begin
+ Builder.new.run
+rescue => e
+ puts e
+ exit 1
+end
+exit 0
View
@@ -5,7 +5,7 @@ set -e
BASEDIR=$(readlink -f .)
echo " * Installing packages..."
-sudo apt-get install -y pbuilder mr bundler ruby1.9.1-full
+sudo apt-get install -y pbuilder mr bundler ruby1.9.1-full grml-debootstrap build-essential alien fakeroot wget
echo " * Setting up pbuilder..."
cat <<EOF | sudo tee /etc/pbuilder/hsrc
View
@@ -0,0 +1,109 @@
+#!/bin/bash
+echo HostingStack postinst starting
+echo Installing postgresql
+apt-get install -y postgresql postgresql-9.1
+echo Installing HostingStack packages
+PACKAGES=$(cat /root/packages/list)
+DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y $PACKAGES
+rm -f /etc/apt/sources.list.d/hslocal.list
+
+echo Services configuration ...
+echo PG
+echo 'host all all 0.0.0.0/0 md5' >> /etc/postgresql/9.1/main/pg_hba.conf
+echo "listen_addresses = '*'" >> /etc/postgresql/9.1/main/postgresql.conf
+invoke-rc.d postgresql restart
+
+echo MySQL
+grep -v 'bind-address' /etc/mysql/my.cnf > /tmp/my.cnf
+cp /tmp/my.cnf /etc/mysql/my.cnf; rm /tmp/my.cnf
+invoke-rc.d mysql restart
+
+echo Monit
+cat <<EOCONF >/etc/monit/monitrc
+set daemon 10
+set logfile syslog facility log_daemon
+set httpd port 2812
+ allow admin:monit
+
+include /var/lib/hs/monit
+EOCONF
+invoke-rc.d monit restart
+
+echo Memcached
+invoke-rc.d memcached stop
+sed -i 's/ENABLE_MEMCACHED=yes/ENABLE_MEMCACHED=no/' /etc/default/memcached
+
+echo HostingStack shared
+cat <<EOCONF >/etc/hs/shared/hs.yml
+production:
+ cc_api_host: http://localhost:9000
+ codemanager_host : http://localhost:9200
+ cc_api_user: CCUSER
+ cc_api_password: CCCHANGEME
+ billing_api_user: BILLINGUSER
+ billing_api_password: BILLINGCHANGEME
+ agent_api_user: AGENTUSER
+ agent_api_password: AGENTCHANGEME
+ redis: 127.0.0.1:6379
+EOCONF
+
+echo HostingStack ERF
+cat <<EOCONF >>/etc/hs/envroot-factory/production.rb
+host_ip = %x{ip route get 4.4.4.4}.chomp.split[6]
+\$config[:apt_sources] = <<EOSOURCES
+deb http://#{host_ip}:3142/cdn.debian.net/debian squeeze main non-free contrib
+deb http://#{host_ip}:3142/cdn.debian.net/debian squeeze-updates main non-free contrib
+deb http://#{host_ip}:3142/security.debian.org/ squeeze/updates main non-free contrib
+EOSOURCES
+EOCONF
+
+echo Debian nginx
+export RAILS_ENV=production
+find /srv/hs-archive -type d -print0 | xargs -0 chown www-data:adm
+rm -f /etc/nginx/sites-enabled/default
+/etc/init.d/nginx restart
+
+echo Enable envroot-factory workers
+update-service -a /usr/lib/hs/envroot-factory/services/hs-envroot-factory-1
+update-service -a /usr/lib/hs/envroot-factory/services/hs-envroot-factory-2
+update-service -a /usr/lib/hs/envroot-factory/services/hs-envroot-factory-3
+update-service -a /usr/lib/hs/envroot-factory/services/hs-envroot-factory-4
+update-service -a /usr/lib/hs/envroot-factory/services/hs-envroot-factory-5
+
+echo HostingStack CloudController DB Setup
+HOST_IP=$(ip route get 4.4.4.4 | awk '/src/{print $7}')
+su - postgres -c "echo UPDATE servers SET name=\'$(hostname -s)\', internal_ip=\'$HOST_IP\', external_ip=\'$HOST_IP\' WHERE name=\'host\' | psql hscloudcontroller"
+echo enable CC workers
+update-service -a /usr/lib/hs/cloudcontroller/services/hs-cloudcontroller-ruote-worker
+update-service -a /usr/lib/hs/cloudcontroller/services/hs-cloudcontroller-rrs-worker
+update-service -a /usr/lib/hs/cloudcontroller/services/hs-cloudcontroller-task-worker
+update-service -a /usr/lib/hs/cloudcontroller/services/hs-cloudcontroller-http
+
+echo HostingStack UCP
+update-service -a /usr/lib/hs/usercontrolpanel/services/hs-usercontrolpanel-http
+
+echo Create user for postgres service agent
+su postgres -c "echo CREATE USER hs_service_pg ENCRYPTED PASSWORD \'CHANGEME\' SUPERUSER | psql"
+echo Create user for mysql service agent
+echo "CREATE USER 'hs_svc'@'localhost' IDENTIFIED BY 'CHANGEME'; GRANT ALL PRIVILEGES ON *.* TO 'hs_svc'@'localhost' WITH GRANT OPTION;" | mysql --defaults-file=/etc/mysql/debian.cnf
+echo HostingStack Agent
+update-service -a /usr/lib/hs/agent/services/hs-agent-main
+update-service -a /usr/lib/hs/agent/services/hs-agent-worker
+
+echo HostingStack ssh-gateway
+update-service -a /usr/lib/hs/sshgateway/services/hs-sshgateway
+
+echo HostingStack http-gateway
+#echo Configure correct ssl certificates
+#sed -i -e 's/ssl_certificate .*/ssl_certificate \/etc\/ssl\/private\/wildcard.solidrails.net.pem;/' /etc/hs/httpgateway/hs-httpgateway.conf
+#sed -i -e 's/ssl_certificate_key .*/ssl_certificate_key \/etc\/ssl\/private\/wildcard.solidrails.net.key;/' /etc/hs/httpgateway/hs-httpgateway.conf
+# remove comments from https block (quick'n'dirty'n'goingtofail)
+#sed -i -e 's/^#//g' /etc/hs/httpgateway/hs-httpgateway.conf
+echo Create empty key-material tcb
+tcbmgr create /var/lib/hs/cloud.tcb
+tcbmgr create /var/lib/hs/key_material.tcb
+/etc/init.d/hs-httpgateway restart
+
+echo hspostinst done
+mv /var/hspostinst /var/hspostinst.done
+
View
@@ -0,0 +1,62 @@
+grub2
+bootlogd
+aptitude
+cron
+file
+less
+ntp
+rsync
+openssh-server
+strace
+time
+procinfo
+locales
+pciutils usbutils
+lvm2
+dhcp3-client
+vzctl
+
+rake
+ruby1.9.1-full
+ruby-switch
+redis-server
+sysstat
+nginx-extras
+memcached
+postgresql-client-9.1
+monit
+daemontools-run
+sqlite3
+squid3
+apt-cacher-ng
+mysql-server
+
+apt-utils
+tokyocabinet-bin
+wget
+vim
+telnet
+iptables
+curl
+perl
+
+wget
+dnsutils
+psmisc
+kexec-tools
+
+alien
+
+libsqlite3-ruby1.8
+
+sudo
+debootstrap
+python2.7
+python2.6
+bundler
+unzip
+pigz
+wamerican-huge
+python-minimal python python-support openssl ca-certificates dbconfig-common liberror-perl git-man git ssl-cert postgresql-common
+python-crypto python-openssl python-pam python-paramiko python-pkg-resources python-serial python-twisted-bin python-zope.interface
+python-twisted-core zip
Oops, something went wrong.

0 comments on commit 1f5da4b

Please sign in to comment.