diff --git a/Gemfile.lock b/Gemfile.lock index 0e6272e..d49bcd8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,7 +8,9 @@ PATH dm-mysql-adapter dm-sqlite-adapter extlib + faraday (= 0.8.9) fog + httparty json logging net-ssh @@ -31,7 +33,7 @@ GEM gyoku (>= 0.4.0) nokogiri (>= 1.4.0) awesome_print (1.2.0) - backports (3.3.5) + backports (3.4.0) bcrypt-ruby (3.1.2) buff-config (0.4.0) buff-extensions (~> 0.3) @@ -69,12 +71,12 @@ GEM dm-transactions (~> 1.2.0) dm-types (~> 1.2.0) dm-validations (~> 1.2.0) - debugger (1.6.3) + debugger (1.6.5) columnize (>= 0.3.1) debugger-linecache (~> 1.2.0) - debugger-ruby_core_source (~> 1.2.4) + debugger-ruby_core_source (~> 1.3.1) debugger-linecache (1.2.0) - debugger-ruby_core_source (1.2.4) + debugger-ruby_core_source (1.3.1) delayed_job (2.1.4) activesupport (~> 3.0) daemons @@ -127,16 +129,16 @@ GEM data_objects (= 0.10.13) do_sqlite3 (0.10.13) data_objects (= 0.10.13) - docile (1.1.1) + docile (1.1.2) erubis (2.7.0) - evalhook (0.5.7) + evalhook (0.5.8) partialruby (~> 0.3) sexp_processor (~> 4.0) - excon (0.28.0) + excon (0.31.0) extlib (0.9.16) factory_girl (4.3.0) activesupport (>= 3.0.0) - faraday (0.8.8) + faraday (0.8.9) multipart-post (~> 1.2.0) fastercsv (1.5.5) ffi (1.9.3) @@ -146,9 +148,9 @@ GEM flog (4.2.0) ruby_parser (~> 3.1, > 3.1.0) sexp_processor (~> 4.4) - fog (1.18.0) + fog (1.19.0) builder - excon (~> 0.28.0) + excon (~> 0.31.0) formatador (~> 0.2.0) mime-types multi_json (~> 1.0) @@ -160,9 +162,12 @@ GEM getsource (0.2.1) gssapi (1.0.3) ffi (>= 1.0.1) - gyoku (1.1.0) + gyoku (1.1.1) builder (>= 2.1.2) hashie (2.0.5) + httparty (0.12.0) + json (~> 1.8) + multi_xml (>= 0.5.2) httpclient (2.3.4.1) httpi (0.9.7) rack @@ -183,13 +188,14 @@ GEM mixlib-authentication (1.3.0) mixlib-log mixlib-log (1.6.0) - multi_json (1.8.2) + multi_json (1.8.4) + multi_xml (0.5.5) multipart-post (1.2.0) - net-http-persistent (2.9) + net-http-persistent (2.9.1) net-scp (1.1.2) net-ssh (>= 2.6.5) net-ssh (2.7.0) - nio4r (0.5.0) + nio4r (1.0.0) nokogiri (1.6.1) mini_portile (~> 0.5.0) nori (1.1.5) @@ -206,17 +212,17 @@ GEM puma (2.7.1) rack (>= 1.1, < 2.0) rack (1.5.2) - rack-protection (1.5.1) + rack-protection (1.5.2) rack rack-test (0.6.2) rack (>= 1.0) racksh (1.0.0) rack (>= 1.0) rack-test (>= 0.5) - rake (10.1.0) + rake (10.1.1) rest-client (1.6.7) mime-types (>= 1.16) - retryable (1.3.3) + retryable (1.3.4) rgl (0.4.0) rake stream (>= 0.5) @@ -255,7 +261,7 @@ GEM ruby2ruby (2.0.7) ruby_parser (~> 3.1) sexp_processor (~> 4.0) - ruby_parser (3.2.2) + ruby_parser (3.3.0) sexp_processor (~> 4.1) rubyntlm (0.1.1) savon (0.9.5) @@ -306,7 +312,7 @@ GEM json (>= 1.4.6) launchy (>= 2.0.0) uuidtools (2.1.4) - varia_model (0.2.0) + varia_model (0.3.2) buff-extensions (~> 0.2) hashie (>= 2.0.2) wasabi (1.0.0) diff --git a/Rakefile b/Rakefile index d587ab4..3be5dc8 100755 --- a/Rakefile +++ b/Rakefile @@ -45,7 +45,7 @@ begin require 'flog_cli' desc "Analyze total code complexity with flog" task :total do - threshold = 10 + threshold = 20 flog = FlogCLI.new flog.flog %w(app lib spec) average = flog.average.round(1) diff --git a/app/harp_user_api.rb b/app/harp_user_api.rb index 4c66705..90d3ce0 100644 --- a/app/harp_user_api.rb +++ b/app/harp_user_api.rb @@ -11,5 +11,9 @@ class HarpUserApiApp < Sinatra::Base get '/key/:key_name' do Key.get_by_name(params[:key_name]).to_json end + + get '/puppet-emc/:master_ip' do + PuppetENC.find(params[:master_ip]).first.attributes[:yaml] + end end diff --git a/harp-runtime.gemspec b/harp-runtime.gemspec index a194bf7..1f848a1 100644 --- a/harp-runtime.gemspec +++ b/harp-runtime.gemspec @@ -34,8 +34,10 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency "puma" gem.add_runtime_dependency "delayed_job" gem.add_runtime_dependency "delayed_job_data_mapper" + gem.add_runtime_dependency "faraday",["= 0.8.9"] gem.add_runtime_dependency "ridley" gem.add_runtime_dependency "ridley-connectors" + gem.add_runtime_dependency "httparty" gem.add_development_dependency "rspec" gem.add_development_dependency "shotgun" diff --git a/lib/harp-runtime/models/assembly.rb b/lib/harp-runtime/models/assembly.rb index a1a1abf..1cf75e2 100644 --- a/lib/harp-runtime/models/assembly.rb +++ b/lib/harp-runtime/models/assembly.rb @@ -6,10 +6,16 @@ class Assembly < HarpResource end class AssemblyChef < HarpResource + property :public_ip_address, String + property :private_ip_address, String end class AssemblyPuppet < HarpResource + property :public_ip_address, String + property :private_ip_address, String end class AssemblySalt < HarpResource + property :public_ip_address, String + property :private_ip_address, String end \ No newline at end of file diff --git a/lib/harp-runtime/models/user_data.rb b/lib/harp-runtime/models/user_data.rb index ea8677d..1461359 100644 --- a/lib/harp-runtime/models/user_data.rb +++ b/lib/harp-runtime/models/user_data.rb @@ -29,3 +29,9 @@ def self.get_by_name(key_name) Key.all(:name => key_name).first end end + +class PuppetENC + include DataMapper::Resource + property :master_ip, String, :key => true + property :yaml, Text +end diff --git a/lib/harp-runtime/resources/assembly/assembly.rb b/lib/harp-runtime/resources/assembly/assembly.rb index e59911e..5878603 100644 --- a/lib/harp-runtime/resources/assembly/assembly.rb +++ b/lib/harp-runtime/resources/assembly/assembly.rb @@ -40,13 +40,14 @@ def self.persistent_type() end def create(service) + @service = service provisioner = init_provisioner atts = self.attribs[:attributes] assembly = service.servers.create(atts[:server_options].symbolize_keys) assembly.wait_for { ready? } self.id = assembly.id provision_server(assembly.public_ip_address,provisioner) - return self + return assembly end def destroy(service) @@ -70,7 +71,7 @@ def get_output(service, persisted) response = service.get_console_output(persisted.id) if response.status == 200 output += "\nstdout:\n" - output += output + response.body['output'] + output += response.body['output'] if !response.body['output'].nil? # escape special characters to ensure valid JSON. output = output.gsub('"', '\\"') output = output.gsub("\r", '') diff --git a/lib/harp-runtime/resources/assembly/assembly_chef.rb b/lib/harp-runtime/resources/assembly/assembly_chef.rb index 519be93..66b7bc6 100644 --- a/lib/harp-runtime/resources/assembly/assembly_chef.rb +++ b/lib/harp-runtime/resources/assembly/assembly_chef.rb @@ -30,11 +30,14 @@ class AssemblyChef < Assembly attribute :config attribute :packages attribute :server_options + + attribute :private_ip_address, :aliases => 'privateIpAddress' + attribute :public_ip_address, :aliases => 'ipAddress' register_resource :assembly_chef, RESOURCES_ASSEMBLY # Only keeping a few properties, simplest define keeps. - @keeps = /^id$/ + @keeps = /^id$|^.*_ip_address/ def self.persistent_type() ::AssemblyChef diff --git a/lib/harp-runtime/resources/assembly/assembly_puppet.rb b/lib/harp-runtime/resources/assembly/assembly_puppet.rb index cf21eed..a5b5ef5 100644 --- a/lib/harp-runtime/resources/assembly/assembly_puppet.rb +++ b/lib/harp-runtime/resources/assembly/assembly_puppet.rb @@ -16,32 +16,108 @@ class AssemblyPuppet < Assembly attribute :live_resource attribute :state attribute :type - + attribute :name attribute :cloud attribute :configurations attribute :tool attribute :cloud_credential attribute :image - + + attribute :config attribute :packages attribute :server_options + + attribute :private_ip_address, :aliases => 'privateIpAddress' + attribute :public_ip_address, :aliases => 'ipAddress' register_resource :assembly_puppet, RESOURCES_ASSEMBLY # Only keeping a few properties, simplest define keeps. - @keeps = /^id$/ + @keeps = /^id$|^.*_ip_address/ def self.persistent_type() ::AssemblyPuppet end - - def provision_server(server_ip) + + def init_provisioner + bootstrap_file = File.open(File.expand_path("../puppet-bootstrap/ubuntu.sh", __FILE__)).read + user_data = "" + line_num=0 + bootstrap_file.each_line do |line| + if line_num == 0 + user_data << line + line_num += 1 + user_data << "puppet_master_ip=" + config["server_url"] + line_num += 1 + else + user_data << line + line_num += 1 + end + end + server_options["user_data"] = user_data + end + + def provision_server(server_ip,provisioner) + # internal_dns = @service.servers.get(id).private_dns_name +# PuppetENC.first_or_create(:master_ip=>internal_dns).update(:master_ip=>internal_dns,:yaml=>parse_packages) + server = @service.servers.find{|i| i.public_ip_address == config['server_url'] || i.dns_name == config['server_url']} + server.username = config["ssh"]["user"] + @ssh_key = Key.get_by_name(config['ssh']['keys'][0]).temp_file + server.private_key_path = @ssh_key.path + server.ssh(['echo "'+parse_packages+'" > /usr/local/bin/puppet_node_classifiers/'+@service.servers.get(id).private_dns_name.downcase])[0].stdout + @ssh_key.unlink + bootstrap_server + end + + def bootstrap_server + server = @service.servers.get(id) + server.username = config["ssh"]["user"] + @ssh_key = Key.get_by_name(config['ssh']['keys'][0]).temp_file + server.private_key_path = @ssh_key.path + @boot_counter = 0 + bootstrap(server) + @ssh_key.unlink end + def bootstrap(server) + @boot_counter += 1 + begin + puts "waiting 30 seconds for bootstrap..." + sleep(30) + puts server.ssh(["sudo apt-get update && apt-get -y upgrade"])[0].stdout + puts server.ssh(["sudo aptitude -y install puppet"])[0].stdout + puts server.ssh(["sudo sed -i /etc/default/puppet -e 's/START=no/START=yes/'"])[0].stdout + puts server.ssh(["sudo sed -i -e '/\[main\]/{:a;n;/^$/!ba;i\pluginsync=true' -e '}' /etc/puppet/puppet.conf"])[0].stdout + puts server.ssh(["sudo service puppet restart"])[0].stdout + rescue + raise 'Bootstrap timeout error' if @boot_counter > 15 + puts "retrying bootstrap: " + bootstrap(server) + end + end + def destroy_provisioner(private_dns_name) end + def parse_packages + classes_list = [] + packages.each { |p| classes_list << p['name']} + yaml_hash = {"classes"=>classes_list} + yaml_hash.to_yaml + end + + def get_provisioner_output(service, persisted) + server = service.servers.get(persisted.id) + server.username = config["ssh"]["user"] + @ssh_key = Key.get_by_name(config['ssh']['keys'][0]).temp_file + server.private_key_path = @ssh_key.path + output = "puppet: \n" + output += server.ssh(['sudo cat /var/log/syslog'])[0].stdout + @ssh_key.unlink + output + end + end end end diff --git a/lib/harp-runtime/resources/assembly/assembly_salt.rb b/lib/harp-runtime/resources/assembly/assembly_salt.rb index 878d061..b2d044a 100644 --- a/lib/harp-runtime/resources/assembly/assembly_salt.rb +++ b/lib/harp-runtime/resources/assembly/assembly_salt.rb @@ -26,17 +26,67 @@ class AssemblySalt < Assembly attribute :packages attribute :server_options + + attribute :private_ip_address, :aliases => 'privateIpAddress' + attribute :public_ip_address, :aliases => 'ipAddress' register_resource :assembly_salt, RESOURCES_ASSEMBLY # Only keeping a few properties, simplest define keeps. - @keeps = /^id$/ + @keeps = /^id$|^.*_ip_address/ def self.persistent_type() ::AssemblySalt end - def provision_server(server_ip) + def init_provisioner + bootstrap_file = File.open(File.expand_path("../salt-bootstrap/salt_bootstrap.sh", __FILE__)).read + user_data = "" + line_num=0 + bootstrap_file.each_line do |line| + if line_num == 0 + user_data << line + line_num += 1 + user_data << 'packages_yaml="' + parse_packages + '"' + line_num += 1 + else + user_data << line + line_num += 1 + end + end + server_options["user_data"] = user_data + end + + def parse_packages + yaml_hash = {} + packages.each do |p| + yaml_hash[p['name']] = {'pkg'=>['installed']} + end + yaml_hash.to_yaml + end + + def provision_server(server_ip,provisioner) + server = @service.servers.get(id) + server.username = config["ssh"]["user"] + @ssh_key = Key.get_by_name(config['ssh']['keys'][0]).temp_file + server.private_key_path = @ssh_key.path + @boot_counter = 0 + bootstrap(server) + @ssh_key.unlink + end + + def bootstrap(server) + @boot_counter += 1 + begin + puts "waiting 30 seconds for bootstrap..." + sleep(30) + puts server.ssh(["wget -O - http://bootstrap.saltstack.org | sudo sh"])[0].stdout + puts server.ssh(["sudo salt-call --local state.highstate -l debug"])[0].stdout + rescue + raise 'Bootstrap timeout error' if @boot_counter > 15 + puts "retrying bootstrap: " + bootstrap(server) + end end def destroy_provisioner(private_dns_name) diff --git a/lib/harp-runtime/resources/assembly/chef_bootstrap/bootstrap.rb b/lib/harp-runtime/resources/assembly/chef_bootstrap/bootstrap.rb deleted file mode 100644 index e174474..0000000 --- a/lib/harp-runtime/resources/assembly/chef_bootstrap/bootstrap.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'rubygems' -require "chef" -require "chef/knife/core/bootstrap_context" -require 'chef/knife' -require 'chef/knife/ssh' -require 'net/ssh' -require 'net/ssh/multi' -require 'chef/knife/bootstrap' - -Chef::Config.from_file(File.expand_path('')) -kb = Chef::Knife::Bootstrap.new -kb.name_args = "" -kb.config[:identity_file] = "" -kb.config[:ssh_user] = "" -kb.config[:ssh_password] = "" -kb.config[:run_list] = ["recipe[apt]","recipe[]"] -kb.config[:use_sudo] = true -kb.config[:use_sudo_password] = true -kb.config[:distro] = "chef-full" -kb.run \ No newline at end of file diff --git a/lib/harp-runtime/resources/assembly/chef_bootstrap/knife.rb b/lib/harp-runtime/resources/assembly/chef_bootstrap/knife.rb deleted file mode 100644 index f219257..0000000 --- a/lib/harp-runtime/resources/assembly/chef_bootstrap/knife.rb +++ /dev/null @@ -1,10 +0,0 @@ -log_level :info -log_location STDOUT -node_name '' -client_key '' -validation_client_name '' -validation_key '' -chef_server_url '' -cache_type 'BasicFile' -cache_options( :path => 'checksums' ) -cookbook_path [ 'cookbooks' ] \ No newline at end of file diff --git a/lib/harp-runtime/resources/assembly/puppet-bootstrap/puppet_node_classifier b/lib/harp-runtime/resources/assembly/puppet-bootstrap/puppet_node_classifier new file mode 100755 index 0000000..87a5fc1 --- /dev/null +++ b/lib/harp-runtime/resources/assembly/puppet-bootstrap/puppet_node_classifier @@ -0,0 +1,2 @@ +#!/bin/bash +cat /usr/local/bin/puppet_node_classifiers/$1 \ No newline at end of file diff --git a/lib/harp-runtime/resources/assembly/puppet-bootstrap/ubuntu.sh b/lib/harp-runtime/resources/assembly/puppet-bootstrap/ubuntu.sh new file mode 100755 index 0000000..c351c45 --- /dev/null +++ b/lib/harp-runtime/resources/assembly/puppet-bootstrap/ubuntu.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +#set -e -x + +# Needed so that the aptitude/apt-get operations will not be interactive +#export DEBIAN_FRONTEND=noninteractive + +#apt-get update && apt-get -y upgrade + +# Find the current IP of the puppet master and make "puppet" point to it +#puppet_master_ip=$(host my_puppet_master.company.com | grep "has address" | head -1 | awk '{print $NF}') +echo $puppet_master_ip puppet >> /etc/hosts + +#aptitude -y install puppet + +# Enable the puppet client +#sed -i /etc/default/puppet -e 's/START=no/START=yes/' + +#sed -i -e '/\[main\]/{:a;n;/^$/!ba;i\pluginsync=true' -e '}' /etc/puppet/puppet.conf + +#service puppet restart \ No newline at end of file diff --git a/lib/harp-runtime/resources/assembly/salt-bootstrap/salt_bootstrap.sh b/lib/harp-runtime/resources/assembly/salt-bootstrap/salt_bootstrap.sh new file mode 100755 index 0000000..796c792 --- /dev/null +++ b/lib/harp-runtime/resources/assembly/salt-bootstrap/salt_bootstrap.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +mkdir /srv/salt + +touch /srv/salt/top.sls + +touch /srv/salt/webserver.sls + +cat > /srv/salt/top.sls << EOF +base: + '*': + - webserver +EOF + +echo -e "$packages_yaml" >> /srv/salt/webserver.sls \ No newline at end of file diff --git a/sample/assemblies.harp b/sample/assemblies.harp new file mode 100644 index 0000000..60de10e --- /dev/null +++ b/sample/assemblies.harp @@ -0,0 +1,70 @@ +# Create some instances on AWS + +template = < "ami-d0f89fb9", @@ -13,7 +14,7 @@ assembly_chef_resource = { "type" => "Std::AssemblyChef", "server_options" => server_options, - + "name" => "ChefAssembly", "image" => "ami-d0f89fb9", "packages" => [ @@ -38,39 +39,65 @@ assembly_puppet_resource = { "type" => "Std::AssemblyPuppet", "server_options" => server_options, - + "name" => "PuppetAssembly", "image" => "ami-d0f89fb9", "packages" => [ - {"name" => "transcend_nexus","type" => "class"}, - {"name" => "transcend_sonar","type" => "class"} - ] + {"name" => "apache","type" => "class"}, + {"name" => "ntp","type" => "class"} + ], + "config" => { + "server_url" => "54.205.121.185", + "ssh" => { + "user" => "ubuntu", + "keys" => ["dev-client-ec2"] + } + } } assembly_salt_resource = { "type" => "Std::AssemblySalt", "server_options" => server_options, - + "name" => "SaltAssembly", "image" => "ami-d0f89fb9", "packages" => [ - {"name" => "activemq","type" => "recipe","version" => "1.0.0"}, - {"name" => "apparmor","type" => "recipe","version" => "0.9.0"}, - {"name" => "apt", "type" => "recipe","version" => "1.1.2"} - ] + {"name" => "apache2"} + ], + "config" => { + "ssh" => { + "user" => "ubuntu", + "keys" => ["dev-client-ec2"] + } + } } shared_context 'when have an assembly' do + let(:mutator_assembly) { + cnf = YAML::load_file(File.join(File.dirname(File.expand_path(__FILE__)), '../config/settings.yaml')) + FactoryGirl.create(:key, name: "dev-client-ec2", value: cnf['default_creds']['dev-client-ec2']) + FactoryGirl.create(:key, name: "harp-client", value: cnf['default_creds']['harp-client']) + FactoryGirl.create(:key, name: "momentumsidev-validator", value: cnf['default_creds']['momentumsidev-validator']) + PuppetENC.first_or_create(:master_ip=>"www.momentumsi.com").update(:yaml=>nil) + interpreter_context = {} + interpreter_context[:cloud_type] = :aws # for the moment, assume AWS cloud + interpreter_context[:debug] = true + interpreter_context[:access] = cnf['default_creds']['access'] + interpreter_context[:secret] = cnf['default_creds']['secret'] + interpreter_context[:harp_script] = FactoryGirl.create(:harp_script) + interpreter_context + Harp::Cloud::CloudMutator.new(interpreter_context) + } let(:assembly_chef) do - @new_assembly_chef = mutator.create("ChefAssembly", assembly_chef_resource) + @new_assembly_chef = mutator_assembly.create("ChefAssembly", assembly_chef_resource) @new_assembly_chef.instance_variable_get(:@id) end let(:assembly_puppet) do - @new_assembly_puppet = mutator.create("PuppetAssembly", assembly_puppet_resource) + @new_assembly_puppet = mutator_assembly.create("PuppetAssembly", assembly_puppet_resource) @new_assembly_puppet.instance_variable_get(:@id) end let(:assembly_salt) do - @new_assembly_salt = mutator.create("SaltAssembly", assembly_salt_resource) + @new_assembly_salt = mutator_assembly.create("SaltAssembly", assembly_salt_resource) @new_assembly_salt.instance_variable_get(:@id) end end @@ -82,15 +109,33 @@ pending "pending due to need of live instance" assembly_chef verify_created(@new_assembly_chef, "ChefAssembly", AssemblyChef) + expect(HTTParty.get("http://#{@new_assembly_chef.public_ip_address}").code).to eq(200) + mutator_assembly.destroy("ChefAssembly", assembly_chef_resource) end it "creates an assembly_puppet" do pending "pending due to need of live instance" assembly_puppet verify_created(@new_assembly_puppet, "PuppetAssembly", AssemblyPuppet) + begin + sleep(60) + expect(HTTParty.get("http://#{@new_assembly_puppet.public_ip_address}").code).to eq(200) + rescue + sleep(60) + expect(HTTParty.get("http://#{@new_assembly_puppet.public_ip_address}").code).to eq(200) + end + mutator_assembly.destroy("PuppetAssembly", assembly_puppet_resource) end it "creates an assembly_salt" do pending "pending due to need of live instance" assembly_salt verify_created(@new_assembly_salt, "SaltAssembly", AssemblySalt) + begin + sleep(60) + expect(HTTParty.get("http://#{@new_assembly_salt.public_ip_address}").code).to eq(200) + rescue + sleep(60) + expect(HTTParty.get("http://#{@new_assembly_salt.public_ip_address}").code).to eq(200) + end + mutator_assembly.destroy("SaltAssembly", assembly_salt_resource) end -end \ No newline at end of file +end diff --git a/spec/factories/harp_factories.rb b/spec/factories/harp_factories.rb index 61429f5..545b64b 100644 --- a/spec/factories/harp_factories.rb +++ b/spec/factories/harp_factories.rb @@ -14,4 +14,9 @@ state "starting" harp_script end + + factory :key do + name "key_resource" + value "valid_key" + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 8887d3e..dc1d744 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -22,7 +22,8 @@ require 'data_mapper' -DataMapper.setup(:default, "sqlite::memory:") +#DataMapper.setup(:default, "sqlite::memory:") +DataMapper.setup(:default, "sqlite3://#{Dir.pwd}/devspec.db") require 'harp-runtime/models/autoscale' require 'harp-runtime/models/base'