diff --git a/Gemfile.lock b/Gemfile.lock index 7c338cd..d49bcd8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,7 +8,9 @@ PATH dm-mysql-adapter dm-sqlite-adapter extlib + faraday (= 0.8.9) fog + httparty json logging net-ssh @@ -69,12 +71,12 @@ GEM dm-transactions (~> 1.2.0) dm-types (~> 1.2.0) dm-validations (~> 1.2.0) - debugger (1.6.3) + debugger (1.6.5) columnize (>= 0.3.1) debugger-linecache (~> 1.2.0) - debugger-ruby_core_source (~> 1.2.4) + debugger-ruby_core_source (~> 1.3.1) debugger-linecache (1.2.0) - debugger-ruby_core_source (1.2.4) + debugger-ruby_core_source (1.3.1) delayed_job (2.1.4) activesupport (~> 3.0) daemons @@ -127,16 +129,16 @@ GEM data_objects (= 0.10.13) do_sqlite3 (0.10.13) data_objects (= 0.10.13) - docile (1.1.1) + docile (1.1.2) erubis (2.7.0) - evalhook (0.5.7) + evalhook (0.5.8) partialruby (~> 0.3) sexp_processor (~> 4.0) - excon (0.28.0) + excon (0.31.0) extlib (0.9.16) factory_girl (4.3.0) activesupport (>= 3.0.0) - faraday (0.8.8) + faraday (0.8.9) multipart-post (~> 1.2.0) fastercsv (1.5.5) ffi (1.9.3) @@ -146,9 +148,9 @@ GEM flog (4.2.0) ruby_parser (~> 3.1, > 3.1.0) sexp_processor (~> 4.4) - fog (1.18.0) + fog (1.19.0) builder - excon (~> 0.28.0) + excon (~> 0.31.0) formatador (~> 0.2.0) mime-types multi_json (~> 1.0) @@ -160,9 +162,12 @@ GEM getsource (0.2.1) gssapi (1.0.3) ffi (>= 1.0.1) - gyoku (1.1.0) + gyoku (1.1.1) builder (>= 2.1.2) hashie (2.0.5) + httparty (0.12.0) + json (~> 1.8) + multi_xml (>= 0.5.2) httpclient (2.3.4.1) httpi (0.9.7) rack @@ -183,13 +188,14 @@ GEM mixlib-authentication (1.3.0) mixlib-log mixlib-log (1.6.0) - multi_json (1.8.2) + multi_json (1.8.4) + multi_xml (0.5.5) multipart-post (1.2.0) - net-http-persistent (2.9) + net-http-persistent (2.9.1) net-scp (1.1.2) net-ssh (>= 2.6.5) net-ssh (2.7.0) - nio4r (0.5.0) + nio4r (1.0.0) nokogiri (1.6.1) mini_portile (~> 0.5.0) nori (1.1.5) @@ -206,17 +212,17 @@ GEM puma (2.7.1) rack (>= 1.1, < 2.0) rack (1.5.2) - rack-protection (1.5.1) + rack-protection (1.5.2) rack rack-test (0.6.2) rack (>= 1.0) racksh (1.0.0) rack (>= 1.0) rack-test (>= 0.5) - rake (10.1.0) + rake (10.1.1) rest-client (1.6.7) mime-types (>= 1.16) - retryable (1.3.3) + retryable (1.3.4) rgl (0.4.0) rake stream (>= 0.5) @@ -255,7 +261,7 @@ GEM ruby2ruby (2.0.7) ruby_parser (~> 3.1) sexp_processor (~> 4.0) - ruby_parser (3.2.2) + ruby_parser (3.3.0) sexp_processor (~> 4.1) rubyntlm (0.1.1) savon (0.9.5) @@ -306,7 +312,7 @@ GEM json (>= 1.4.6) launchy (>= 2.0.0) uuidtools (2.1.4) - varia_model (0.2.0) + varia_model (0.3.2) buff-extensions (~> 0.2) hashie (>= 2.0.2) wasabi (1.0.0) diff --git a/Rakefile b/Rakefile index d587ab4..8e23ff1 100755 --- a/Rakefile +++ b/Rakefile @@ -35,6 +35,22 @@ task :jslint do end end +namespace :db do + require "data_mapper" + require 'harp-runtime/models/user_data' + DataMapper.setup(:default, ENV['DATABASE_URL'] || "sqlite3://#{Dir.pwd}/dev.db") + DataMapper.finalize + DataMapper.auto_upgrade! + + desc "Seed the database with configuration settings." + task :seed do + cnf = YAML::load_file(File.join(File.dirname(File.expand_path(__FILE__)), 'config/settings.yaml')) + raise "Default Access/Secret keys not configured." if cnf['default_creds']['access']== 12345 || cnf['default_creds']['secret']== 67890 + cnf['default_creds']['keys'].each{|k| Key.first_or_create(:name=>k['name']).update(:value=>k['private_key']) } + puts "Seeded Keys..." + end +end + begin namespace :analyzer do desc "run all code analyzing tools (flog)" @@ -45,7 +61,7 @@ begin require 'flog_cli' desc "Analyze total code complexity with flog" task :total do - threshold = 10 + threshold = 20 flog = FlogCLI.new flog.flog %w(app lib spec) average = flog.average.round(1) diff --git a/app/harp_user_api.rb b/app/harp_user_api.rb new file mode 100644 index 0000000..90d3ce0 --- /dev/null +++ b/app/harp_user_api.rb @@ -0,0 +1,19 @@ +require 'sinatra/base' +require 'logging' + +class HarpUserApiApp < Sinatra::Base + + post '/key' do + key = JSON.parse(request.body.read).symbolize_keys + Key.create(key).to_json + end + + get '/key/:key_name' do + Key.get_by_name(params[:key_name]).to_json + end + + get '/puppet-emc/:master_ip' do + PuppetENC.find(params[:master_ip]).first.attributes[:yaml] + end + +end diff --git a/config.ru b/config.ru index f9973f8..55d371d 100644 --- a/config.ru +++ b/config.ru @@ -21,6 +21,7 @@ require 'app/root' require 'app/api_base' require 'app/harp_api' require 'app/harp_debug_api' +require 'app/harp_user_api' require 'app/editor' require 'data_mapper' require 'delayed_job_data_mapper' @@ -69,6 +70,17 @@ map "/api/v1/harp" do run HarpApiApp end +# +# Harp User API +# +##~ a = sapi.apis.add +## +##~ a.set :path => "/user.{format}", :format => "json" +##~ a.description = "Harp User API" +map "/api/v1/user" do + run HarpUserApiApp +end + map "/edit" do run EditorApp end diff --git a/config/settings.yaml.sample b/config/settings.yaml.sample index 476fa39..bb89a31 100644 --- a/config/settings.yaml.sample +++ b/config/settings.yaml.sample @@ -8,3 +8,34 @@ default_creds: access: 12345 secret: 67890 + keys: + - + name: default + private_key: | + -----BEGIN RSA PRIVATE KEY----- + MIIEowIBAAKCAQEAyXrPvzeVEj796mWg5xV3yi39SAXBXPFH/03xQFe0v8egSdOi + +G0riGKkeFvhjqvRPvxsX0Q+JGdJaYZYkQiphKHxIC7avKgyNgFzLeHHhQvcs9PF + pc4MR0d1xyemwTxKmERb4zlITDCNT2acDAR4sHhx6lPwM31Ga9FJfZDpqpXZUGdQ + d6yt5+sIfQL421yx7CfC3+wLL1Flp+ISgzcn4waHQiQNxXY0O7LMWuEDwD699fvn + 6/EQjLteVQHlm9NPYBYmUYGks5XoWTcQe2xArjspluqhzgxNWdqP7dWrgpwgkGdz + /V102QNjwV2C9thZHqaDZm/PNQzC/23bJ5QNBQIDAQABAoIBAF7UXP+FK7bAy0YN + u2PIaO9SIgpKGy6hqd9yUgCUx1AUp07YO7Ztu7WfqAW2ST2bhQr+SAu21QDRVG5E + e18YEqGIjzM0mmN9TtS4bfvEbuIiKSjCnh5iEslhyv+HnCcLN5jfY9hY17x0ha4/ + f2bwm9WDeyptg3rYldKdWNeu4ywUMN2Q8z8ap6rl6u7AxhgOGmqe+bWAtiuT44+L + dvDz5GgJCVD6nSerO/AF5t8NglVGRV7hVjZH9LoHAnIO78nWlxI/s4928aWfqboe + PCQ2MQYFLa9xYODQSSwQ2UAYUdC5CtELPdYOxFmK+MVicX5WHf9Il/FYYtezjhFd + xODriykCgYEA+MIUW7ih5G7rvK7o1Fuvbpj4UBR3E+jVXE7oHl5/439BJQO62l0D + UWT3KVO//+Ua/Kmjsq6MKCtjD/59Kx5E7XGBBl9TFgEHT4sVLVeTFK4djDvspheV + 01DYaYiOWCoN6x1PJ8ghZeDM96HFNKPy+GkoFXzbMBLxR634vBruXZMCgYEAz1hh + UtwbaZHZSWjJaqdyBMrEQIN/n/sgXqLD41eZqRxNWSGL7IeZ4SKjvnlXNuxhbhi3 + kwqmoTjleRqJS1tCpvmhecjUojZ0gmHgb9ew9bioNx6dPnh9mP0XJyJ9Tg8VQjrT + QcG0FFhOtBeI2GrmmnGeChsKpxEOq2WLC8XRSgcCgYEA7E7qKgul9ywL0y1RTtJe + QDEeymv2hrpLXSo0RQ0+bi3oA7KcYW20hjDkvqSpovzQey7nrAk2ab51l68I1tol + Xm7Rw+zwL9oASOb10iH8Pcv9k8clUs7a7GhDfVVU65nfqPcpx7/NQkugSqmpfg00 + 9hhnFKqMSgnknVQ+Eut+fvMCgYAyumrFPPiv8XD51ttAbsskhP9S0VlOKMCYprXK + T0pW54e9AModCmgJftIU7db6pwaHSyDOptJxq1vpLybb+u3psDVjeFEdhrkmlQ+2 + DU1MMOtm4OSTviLsZS9e8b86c9I9q+q3sc5/E8Pdp8xhNnZI3vdNfaQzj6bJJETK + kpFLzQKBgFzSMD0t8Kbxb4O226HSgZevqNqXSuk/KbAsc3WB4xhCbCb8rd2v3q+6 + LMWSJN9ynkkrf1KUfR0MHh0rQLCI3GUig5+JRf9ljnpPhe14zn8h/2tX4pW5AWSF + jz3nB3v6Tj0KktAKoOAYHlezQMgkiZ57nZwDGRdFoj838Hm1hwHz + -----END RSA PRIVATE KEY----- diff --git a/harp-runtime.gemspec b/harp-runtime.gemspec index a194bf7..1f848a1 100644 --- a/harp-runtime.gemspec +++ b/harp-runtime.gemspec @@ -34,8 +34,10 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency "puma" gem.add_runtime_dependency "delayed_job" gem.add_runtime_dependency "delayed_job_data_mapper" + gem.add_runtime_dependency "faraday",["= 0.8.9"] gem.add_runtime_dependency "ridley" gem.add_runtime_dependency "ridley-connectors" + gem.add_runtime_dependency "httparty" gem.add_development_dependency "rspec" gem.add_development_dependency "shotgun" diff --git a/lib/harp-runtime/cloud/cloud_mutator.rb b/lib/harp-runtime/cloud/cloud_mutator.rb index ba0b975..5f9fab7 100644 --- a/lib/harp-runtime/cloud/cloud_mutator.rb +++ b/lib/harp-runtime/cloud/cloud_mutator.rb @@ -106,13 +106,14 @@ def get_harp_resource(resource_name) end def get_output(resource, persisted) - resource = Harp::Resources::AvailableResource.from_name resource['type'] - if resource.nil? - @@logger.error "No resource type #{resource_def['type']}" + res = Harp::Resources::AvailableResource.from_name resource['type'] + if res.nil? + @@logger.error "No resource type #{resource['type']}" return end - service = establish_connect(resource) - output = resource.get_output(service, persisted) + res.populate(resource) + service = establish_connect(res) + output = res.get_output(service, persisted) end def add_all(resources) diff --git a/lib/harp-runtime/models/assembly.rb b/lib/harp-runtime/models/assembly.rb index 5b09926..e4b9ad9 100644 --- a/lib/harp-runtime/models/assembly.rb +++ b/lib/harp-runtime/models/assembly.rb @@ -8,16 +8,22 @@ class Assembly < HarpResource # AssemblyChef is the datastore representation of a Harp Assembly with Chef. class AssemblyChef < HarpResource + property :public_ip_address, String + property :private_ip_address, String end # AssemblyPuppet is the datastore representation of an Assembly with Puppet. class AssemblyPuppet < HarpResource + property :public_ip_address, String + property :private_ip_address, String end # AssemblySalt is the datastore representation of an Assembly with Salt. class AssemblySalt < HarpResource + property :public_ip_address, String + property :private_ip_address, String end # AssemblyDocker is the datastore representation of an Assembly with Docker. class AssemblyDocker < HarpResource -end +end \ No newline at end of file diff --git a/lib/harp-runtime/models/user_data.rb b/lib/harp-runtime/models/user_data.rb new file mode 100644 index 0000000..1461359 --- /dev/null +++ b/lib/harp-runtime/models/user_data.rb @@ -0,0 +1,37 @@ +require 'harp-runtime/models/base' + +# Represents a harp user. +class HarpUser + include DataMapper::Resource + property :id, Serial, :key => true + + property :name, String + + has n, :harp_scripts, :through => Resource + has n, :keys, :through => Resource +end + +class Key + include DataMapper::Resource + property :id, Serial + property :name, String, :key => true + property :value, Text + + has n, :harp_users, :through => Resource + + def temp_file + t = Tempfile.new(name) + t << value + t.close + t + end + def self.get_by_name(key_name) + Key.all(:name => key_name).first + end +end + +class PuppetENC + include DataMapper::Resource + property :master_ip, String, :key => true + property :yaml, Text +end diff --git a/lib/harp-runtime/resources/assembly/assembly.rb b/lib/harp-runtime/resources/assembly/assembly.rb index 22e3c28..0c54665 100644 --- a/lib/harp-runtime/resources/assembly/assembly.rb +++ b/lib/harp-runtime/resources/assembly/assembly.rb @@ -1,6 +1,8 @@ require 'set' require 'fog/core/model' require 'harp-runtime/models/assembly' +require 'harp-runtime/models/user_data' +require 'harp-runtime/resources/compute/instance' require 'json' module Harp @@ -9,7 +11,7 @@ module Resources # An Assembly is an operating system image combined with some bootstrapped # configuration. For example, an Ubuntu image with a Chef client and an # initial Chef role can be saved as an assembly. - class Assembly < AvailableResource + class Assembly < ComputeInstance include Harp::Resources @@ -41,12 +43,14 @@ def self.persistent_type() end def create(service) + @service = service + provisioner = init_provisioner atts = self.attribs[:attributes] assembly = service.servers.create(atts[:server_options].symbolize_keys) assembly.wait_for { ready? } self.id = assembly.id - provision_server(assembly.public_ip_address) - return self + provision_server(assembly.public_ip_address,provisioner) + return assembly end def destroy(service) @@ -59,6 +63,27 @@ def destroy(service) return self end + # Return a token to signify output from the current action + def output_token(args={}) + return "#{name}:#{id}" + end + + def get_output(service, persisted) + output = "" + output = get_provisioner_output(service, persisted) + response = service.get_console_output(persisted.id) + if response.status == 200 + output += "\nstdout:\n" + output += response.body['output'] if !response.body['output'].nil? + # escape special characters to ensure valid JSON. + output = output.gsub('"', '\\"') + output = output.gsub("\r", '') + output = output.gsub("\n", '\\n') + output = output.gsub("\e", 'ESC') + end + output + end + end end end diff --git a/lib/harp-runtime/resources/assembly/assembly_chef.rb b/lib/harp-runtime/resources/assembly/assembly_chef.rb index c137444..66b7bc6 100644 --- a/lib/harp-runtime/resources/assembly/assembly_chef.rb +++ b/lib/harp-runtime/resources/assembly/assembly_chef.rb @@ -19,68 +19,82 @@ class AssemblyChef < Assembly attribute :live_resource attribute :state attribute :type - + attribute :name attribute :cloud attribute :configurations attribute :tool attribute :cloud_credential attribute :image - + attribute :config attribute :packages attribute :server_options + + attribute :private_ip_address, :aliases => 'privateIpAddress' + attribute :public_ip_address, :aliases => 'ipAddress' register_resource :assembly_chef, RESOURCES_ASSEMBLY # Only keeping a few properties, simplest define keeps. - @keeps = /^id$/ + @keeps = /^id$|^.*_ip_address/ def self.persistent_type() ::AssemblyChef end - - def provision_server(server_ip) - ridley = init_ridley(config) + + def provision_server(server_ip,ridley) + @boot_counter = 0 bootstrap_server(ridley,server_ip,parse_packages) end - + def bootstrap_server(ridley,server_ip,parse_packages) + @boot_counter += 1 begin puts "waiting 60 seconds for bootstrap: " + server_ip sleep(60) ridley.node.bootstrap(server_ip,run_list: parse_packages) + @client_key.unlink + @validator_key.unlink + @ssh_key.unlink rescue + raise 'Bootstrap timeout error' if @boot_counter > 5 puts "retrying bootstrap: " + server_ip bootstrap_server(ridley,server_ip,parse_packages) end end - + def destroy_provisioner(private_dns_name) - ridley = init_ridley(config) + ridley = init_provisioner ridley.node.delete(private_dns_name) ridley.client.delete(private_dns_name) + @client_key.unlink + @validator_key.unlink + @ssh_key.unlink end - + def parse_packages run_list = [] packages.each { |p| run_list << p['type'] + "[" + p['name'] +"]"} run_list end - - def init_ridley(config) + + def init_provisioner defaults = {"winrm" => {"port" => 5985},"ssh" => {"port" => 22}} defaults.merge!(config) + @client_key = Key.get_by_name(defaults['client_key']).temp_file + @validator_key = Key.get_by_name(defaults['validator_path']).temp_file + @ssh_key = Key.get_by_name(defaults['ssh']['keys'][0]).temp_file ridley = Ridley.new( server_url: defaults['server_url'], client_name: defaults['client_name'], - client_key: defaults['client_key'], + client_key: @client_key.path, validator_client: defaults['validator_client'], - validator_path: defaults['validator_path'], + validator_path: @validator_key.path, ssh: { user: defaults['ssh']['user'], password: defaults['ssh']['password'], - keys: defaults['ssh']['keys'], + keys: [@ssh_key.path], port: defaults['ssh']['port'], sudo: defaults['ssh']['sudo'] }, @@ -93,6 +107,20 @@ def init_ridley(config) return ridley end + def get_provisioner_output(service, persisted) + server = service.servers.get(persisted.id) + server.username = config["ssh"]["user"] + @ssh_key = Key.get_by_name(config['ssh']['keys'][0]).temp_file + server.private_key_path = @ssh_key.path + output = "chef-client: \n" + output += "/etc/chef/client.rb: \n" + output += server.ssh(['cat /etc/chef/client.rb'])[0].stdout + output += "\n/etc/chef/first-boot.json: \n" + output += server.ssh(['cat /etc/chef/first-boot.json'])[0].stdout + @ssh_key.unlink + output + end + end end end diff --git a/lib/harp-runtime/resources/assembly/assembly_puppet.rb b/lib/harp-runtime/resources/assembly/assembly_puppet.rb index cf21eed..a5b5ef5 100644 --- a/lib/harp-runtime/resources/assembly/assembly_puppet.rb +++ b/lib/harp-runtime/resources/assembly/assembly_puppet.rb @@ -16,32 +16,108 @@ class AssemblyPuppet < Assembly attribute :live_resource attribute :state attribute :type - + attribute :name attribute :cloud attribute :configurations attribute :tool attribute :cloud_credential attribute :image - + + attribute :config attribute :packages attribute :server_options + + attribute :private_ip_address, :aliases => 'privateIpAddress' + attribute :public_ip_address, :aliases => 'ipAddress' register_resource :assembly_puppet, RESOURCES_ASSEMBLY # Only keeping a few properties, simplest define keeps. - @keeps = /^id$/ + @keeps = /^id$|^.*_ip_address/ def self.persistent_type() ::AssemblyPuppet end - - def provision_server(server_ip) + + def init_provisioner + bootstrap_file = File.open(File.expand_path("../puppet-bootstrap/ubuntu.sh", __FILE__)).read + user_data = "" + line_num=0 + bootstrap_file.each_line do |line| + if line_num == 0 + user_data << line + line_num += 1 + user_data << "puppet_master_ip=" + config["server_url"] + line_num += 1 + else + user_data << line + line_num += 1 + end + end + server_options["user_data"] = user_data + end + + def provision_server(server_ip,provisioner) + # internal_dns = @service.servers.get(id).private_dns_name +# PuppetENC.first_or_create(:master_ip=>internal_dns).update(:master_ip=>internal_dns,:yaml=>parse_packages) + server = @service.servers.find{|i| i.public_ip_address == config['server_url'] || i.dns_name == config['server_url']} + server.username = config["ssh"]["user"] + @ssh_key = Key.get_by_name(config['ssh']['keys'][0]).temp_file + server.private_key_path = @ssh_key.path + server.ssh(['echo "'+parse_packages+'" > /usr/local/bin/puppet_node_classifiers/'+@service.servers.get(id).private_dns_name.downcase])[0].stdout + @ssh_key.unlink + bootstrap_server + end + + def bootstrap_server + server = @service.servers.get(id) + server.username = config["ssh"]["user"] + @ssh_key = Key.get_by_name(config['ssh']['keys'][0]).temp_file + server.private_key_path = @ssh_key.path + @boot_counter = 0 + bootstrap(server) + @ssh_key.unlink end + def bootstrap(server) + @boot_counter += 1 + begin + puts "waiting 30 seconds for bootstrap..." + sleep(30) + puts server.ssh(["sudo apt-get update && apt-get -y upgrade"])[0].stdout + puts server.ssh(["sudo aptitude -y install puppet"])[0].stdout + puts server.ssh(["sudo sed -i /etc/default/puppet -e 's/START=no/START=yes/'"])[0].stdout + puts server.ssh(["sudo sed -i -e '/\[main\]/{:a;n;/^$/!ba;i\pluginsync=true' -e '}' /etc/puppet/puppet.conf"])[0].stdout + puts server.ssh(["sudo service puppet restart"])[0].stdout + rescue + raise 'Bootstrap timeout error' if @boot_counter > 15 + puts "retrying bootstrap: " + bootstrap(server) + end + end + def destroy_provisioner(private_dns_name) end + def parse_packages + classes_list = [] + packages.each { |p| classes_list << p['name']} + yaml_hash = {"classes"=>classes_list} + yaml_hash.to_yaml + end + + def get_provisioner_output(service, persisted) + server = service.servers.get(persisted.id) + server.username = config["ssh"]["user"] + @ssh_key = Key.get_by_name(config['ssh']['keys'][0]).temp_file + server.private_key_path = @ssh_key.path + output = "puppet: \n" + output += server.ssh(['sudo cat /var/log/syslog'])[0].stdout + @ssh_key.unlink + output + end + end end end diff --git a/lib/harp-runtime/resources/assembly/assembly_salt.rb b/lib/harp-runtime/resources/assembly/assembly_salt.rb index 878d061..b2d044a 100644 --- a/lib/harp-runtime/resources/assembly/assembly_salt.rb +++ b/lib/harp-runtime/resources/assembly/assembly_salt.rb @@ -26,17 +26,67 @@ class AssemblySalt < Assembly attribute :packages attribute :server_options + + attribute :private_ip_address, :aliases => 'privateIpAddress' + attribute :public_ip_address, :aliases => 'ipAddress' register_resource :assembly_salt, RESOURCES_ASSEMBLY # Only keeping a few properties, simplest define keeps. - @keeps = /^id$/ + @keeps = /^id$|^.*_ip_address/ def self.persistent_type() ::AssemblySalt end - def provision_server(server_ip) + def init_provisioner + bootstrap_file = File.open(File.expand_path("../salt-bootstrap/salt_bootstrap.sh", __FILE__)).read + user_data = "" + line_num=0 + bootstrap_file.each_line do |line| + if line_num == 0 + user_data << line + line_num += 1 + user_data << 'packages_yaml="' + parse_packages + '"' + line_num += 1 + else + user_data << line + line_num += 1 + end + end + server_options["user_data"] = user_data + end + + def parse_packages + yaml_hash = {} + packages.each do |p| + yaml_hash[p['name']] = {'pkg'=>['installed']} + end + yaml_hash.to_yaml + end + + def provision_server(server_ip,provisioner) + server = @service.servers.get(id) + server.username = config["ssh"]["user"] + @ssh_key = Key.get_by_name(config['ssh']['keys'][0]).temp_file + server.private_key_path = @ssh_key.path + @boot_counter = 0 + bootstrap(server) + @ssh_key.unlink + end + + def bootstrap(server) + @boot_counter += 1 + begin + puts "waiting 30 seconds for bootstrap..." + sleep(30) + puts server.ssh(["wget -O - http://bootstrap.saltstack.org | sudo sh"])[0].stdout + puts server.ssh(["sudo salt-call --local state.highstate -l debug"])[0].stdout + rescue + raise 'Bootstrap timeout error' if @boot_counter > 15 + puts "retrying bootstrap: " + bootstrap(server) + end end def destroy_provisioner(private_dns_name) diff --git a/lib/harp-runtime/resources/assembly/chef_bootstrap/bootstrap.rb b/lib/harp-runtime/resources/assembly/chef_bootstrap/bootstrap.rb deleted file mode 100644 index e174474..0000000 --- a/lib/harp-runtime/resources/assembly/chef_bootstrap/bootstrap.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'rubygems' -require "chef" -require "chef/knife/core/bootstrap_context" -require 'chef/knife' -require 'chef/knife/ssh' -require 'net/ssh' -require 'net/ssh/multi' -require 'chef/knife/bootstrap' - -Chef::Config.from_file(File.expand_path('')) -kb = Chef::Knife::Bootstrap.new -kb.name_args = "" -kb.config[:identity_file] = "" -kb.config[:ssh_user] = "" -kb.config[:ssh_password] = "" -kb.config[:run_list] = ["recipe[apt]","recipe[]"] -kb.config[:use_sudo] = true -kb.config[:use_sudo_password] = true -kb.config[:distro] = "chef-full" -kb.run \ No newline at end of file diff --git a/lib/harp-runtime/resources/assembly/chef_bootstrap/knife.rb b/lib/harp-runtime/resources/assembly/chef_bootstrap/knife.rb deleted file mode 100644 index f219257..0000000 --- a/lib/harp-runtime/resources/assembly/chef_bootstrap/knife.rb +++ /dev/null @@ -1,10 +0,0 @@ -log_level :info -log_location STDOUT -node_name '' -client_key '' -validation_client_name '' -validation_key '' -chef_server_url '' -cache_type 'BasicFile' -cache_options( :path => 'checksums' ) -cookbook_path [ 'cookbooks' ] \ No newline at end of file diff --git a/lib/harp-runtime/resources/assembly/puppet-bootstrap/puppet_node_classifier b/lib/harp-runtime/resources/assembly/puppet-bootstrap/puppet_node_classifier new file mode 100755 index 0000000..87a5fc1 --- /dev/null +++ b/lib/harp-runtime/resources/assembly/puppet-bootstrap/puppet_node_classifier @@ -0,0 +1,2 @@ +#!/bin/bash +cat /usr/local/bin/puppet_node_classifiers/$1 \ No newline at end of file diff --git a/lib/harp-runtime/resources/assembly/puppet-bootstrap/ubuntu.sh b/lib/harp-runtime/resources/assembly/puppet-bootstrap/ubuntu.sh new file mode 100755 index 0000000..c351c45 --- /dev/null +++ b/lib/harp-runtime/resources/assembly/puppet-bootstrap/ubuntu.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +#set -e -x + +# Needed so that the aptitude/apt-get operations will not be interactive +#export DEBIAN_FRONTEND=noninteractive + +#apt-get update && apt-get -y upgrade + +# Find the current IP of the puppet master and make "puppet" point to it +#puppet_master_ip=$(host my_puppet_master.company.com | grep "has address" | head -1 | awk '{print $NF}') +echo $puppet_master_ip puppet >> /etc/hosts + +#aptitude -y install puppet + +# Enable the puppet client +#sed -i /etc/default/puppet -e 's/START=no/START=yes/' + +#sed -i -e '/\[main\]/{:a;n;/^$/!ba;i\pluginsync=true' -e '}' /etc/puppet/puppet.conf + +#service puppet restart \ No newline at end of file diff --git a/lib/harp-runtime/resources/assembly/salt-bootstrap/salt_bootstrap.sh b/lib/harp-runtime/resources/assembly/salt-bootstrap/salt_bootstrap.sh new file mode 100755 index 0000000..796c792 --- /dev/null +++ b/lib/harp-runtime/resources/assembly/salt-bootstrap/salt_bootstrap.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +mkdir /srv/salt + +touch /srv/salt/top.sls + +touch /srv/salt/webserver.sls + +cat > /srv/salt/top.sls << EOF +base: + '*': + - webserver +EOF + +echo -e "$packages_yaml" >> /srv/salt/webserver.sls \ No newline at end of file diff --git a/public/js/views/debugger.js b/public/js/views/debugger.js index 15c4c52..bd62685 100644 --- a/public/js/views/debugger.js +++ b/public/js/views/debugger.js @@ -102,6 +102,8 @@ define([ if (!this.spinner) { spin_target = document.getElementById("working"); this.spinner = new Spinner(this.spinner_opts).spin(spin_target); + }else{ + this.spinner.spin(document.getElementById("working")); } $.ajax({ diff --git a/sample/assemblies.harp b/sample/assemblies.harp new file mode 100644 index 0000000..60de10e --- /dev/null +++ b/sample/assemblies.harp @@ -0,0 +1,70 @@ +# Create some instances on AWS + +template = < "ami-d0f89fb9", @@ -13,7 +14,7 @@ assembly_chef_resource = { "type" => "Std::AssemblyChef", "server_options" => server_options, - + "name" => "ChefAssembly", "image" => "ami-d0f89fb9", "packages" => [ @@ -23,12 +24,12 @@ "config" => { "server_url" => "https://api.opscode.com/organizations/momentumsidev", "client_name" => "harp-client", - "client_key" => "~/chef_keys/harp-client.pem", + "client_key" => "harp-client", "validator_client" => "momentumsidev-validator", - "validator_path" => "~/chef_keys/momentumsidev-validator.pem", + "validator_path" => "momentumsidev-validator", "ssh" => { "user" => "ubuntu", - "keys" => ["~/chef_keys/dev-client-ec2.pem"], + "keys" => ["dev-client-ec2"], "port" => 22, "sudo" => true } @@ -38,39 +39,68 @@ assembly_puppet_resource = { "type" => "Std::AssemblyPuppet", "server_options" => server_options, - + "name" => "PuppetAssembly", "image" => "ami-d0f89fb9", "packages" => [ - {"name" => "transcend_nexus","type" => "class"}, - {"name" => "transcend_sonar","type" => "class"} - ] + {"name" => "apache","type" => "class"}, + {"name" => "ntp","type" => "class"} + ], + "config" => { + "server_url" => "54.205.121.185", + "ssh" => { + "user" => "ubuntu", + "keys" => ["dev-client-ec2"] + } + } } assembly_salt_resource = { "type" => "Std::AssemblySalt", "server_options" => server_options, - + "name" => "SaltAssembly", "image" => "ami-d0f89fb9", "packages" => [ - {"name" => "activemq","type" => "recipe","version" => "1.0.0"}, - {"name" => "apparmor","type" => "recipe","version" => "0.9.0"}, - {"name" => "apt", "type" => "recipe","version" => "1.1.2"} - ] + {"name" => "apache2"} + ], + "config" => { + "ssh" => { + "user" => "ubuntu", + "keys" => ["dev-client-ec2"] + } + } } shared_context 'when have an assembly' do + let(:mutator_assembly) { + cnf = YAML::load_file(File.join(File.dirname(File.expand_path(__FILE__)), '../config/settings.yaml')) + cnf['dev-client-ec2'] = cnf['default_creds']['keys'].select {|k| k['name'] == 'dev-client-ec2'}[0]['private_key'] + cnf['harp-client'] = cnf['default_creds']['keys'].select {|k| k['name'] == 'harp-client'}[0]['private_key'] + cnf['momentumsidev-validator'] = cnf['default_creds']['keys'].select {|k| k['name'] == 'momentumsidev-validator'}[0]['private_key'] + FactoryGirl.create(:key, name: "dev-client-ec2", value: cnf['dev-client-ec2']) + FactoryGirl.create(:key, name: "harp-client", value: cnf['harp-client']) + FactoryGirl.create(:key, name: "momentumsidev-validator", value: cnf['momentumsidev-validator']) + PuppetENC.first_or_create(:master_ip=>"www.momentumsi.com").update(:yaml=>nil) + interpreter_context = {} + interpreter_context[:cloud_type] = :aws # for the moment, assume AWS cloud + interpreter_context[:debug] = true + interpreter_context[:access] = cnf['default_creds']['access'] + interpreter_context[:secret] = cnf['default_creds']['secret'] + interpreter_context[:harp_script] = FactoryGirl.create(:harp_script) + interpreter_context + Harp::Cloud::CloudMutator.new(interpreter_context) + } let(:assembly_chef) do - @new_assembly_chef = mutator.create("ChefAssembly", assembly_chef_resource) + @new_assembly_chef = mutator_assembly.create("ChefAssembly", assembly_chef_resource) @new_assembly_chef.instance_variable_get(:@id) end let(:assembly_puppet) do - @new_assembly_puppet = mutator.create("PuppetAssembly", assembly_puppet_resource) + @new_assembly_puppet = mutator_assembly.create("PuppetAssembly", assembly_puppet_resource) @new_assembly_puppet.instance_variable_get(:@id) end let(:assembly_salt) do - @new_assembly_salt = mutator.create("SaltAssembly", assembly_salt_resource) + @new_assembly_salt = mutator_assembly.create("SaltAssembly", assembly_salt_resource) @new_assembly_salt.instance_variable_get(:@id) end end @@ -82,15 +112,33 @@ pending "pending due to need of live instance" assembly_chef verify_created(@new_assembly_chef, "ChefAssembly", AssemblyChef) + expect(HTTParty.get("http://#{@new_assembly_chef.public_ip_address}").code).to eq(200) + mutator_assembly.destroy("ChefAssembly", assembly_chef_resource) end it "creates an assembly_puppet" do pending "pending due to need of live instance" assembly_puppet verify_created(@new_assembly_puppet, "PuppetAssembly", AssemblyPuppet) + begin + sleep(60) + expect(HTTParty.get("http://#{@new_assembly_puppet.public_ip_address}").code).to eq(200) + rescue + sleep(60) + expect(HTTParty.get("http://#{@new_assembly_puppet.public_ip_address}").code).to eq(200) + end + mutator_assembly.destroy("PuppetAssembly", assembly_puppet_resource) end it "creates an assembly_salt" do pending "pending due to need of live instance" assembly_salt verify_created(@new_assembly_salt, "SaltAssembly", AssemblySalt) + begin + sleep(60) + expect(HTTParty.get("http://#{@new_assembly_salt.public_ip_address}").code).to eq(200) + rescue + sleep(60) + expect(HTTParty.get("http://#{@new_assembly_salt.public_ip_address}").code).to eq(200) + end + mutator_assembly.destroy("SaltAssembly", assembly_salt_resource) end -end \ No newline at end of file +end diff --git a/spec/factories/harp_factories.rb b/spec/factories/harp_factories.rb index 61429f5..545b64b 100644 --- a/spec/factories/harp_factories.rb +++ b/spec/factories/harp_factories.rb @@ -14,4 +14,9 @@ state "starting" harp_script end + + factory :key do + name "key_resource" + value "valid_key" + end end diff --git a/spec/harp_user_cloud_spec.rb b/spec/harp_user_cloud_spec.rb new file mode 100644 index 0000000..01797b6 --- /dev/null +++ b/spec/harp_user_cloud_spec.rb @@ -0,0 +1,58 @@ +require File.dirname(__FILE__) + '/spec_helper' +require "rubygems" +require "harp_runtime" + +shared_context 'when have a harp user' do + let(:harp_user) do + @new_harp_user = HarpUser.create(:name => 'TestUser1') + @new_harp_user.harp_scripts << harp_script + @new_harp_user.keys << key + @new_harp_user.save + @new_harp_user.instance_variable_get(:@id) + end + let(:harp_script) { @new_harp_script = harp_resource.harp_script } + + let(:key) { @new_key = Key.create(:name=>'testKey',:value => '0testKeyValue0') } + + let(:harp_resource) { FactoryGirl.create(:harp_resource) } + + let(:found_script) { HarpScript.get(harp_script.id) } + + let(:found_user) { @found_user = HarpUser.get(harp_user) } + + let(:found_key) { + @found_key = Key.get_by_name(key.name) + temp_file = @found_key.temp_file + @found_value = temp_file.open.read + temp_file.close + temp_file.unlink + @found_key + } + + let(:harp_resource2) do + harp_resource2 = FactoryGirl.build(:harp_resource) + end +end + +describe HarpUser, "#create" do + include_context "when have a harp user" + it "creates a harp user" do + harp_user + expect(HarpUser.entries.count).to eq(1) + expect(@new_harp_user.name).to eq('TestUser1') + end + it "associates a harp script with a harp user" do + found_user + expect(@found_user.harp_scripts.entries.count).to eq(1) + expect(@found_user.harp_scripts.first).to eq(@new_harp_script) + end + it "associates a key with a harp user" do + found_user + expect(@found_user.keys.entries.count).to eq(1) + expect(@found_user.keys.first).to eq(@new_key) + end + it "retrieves a key based on name" do + found_key + expect(@found_value).to eq(@new_key.value) + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index afb2093..dc1d744 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -22,10 +22,12 @@ require 'data_mapper' -DataMapper.setup(:default, "sqlite::memory:") +#DataMapper.setup(:default, "sqlite::memory:") +DataMapper.setup(:default, "sqlite3://#{Dir.pwd}/devspec.db") require 'harp-runtime/models/autoscale' require 'harp-runtime/models/base' +require 'harp-runtime/models/user_data' require 'harp-runtime/models/compute' require 'harp-runtime/models/rds' require 'harp-runtime/models/elastic_load_balancing'