forked from whopper/puppetstein
-
Notifications
You must be signed in to change notification settings - Fork 5
/
puppetstein.rb
executable file
·304 lines (265 loc) · 10.9 KB
/
puppetstein.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
#! /usr/bin/env ruby
require 'cri'
require 'git'
require 'yaml'
require 'json'
require_relative 'lib/host'
require_relative 'lib/util/common_utils.rb'
require_relative 'lib/util/platform_utils.rb'
require_relative 'lib/util/git_utils.rb'
require_relative 'lib/util/log_utils.rb'
include Puppetstein
include Puppetstein::PlatformUtils
include Puppetstein::BeakerUtils
include Puppetstein::GitUtils
include Puppetstein::LogUtils
include Beaker::DSL::InstallUtils::FOSSUtils
command = Cri::Command.define do
name 'puppetstein'
usage 'puppetstein [options] [arguments]
Example: puppetstein --puppet=whopper:my_branch --tests=facter:tests/facts/el.rb --platform=centos-7-x86_64'
summary 'Standalone puppet-agent composing and testing tool'
description 'A tool to automate the building and composition of various versions of
puppet-agent components for development and testing'
flag :h, :help, 'show help for this command' do |value, cmd|
puts cmd.help
exit 0
end
# TODO: allow tests from multiple projects to be run
# TODO; allow testing multiple agent platforms at once
# TODO: glob tests and test libs together for uber test
option nil, :puppet_agent, 'specify base puppet-agent version', argument: :optional
option :p, :platform, 'which platform to install on', argument: :optional
flag :b, :build, 'build mode: force puppetstein to build a new PA', argument: :optional
option nil, :package, 'path to a puppet-agent package to install', argument: :optional
option nil, :puppet, 'separated with a :', argument: :optional
option nil, :facter, 'separated with a :', argument: :optional
option nil, :hiera, 'separated with a :', argument: :optional
option nil, :agent, 'use a pre-provisioned agent. Useful for re-running tests. Requires --master option as well', argument: :optional
option nil, :master, 'use a pre-provisioned master. Useful for re-running tests. Requires --agent option as well', argument: :optional
flag nil, :use_last, 'use hosts from the last run', argument: :optional
option :t, :tests, 'tests to run against a puppet-agent installation', argument: :optional
option nil, :acceptancedir, 'colon separated list of directories where tests and test libraries can be found', argument: :optional
option :k, :keyfile, 'keyfile to use with vmpooler', argument: :optional
flag :nil, :noop, 'noop mode - output beaker command to run but don\'t execute', argument: :optional
run do |opts, args, cmd|
if opts[:platform]
agent = validate_platform(opts.fetch(:platform))
master = 'redhat7-64'
else
agent = 'ubuntu1604-64'
master = 'redhat7-64'
end
keyfile = opts[:keyfile] ? opts.fetch(:keyfile) : nil
build_mode = opts.fetch(:build) if opts[:build]
use_last = opts.fetch(:use_last) if opts[:use_last]
package = opts.fetch(:package) if opts[:package]
tests = opts.fetch(:tests) if opts[:tests]
acceptancedir = opts.fetch(:acceptancedir) if opts[:acceptancedir]
tmp = tmpdir
config = "#{tmp}/hosts.yaml"
# Check for conflicting options
if (use_last || opts[:agent]) && (opts[:puppet_agent] || opts[:puppet] || opts[:hiera] || opts[:facter])
log_notice('ERROR: using preprovisioned system - ignoring request for modified components')
exit 1
end
###
# Get puppet_agent version info
###
if opts[:puppet_agent]
pa_version = parse_project_version(opts.fetch(:puppet_agent))
# If a fork has been specified, trigger a rebuild
if pa_version[:fork] != 'puppetlabs'
log_notice("Puppet Agent fork specified, triggering a rebuild.")
build_mode = true
end
else
pa_version = Hash.new
pa_version[:fork] = 'puppetlabs'
if opts[:noop]
# don't curl the URL when in noop mode
pa_version[:sha] = 'latest'
else
pa_version[:sha] = opts[:build] ? 'master' : `curl http://builds.puppetlabs.lan/passing-agent-SHAs/puppet-agent-master`
end
end
ENV['PA_SHA'] = pa_version[:sha]
ENV['PA_SUITE'] = opts.fetch(:puppet_agent_suite_version) if opts[:puppet_agent_suite_version]
log_notice("Using puppet-agent base version #{pa_version[:sha]}")
###
# Setup tests: clone the proper repo(s) at the proper SHAs
###
if tests
project, test = tests.split(':')
if acceptancedir
ENV['RUBYLIB'] = "#{acceptancedir}/lib"
test_location = "#{acceptancedir}/#{test}"
log_notice("Using acceptancedir #{acceptancedir}/lib and test location #{acceptancedir}/#{test}")
else
if opts[:"#{project}"]
# A topic branch may contain new acceptance tests, so clone it for tests.
if pr = /pr_(\d+)/.match(opts[:"#{project}"])
# This is a pull request number. Get the fork and branch
v = parse_project_version(get_ref_from_pull_request(p, pr[1]))
else
v = parse_project_version(opts[:"#{project}"])
end
else
v = Hash.new
v[:fork] = 'puppetlabs'
v[:sha] = 'master'
end
log_notice("Cloning tests: #{project}: #{v[:fork]}:#{v[:sha]}")
clone_repo(project, v[:fork], v[:sha], tmp) if !opts[:noop]
ENV['RUBYLIB'] = "#{tmp}/#{project}/acceptance/lib"
test_location = "#{tmp}/#{project}/acceptance/#{test}"
log_notice("Using acceptancedir #{tmp}/#{project}/acceptance/lib and test location #{tmp}/#{project}/acceptance/#{test}")
end
end
###
# use_last mode: use the last host config we have with the given tests
###
if use_last
log_notice("Using last pre-provisioned hosts...")
options = {'hosts' => 'log/latest/hosts_preserved.yml'}
options['tests'] = test_location if tests
options['keyfile'] = keyfile if keyfile
options['noop'] = opts[:noop]
run_beaker(options)
if !opts[:noop]
log = get_latest_host_config
print_report({:agent => log[:HOSTS].keys[0], :master => log[:HOSTS].keys[1], :puppet_agent => "#{pa_version[:fork]}:#{pa_version[:sha]}"})
end
exit 0
end
###
# build_mode: build a puppet_agent package with given component SHAs
# Used automatically if a Facter version is specified, since we can't hot-swap Facter
###
if build_mode || opts[:facter]
clone_repo('puppet-agent', pa_version[:fork], pa_version[:sha], tmp) if !opts[:noop]
create_host_config([agent, master], config)
##
# Update the PA components with specified versions
['puppet', 'facter', 'hiera'].each do |p|
if opts[:"#{p}"]
if pr = /pr_(\d+)/.match(opts[:"#{p}"])
# This is a pull request number. Get the fork and branch
v = parse_project_version(get_ref_from_pull_request(p, pr[1]))
else
v = parse_project_version(opts[:"#{p}"])
end
change_component_ref(p, "git://github.com/#{v[:fork]}/#{p}.git", v[:sha], tmp, opts[:noop])
end
end
if !opts[:noop]
build_puppet_agent(agent, keyfile, tmp)
package = save_puppet_agent_artifact(agent, tmp)
end
ENV['PACKAGE'] = package
log_notice("Using newly built package #{package}")
pre_suites = ['lib/setup/build/pre-suite', 'lib/setup/common/pre-suite']
options = {'hosts' => config, 'pre-suite' => pre_suites.join(',')}
options['tests'] = test_location if tests
options['keyfile'] = keyfile if keyfile
options['noop'] = opts[:noop]
run_beaker(options)
if !opts[:noop]
log = get_latest_host_config
print_report({:agent => log[:HOSTS].keys[0], :master => log[:HOSTS].keys[1], :puppet_agent => "#{pa_version[:fork]}:#{pa_version[:sha]}"})
end
exit 0
end
###
# package mode: use an existing package on the local filesystem
###
if package
create_host_config([agent, master], config)
ENV['PACKAGE'] = package
log_notice("Using prebuilt package #{package}")
pre_suites = ['lib/setup/build/pre-suite', 'lib/setup/common/pre-suite']
options = {'hosts' => config, 'pre-suite' => pre_suites.join(',')}
options['tests'] = test_location if tests
options['keyfile'] = keyfile if keyfile
options['noop'] = opts[:noop]
run_beaker(options)
if !opts[:noop]
log = get_latest_host_config
print_report({:agent => log[:HOSTS].keys[0], :master => log[:HOSTS].keys[1], :puppet_agent => "#{pa_version[:fork]}:#{pa_version[:sha]}"})
end
exit 0
end
#
###
# Patch mode: If no other mode was specifically requested, try to patch a component
###
patchable_projects = ['puppet', 'hiera']
patchable_projects.each do |p|
if opts[:"#{p}"]
if pr = /pr_(\d+)/.match(opts[:"#{p}"])
# This is a pull request number. Get the fork and branch
v = parse_project_version(get_ref_from_pull_request(p, pr[1]))
else
v = parse_project_version(opts[:"#{p}"])
end
ENV["#{p}"] = "#{v[:fork]}:#{v[:sha]}"
log_notice("Using #{p}: #{v[:fork]}:#{v[:sha]}")
end
end
create_host_config([agent, master], config)
pre_suites = ['lib/setup/patch/pre-suite', 'lib/setup/common/pre-suite']
options = {'hosts' => config, 'pre-suite' => pre_suites.join(',')}
options['tests'] = test_location if tests
options['keyfile'] = keyfile if keyfile
options['noop'] = opts[:noop]
run_beaker(options)
if !opts[:noop]
log = get_latest_host_config
print_report({:agent => log[:HOSTS].keys[0], :master => log[:HOSTS].keys[1], :puppet_agent => "#{pa_version[:fork]}:#{pa_version[:sha]}"})
end
exit 0
end
end
def parse_project_version(option)
keys = option.split(':')
if keys.length == 2
project_fork = keys[0]
project_sha = keys[1]
else
project_fork = 'puppetlabs'
project_sha = keys[0]
end
project_sha = 'nightly' if project_sha == 'latest'
{:fork => project_fork, :sha => project_sha}
end
def tmpdir
`mktemp -d /tmp/puppetstein.XXXXX`.chomp!
end
def change_component_ref(component_name, url, ref, tmp, noop=nil)
new_ref = Hash.new
new_ref['url'] = url
new_ref['ref'] = ref
File.write("#{tmp}/puppet-agent/configs/components/#{component_name}.json", JSON.pretty_generate(new_ref)) if !noop
log_notice("Updated #{tmp}/puppet-agent/configs/components/#{component_name}.json with url #{url} and ref #{ref}") end
def build_puppet_agent(host, keyfile, tmp)
log_notice("building puppet-agent for #{HOST_MAP[host]}")
ENV['VANAGON_SSH_KEY'] = keyfile if keyfile
cmd = "pushd #{tmp}/puppet-agent && bundle install && bundle exec build puppet-agent" +
" #{HOST_MAP[host]} && popd"
execute(cmd)
if $? != 0
raise "Puppet Agent build failed, aborting."
end
end
def build_facter
# 1) Clone, build and install leatherman (check if it exists? Require it?)
# 1.5) Clone, build and install cpp-hocon??
# 2) Clone and build facter
# 3) copy libfacter.so to VM and put in on top of the old one
end
def cleanup
`rm -rf #{tmpdir}`
end
if __FILE__ == $0
command.run(ARGV)
end