-
Notifications
You must be signed in to change notification settings - Fork 2.2k
/
source.rb
356 lines (297 loc) · 12.6 KB
/
source.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
require 'puppet/file_serving/content'
require 'puppet/file_serving/metadata'
require 'puppet/file_serving/terminus_helper'
require 'puppet/util/http_proxy'
require 'puppet/network/http'
require 'puppet/network/http/api/indirected_routes'
require 'puppet/network/http/compression'
module Puppet
# Copy files from a local or remote source. This state *only* does any work
# when the remote file is an actual file; in that case, this state copies
# the file down. If the remote file is a dir or a link or whatever, then
# this state, during retrieval, modifies the appropriate other states
# so that things get taken care of appropriately.
Puppet::Type.type(:file).newparam(:source) do
include Puppet::Network::HTTP::Compression.module
BINARY_MIME_TYPES = [
Puppet::Network::FormatHandler.format_for('binary').mime
].join(', ').freeze
attr_accessor :source, :local
desc <<-'EOT'
A source file, which will be copied into place on the local system. This
attribute is mutually exclusive with `content` and `target`. Allowed
values are:
* `puppet:` URIs, which point to files in modules or Puppet file server
mount points.
* Fully qualified paths to locally available files (including files on NFS
shares or Windows mapped drives).
* `file:` URIs, which behave the same as local file paths.
* `http:` URIs, which point to files served by common web servers
The normal form of a `puppet:` URI is:
`puppet:///modules/<MODULE NAME>/<FILE PATH>`
This will fetch a file from a module on the Puppet master (or from a
local module when using Puppet apply). Given a `modulepath` of
`/etc/puppetlabs/code/modules`, the example above would resolve to
`/etc/puppetlabs/code/modules/<MODULE NAME>/files/<FILE PATH>`.
Unlike `content`, the `source` attribute can be used to recursively copy
directories if the `recurse` attribute is set to `true` or `remote`. If
a source directory contains symlinks, use the `links` attribute to
specify whether to recreate links or follow them.
*HTTP* URIs cannot be used to recursively synchronize whole directory
trees. It is also not possible to use `source_permissions` values other
than `ignore`. That's because HTTP servers do not transfer any metadata
that translates to ownership or permission details.
Multiple `source` values can be specified as an array, and Puppet will
use the first source that exists. This can be used to serve different
files to different system types:
file { '/etc/nfs.conf':
source => [
"puppet:///modules/nfs/conf.${host}",
"puppet:///modules/nfs/conf.${operatingsystem}",
'puppet:///modules/nfs/conf'
]
}
Alternately, when serving directories recursively, multiple sources can
be combined by setting the `sourceselect` attribute to `all`.
EOT
validate do |sources|
sources = [sources] unless sources.is_a?(Array)
sources.each do |source|
next if Puppet::Util.absolute_path?(source)
begin
uri = URI.parse(Puppet::Util.uri_encode(source))
rescue => detail
self.fail Puppet::Error, "Could not understand source #{source}: #{detail}", detail
end
self.fail "Cannot use relative URLs '#{source}'" unless uri.absolute?
self.fail "Cannot use opaque URLs '#{source}'" unless uri.hierarchical?
unless %w{file puppet http https}.include?(uri.scheme)
self.fail "Cannot use URLs of type '#{uri.scheme}' as source for fileserving"
end
end
end
SEPARATOR_REGEX = [Regexp.escape(File::SEPARATOR.to_s), Regexp.escape(File::ALT_SEPARATOR.to_s)].join
munge do |sources|
sources = [sources] unless sources.is_a?(Array)
sources.map do |source|
source = self.class.normalize(source)
if Puppet::Util.absolute_path?(source)
# CGI.unescape will butcher properly escaped URIs
uri_string = Puppet::Util.path_to_uri(source).to_s
# Ruby 1.9.3 and earlier have a URI bug in URI
# to_s returns an ASCII string despite UTF-8 fragments
# since its escaped its safe to universally call encode
# URI.unescape always returns strings in the original encoding
URI.unescape(uri_string.encode(Encoding::UTF_8))
else
source
end
end
end
def self.normalize(source)
source.sub(/[#{SEPARATOR_REGEX}]+$/, '')
end
def change_to_s(currentvalue, newvalue)
# newvalue = "{md5}#{@metadata.checksum}"
if resource.property(:ensure).retrieve == :absent
return "creating from source #{metadata.source} with contents #{metadata.checksum}"
else
return "replacing from source #{metadata.source} with contents #{metadata.checksum}"
end
end
def checksum
metadata && metadata.checksum
end
# Look up (if necessary) and return local content.
def content
return @content if @content
raise Puppet::DevError, "No source for content was stored with the metadata" unless metadata.source
unless tmp = Puppet::FileServing::Content.indirection.find(metadata.source, :environment => resource.catalog.environment_instance, :links => resource[:links])
self.fail "Could not find any content at %s" % metadata.source
end
@content = tmp.content
end
# Copy the values from the source to the resource. Yay.
def copy_source_values
devfail "Somehow got asked to copy source values without any metadata" unless metadata
# conditionally copy :checksum
if metadata.ftype != "directory" && !(metadata.ftype == "link" && metadata.links == :manage)
copy_source_value(:checksum)
end
# Take each of the stats and set them as states on the local file
# if a value has not already been provided.
[:owner, :mode, :group].each do |metadata_method|
next if metadata_method == :owner and !Puppet.features.root?
next if metadata_method == :group and !Puppet.features.root?
case resource[:source_permissions]
when :ignore
next
when :use_when_creating
next if Puppet::FileSystem.exist?(resource[:path])
end
copy_source_value(metadata_method)
end
if resource[:ensure] == :absent
# We know all we need to
elsif metadata.ftype != "link"
resource[:ensure] = metadata.ftype
elsif resource[:links] == :follow
resource[:ensure] = :present
else
resource[:ensure] = "link"
resource[:target] = metadata.destination
end
end
attr_writer :metadata
# Provide, and retrieve if necessary, the metadata for this file. Fail
# if we can't find data about this host, and fail if there are any
# problems in our query.
def metadata
return @metadata if @metadata
if @metadata = resource.catalog.metadata[resource.title]
return @metadata
end
return nil unless value
value.each do |source|
begin
options = {
:environment => resource.catalog.environment_instance,
:links => resource[:links],
:checksum_type => resource[:checksum],
:source_permissions => resource[:source_permissions]
}
if data = Puppet::FileServing::Metadata.indirection.find(source, options)
@metadata = data
@metadata.source = source
break
end
rescue => detail
self.fail Puppet::Error, "Could not retrieve file metadata for #{source}: #{detail}", detail
end
end
self.fail "Could not retrieve information from environment #{resource.catalog.environment} source(s) #{value.join(", ")}" unless @metadata
@metadata
end
def local?
found? and scheme == "file"
end
def full_path
Puppet::Util.uri_to_path(uri) if found?
end
def server?
uri and uri.host
end
def server
(uri and uri.host) or Puppet.settings[:server]
end
def port
(uri and uri.port) or Puppet.settings[:masterport]
end
def uri
@uri ||= URI.parse(Puppet::Util.uri_encode(metadata.source))
end
def write(file)
resource.parameter(:checksum).sum_stream { |sum|
each_chunk_from { |chunk|
sum << chunk
file.print chunk
}
}
end
private
def scheme
(uri and uri.scheme)
end
def found?
! (metadata.nil? or metadata.ftype.nil?)
end
def copy_source_value(metadata_method)
param_name = (metadata_method == :checksum) ? :content : metadata_method
if resource[param_name].nil? or resource[param_name] == :absent
if Puppet.features.microsoft_windows? && [:owner, :group, :mode].include?(metadata_method)
devfail "Should not have tried to use source owner/mode/group on Windows"
end
value = metadata.send(metadata_method)
# Force the mode value in file resources to be a string containing octal.
value = value.to_s(8) if param_name == :mode && value.is_a?(Numeric)
resource[param_name] = value
if (metadata_method == :checksum)
# If copying checksum, also copy checksum_type
resource[:checksum] = metadata.checksum_type
end
end
end
def each_chunk_from
if Puppet[:default_file_terminus] == :file_server
yield content
elsif local?
chunk_file_from_disk { |chunk| yield chunk }
else
chunk_file_from_source { |chunk| yield chunk }
end
end
def chunk_file_from_disk
File.open(full_path, "rb") do |src|
while chunk = src.read(8192)
yield chunk
end
end
end
def get_from_puppet_source(source_uri, content_uri, &block)
options = { :environment => resource.catalog.environment_instance }
if content_uri
options[:code_id] = resource.catalog.code_id
request = Puppet::Indirector::Request.new(:static_file_content, :find, content_uri, nil, options)
else
request = Puppet::Indirector::Request.new(:file_content, :find, source_uri, nil, options)
end
request.do_request(:fileserver) do |req|
connection = Puppet::Network::HttpPool.http_instance(req.server, req.port)
connection.request_get(Puppet::Network::HTTP::API::IndirectedRoutes.request_to_uri(req), add_accept_encoding({"Accept" => BINARY_MIME_TYPES}), &block)
end
end
def get_from_http_source(source_uri, &block)
Puppet::Util::HttpProxy.request_with_redirects(URI(source_uri), :get, &block)
end
def get_from_source(&block)
source_uri = metadata.source
if source_uri =~ /^https?:/
get_from_http_source(source_uri, &block)
else
get_from_puppet_source(source_uri, metadata.content_uri, &block)
end
end
def chunk_file_from_source
get_from_source do |response|
case response.code
when /^2/; uncompress(response) { |uncompressor| response.read_body { |chunk| yield uncompressor.uncompress(chunk) } }
else
# Raise the http error if we didn't get a 'success' of some kind.
message = "Error #{response.code} on SERVER: #{(response.body||'').empty? ? response.message : uncompress_body(response)}"
raise Net::HTTPError.new(message, response)
end
end
end
end
Puppet::Type.type(:file).newparam(:source_permissions) do
desc <<-'EOT'
Whether (and how) Puppet should copy owner, group, and mode permissions from
the `source` to `file` resources when the permissions are not explicitly
specified. (In all cases, explicit permissions will take precedence.)
Valid values are `use`, `use_when_creating`, and `ignore`:
* `ignore` (the default) will never apply the owner, group, or mode from
the `source` when managing a file. When creating new files without explicit
permissions, the permissions they receive will depend on platform-specific
behavior. On POSIX, Puppet will use the umask of the user it is running as.
On Windows, Puppet will use the default DACL associated with the user it is
running as.
* `use` will cause Puppet to apply the owner, group,
and mode from the `source` to any files it is managing.
* `use_when_creating` will only apply the owner, group, and mode from the
`source` when creating a file; existing files will not have their permissions
overwritten.
EOT
defaultto :ignore
newvalues(:use, :use_when_creating, :ignore)
end
end