-
-
Notifications
You must be signed in to change notification settings - Fork 605
/
build_files.py
400 lines (325 loc) · 14.7 KB
/
build_files.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
# coding=utf-8
# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
unicode_literals, with_statement)
import collections
from os.path import dirname, join
import six
from pants.base.project_tree import Dir
from pants.base.specs import (AscendantAddresses, DescendantAddresses, SiblingAddresses,
SingleAddress, Spec)
from pants.build_graph.address import Address, BuildFileAddress
from pants.build_graph.address_lookup_error import AddressLookupError
from pants.engine.addressable import (AddressableDescriptor, BuildFileAddresses, Collection,
Exactly, TypeConstraintError)
from pants.engine.fs import FilesContent, PathGlobs, Snapshot
from pants.engine.mapper import AddressFamily, AddressMap, AddressMapper, ResolveError
from pants.engine.objects import Locatable, SerializableFactory, Validatable
from pants.engine.rules import RootRule, SingletonRule, TaskRule, rule
from pants.engine.selectors import Select, SelectDependencies, SelectProjection
from pants.engine.struct import Struct
from pants.util.dirutil import fast_relpath_optional
from pants.util.objects import datatype
class ResolvedTypeMismatchError(ResolveError):
"""Indicates a resolved object was not of the expected type."""
def _key_func(entry):
key, value = entry
return key
class BuildDirs(datatype('BuildDirs', ['dependencies'])):
"""A list of Stat objects for directories containing build files."""
class BuildFiles(datatype('BuildFiles', ['files_content'])):
"""The FileContents of BUILD files in some directory"""
class BuildFileGlobs(datatype('BuildFilesGlobs', ['path_globs'])):
"""A wrapper around PathGlobs that are known to match a build file pattern."""
class Specs(Collection.of(Spec)):
"""A collection of Spec subclasses."""
@rule(BuildFiles,
[SelectProjection(FilesContent, PathGlobs, 'path_globs', BuildFileGlobs)])
def build_files(files_content):
return BuildFiles(files_content)
@rule(BuildFileGlobs, [Select(AddressMapper), Select(Dir)])
def buildfile_path_globs_for_dir(address_mapper, directory):
patterns = address_mapper.build_patterns
return BuildFileGlobs(PathGlobs.create(directory.path, include=patterns, exclude=()))
@rule(AddressFamily, [Select(AddressMapper), Select(Dir), Select(BuildFiles)])
def parse_address_family(address_mapper, path, build_files):
"""Given the contents of the build files in one directory, return an AddressFamily.
The AddressFamily may be empty, but it will not be None.
"""
files_content = build_files.files_content.dependencies
if not files_content:
raise ResolveError('Directory "{}" does not contain build files.'.format(path))
address_maps = []
paths = (f.path for f in files_content)
for filecontent_product in files_content:
address_maps.append(AddressMap.parse(filecontent_product.path,
filecontent_product.content,
address_mapper.parser))
return AddressFamily.create(path.path, address_maps)
class UnhydratedStruct(datatype('UnhydratedStruct', ['address', 'struct', 'dependencies'])):
"""A product type that holds a Struct which has not yet been hydrated.
A Struct counts as "hydrated" when all of its members (which are not themselves dependencies
lists) have been resolved from the graph. This means that hydrating a struct is eager in terms
of inline addressable fields, but lazy in terms of the complete graph walk represented by
the `dependencies` field of StructWithDeps.
"""
def __eq__(self, other):
if type(self) != type(other):
return NotImplemented
return self.struct == other.struct
def __ne__(self, other):
return not (self == other)
def __hash__(self):
return hash(self.struct)
def _raise_did_you_mean(address_family, name):
names = [a.target_name for a in address_family.addressables]
possibilities = '\n '.join(':{}'.format(target_name) for target_name in sorted(names))
raise ResolveError('"{}" was not found in namespace "{}". '
'Did you mean one of:\n {}'
.format(name, address_family.namespace, possibilities))
@rule(UnhydratedStruct,
[Select(AddressMapper),
SelectProjection(AddressFamily, Dir, 'spec_path', Address),
Select(Address)])
def resolve_unhydrated_struct(address_mapper, address_family, address):
"""Given an Address and its AddressFamily, resolve an UnhydratedStruct.
Recursively collects any embedded addressables within the Struct, but will not walk into a
dependencies field, since those are requested explicitly by tasks using SelectDependencies.
"""
struct = address_family.addressables.get(address)
addresses = address_family.addressables
if not struct or address not in addresses:
_raise_did_you_mean(address_family, address.target_name)
dependencies = []
def maybe_append(outer_key, value):
if isinstance(value, six.string_types):
if outer_key != 'dependencies':
dependencies.append(Address.parse(value,
relative_to=address.spec_path,
subproject_roots=address_mapper.subproject_roots))
elif isinstance(value, Struct):
collect_dependencies(value)
def collect_dependencies(item):
for key, value in sorted(item._asdict().items(), key=_key_func):
if not AddressableDescriptor.is_addressable(item, key):
continue
if isinstance(value, collections.MutableMapping):
for _, v in sorted(value.items(), key=_key_func):
maybe_append(key, v)
elif isinstance(value, collections.MutableSequence):
for v in value:
maybe_append(key, v)
else:
maybe_append(key, value)
collect_dependencies(struct)
return UnhydratedStruct(
filter(lambda build_address: build_address == address, addresses)[0], struct, dependencies)
def hydrate_struct(address_mapper, unhydrated_struct, dependencies):
"""Hydrates a Struct from an UnhydratedStruct and its satisfied embedded addressable deps.
Note that this relies on the guarantee that DependenciesNode provides dependencies in the
order they were requested.
"""
address = unhydrated_struct.address
struct = unhydrated_struct.struct
def maybe_consume(outer_key, value):
if isinstance(value, six.string_types):
if outer_key == 'dependencies':
# Don't recurse into the dependencies field of a Struct, since those will be explicitly
# requested by tasks. But do ensure that their addresses are absolute, since we're
# about to lose the context in which they were declared.
value = Address.parse(value,
relative_to=address.spec_path,
subproject_roots=address_mapper.subproject_roots)
else:
value = dependencies[maybe_consume.idx]
maybe_consume.idx += 1
elif isinstance(value, Struct):
value = consume_dependencies(value)
return value
# NB: Some pythons throw an UnboundLocalError for `idx` if it is a simple local variable.
maybe_consume.idx = 0
# 'zip' the previously-requested dependencies back together as struct fields.
def consume_dependencies(item, args=None):
hydrated_args = args or {}
for key, value in sorted(item._asdict().items(), key=_key_func):
if not AddressableDescriptor.is_addressable(item, key):
hydrated_args[key] = value
continue
if isinstance(value, collections.MutableMapping):
container_type = type(value)
hydrated_args[key] = container_type((k, maybe_consume(key, v))
for k, v in sorted(value.items(), key=_key_func))
elif isinstance(value, collections.MutableSequence):
container_type = type(value)
hydrated_args[key] = container_type(maybe_consume(key, v) for v in value)
else:
hydrated_args[key] = maybe_consume(key, value)
return _hydrate(type(item), address.spec_path, **hydrated_args)
return consume_dependencies(struct, args={'address': address})
def _hydrate(item_type, spec_path, **kwargs):
# If the item will be Locatable, inject the spec_path.
if issubclass(item_type, Locatable):
kwargs['spec_path'] = spec_path
try:
item = item_type(**kwargs)
except TypeConstraintError as e:
raise ResolvedTypeMismatchError(e)
# Let factories replace the hydrated object.
if isinstance(item, SerializableFactory):
item = item.create()
# Finally make sure objects that can self-validate get a chance to do so.
if isinstance(item, Validatable):
item.validate()
return item
@rule(BuildFileAddresses,
[Select(AddressMapper),
SelectDependencies(AddressFamily, BuildDirs, field_types=(Dir,)),
Select(Specs)])
def addresses_from_address_families(address_mapper, address_families, specs):
"""Given a list of AddressFamilies matching a list of Specs, return matching Addresses.
Raises a AddressLookupError if:
- there were no matching AddressFamilies, or
- the Spec matches no addresses for SingleAddresses.
"""
# NB: `@memoized` does not work on local functions.
def by_directory():
if by_directory.cached is None:
by_directory.cached = {af.namespace: af for af in address_families}
return by_directory.cached
by_directory.cached = None
def raise_empty_address_family(spec):
raise ResolveError('Path "{}" contains no BUILD files.'.format(spec.directory))
def exclude_address(address):
if address_mapper.exclude_patterns:
address_str = address.spec
return any(p.search(address_str) is not None for p in address_mapper.exclude_patterns)
return False
addresses = []
included = set()
def include(address_families, predicate=None):
matched = False
for af in address_families:
for a in af.addressables.keys():
if a in included:
continue
if not exclude_address(a) and (predicate is None or predicate(a)):
matched = True
addresses.append(a)
included.add(a)
return matched
for spec in specs.dependencies:
if type(spec) is DescendantAddresses:
matched = include(
af
for af in address_families
if fast_relpath_optional(af.namespace, spec.directory) is not None
)
if not matched:
raise AddressLookupError(
'Spec {} does not match any targets.'.format(spec))
elif type(spec) is SiblingAddresses:
address_family = by_directory().get(spec.directory)
if not address_family:
raise_empty_address_family(spec)
include([address_family])
elif type(spec) is SingleAddress:
address_family = by_directory().get(spec.directory)
if not address_family:
raise_empty_address_family(spec)
if not include([address_family], predicate=lambda a: a.target_name == spec.name):
_raise_did_you_mean(address_family, spec.name)
elif type(spec) is AscendantAddresses:
include(
af
for af in address_families
if fast_relpath_optional(spec.directory, af.namespace) is not None
)
else:
raise ValueError('Unrecognized Spec type: {}'.format(spec))
return BuildFileAddresses(addresses)
@rule(BuildDirs, [Select(AddressMapper), Select(Snapshot)])
def filter_build_dirs(address_mapper, snapshot):
"""Given a Snapshot matching a build pattern, return parent directories as BuildDirs."""
dirnames = set(dirname(f.stat.path) for f in snapshot.files)
return BuildDirs(tuple(Dir(d) for d in dirnames))
@rule(PathGlobs, [Select(AddressMapper), Select(Specs)])
def spec_to_globs(address_mapper, specs):
"""Given a Spec object, return a PathGlobs object for the build files that it matches.
"""
patterns = set()
for spec in specs.dependencies:
if type(spec) is DescendantAddresses:
patterns.update(join(spec.directory, '**', pattern)
for pattern in address_mapper.build_patterns)
elif type(spec) in (SiblingAddresses, SingleAddress):
patterns.update(join(spec.directory, pattern)
for pattern in address_mapper.build_patterns)
elif type(spec) is AscendantAddresses:
patterns.update(join(f, pattern)
for pattern in address_mapper.build_patterns
for f in _recursive_dirname(spec.directory))
else:
raise ValueError('Unrecognized Spec type: {}'.format(spec))
return PathGlobs.create('', include=patterns, exclude=address_mapper.build_ignore_patterns)
def _recursive_dirname(f):
"""Given a relative path like 'a/b/c/d', yield all ascending path components like:
'a/b/c/d'
'a/b/c'
'a/b'
'a'
''
"""
while f:
yield f
f = dirname(f)
yield ''
# TODO: This is a bit of a lie: `Struct` is effectively abstract, so this collection
# will contain subclasses of `Struct` for the symbol table types. These APIs need more
# polish before we make them public: see #4535 in particular.
HydratedStructs = Collection.of(Struct)
BuildFilesCollection = Collection.of(BuildFiles)
def create_graph_rules(address_mapper, symbol_table):
"""Creates tasks used to parse Structs from BUILD files.
:param address_mapper_key: The subject key for an AddressMapper instance.
:param symbol_table: A SymbolTable instance to provide symbols for Address lookups.
"""
symbol_table_constraint = symbol_table.constraint()
return [
TaskRule(BuildFilesCollection,
[SelectDependencies(BuildFiles, BuildDirs, field_types=(Dir,))],
BuildFilesCollection),
# A singleton to provide the AddressMapper.
SingletonRule(AddressMapper, address_mapper),
# Support for resolving Structs from Addresses.
TaskRule(
symbol_table_constraint,
[Select(AddressMapper),
Select(UnhydratedStruct),
SelectDependencies(symbol_table_constraint, UnhydratedStruct, field_types=(Address,))],
hydrate_struct
),
resolve_unhydrated_struct,
TaskRule(
HydratedStructs,
[SelectDependencies(symbol_table_constraint,
BuildFileAddresses,
field_types=(Address,),
field='addresses')],
HydratedStructs
),
# BUILD file parsing.
parse_address_family,
build_files,
buildfile_path_globs_for_dir,
# Spec handling: locate directories that contain build files, and request
# AddressFamilies for each of them.
addresses_from_address_families,
filter_build_dirs,
spec_to_globs,
# Root rules representing parameters that might be provided via root subjects.
RootRule(Address),
RootRule(BuildFileAddress),
RootRule(BuildFileAddresses),
RootRule(Specs),
]