-
Notifications
You must be signed in to change notification settings - Fork 187
/
repair_skipping_properties.go
479 lines (402 loc) · 16.5 KB
/
repair_skipping_properties.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
/*
* Copyright (c) Microsoft Corporation.
* Licensed under the MIT license.
*/
package pipeline
import (
"context"
"fmt"
"github.com/pkg/errors"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"github.com/Azure/azure-service-operator/v2/internal/set"
"github.com/Azure/azure-service-operator/v2/tools/generator/internal/astmodel"
"github.com/Azure/azure-service-operator/v2/tools/generator/internal/codegen/storage"
)
const RepairSkippingPropertiesStageID = "repairSkippingProperties"
// RepairSkippingProperties repairs any properties that skip one or more versions of an object and are reintroduced.
// As described in issue #1776, such properties are a problem because they might create incompatibilities between
// different versions of ASO, or between different versions of a given resource.
//
// To repair these, we need to ensure that objects stored in property bags are always serialized with the same
// shape. For more details, see https://azure.github.io/azure-service-operator/design/adr-2023-09-skipping-properties/
//
// Repair works by scanning for properties that are dropped between versions of a resource. We keep track of all
// these properties, and if a specific property appears more than once (implying there are two or more sequence versions
// of that property) we know a repair is required.
//
// To illustrate, assume we have the following set of objects across versions of our mythical CRM service:
//
// v1: Person(GivenName, FamilyName, Address)
// v2: Person(KnownAs, FullName, FamilyName)
// v3: Person(KnownAs, FullName, FamilyName, Address)
//
// Scanning these types, we find:
//
// FamilyName: present in (v1, v2, v3); no issue
// FullName: present in (v2, v3); no issue
// KnownAs: also present in (v2, v3); no issue
// GivenName: present in (v1); no issue
//
// Address: present in (v1, v3); (skipping v2), repair required.
func RepairSkippingProperties() *Stage {
stage := NewStage(
RepairSkippingPropertiesStageID,
"Repair property bag serialization for properties that skip resource or object versions",
func(ctx context.Context, state *State) (*State, error) {
repairer := newSkippingPropertyRepairer(state.Definitions(), state.ConversionGraph())
// Add resources and objects to the graph
for _, def := range state.Definitions() {
if t, ok := astmodel.AsPropertyContainer(def.Type()); ok {
err := repairer.AddProperties(def.Name(), t.Properties().AsSlice()...)
if err != nil {
return nil, err
}
}
}
defs, err := repairer.RepairSkippedProperties()
if err != nil {
return nil, err
}
return state.WithOverlaidDefinitions(defs), err
})
// We have to have a conversion graph to detect skipping properties
stage.RequiresPrerequisiteStages(CreateConversionGraphStageId)
// We also require the Conversion Graph to be recreated afterwards,
// but our stage dependencies can't currently capture that.
return stage
}
type skippingPropertyRepairer struct {
links map[astmodel.PropertyReference]astmodel.PropertyReference // Individual links in chains of related properties
observedProperties *astmodel.PropertyReferenceSet // Set of properties we've observed
definitions astmodel.TypeDefinitionSet // Set of all known type definitions
conversionGraph *storage.ConversionGraph // Graph of conversions between types
comparer *structuralComparer // Helper used to compare types for structural equality
}
// newSkippingPropertyRepairer creates a new graph for tracking chains of properties as they evolve through different
// versions of a resource or object.
// definitions is a set of all known types.
// conversionGraph contains every conversion/transition between versions.
func newSkippingPropertyRepairer(
definitions astmodel.TypeDefinitionSet,
conversionGraph *storage.ConversionGraph,
) *skippingPropertyRepairer {
return &skippingPropertyRepairer{
links: make(map[astmodel.PropertyReference]astmodel.PropertyReference),
observedProperties: astmodel.NewPropertyReferenceSet(),
definitions: definitions,
conversionGraph: conversionGraph,
comparer: newStructuralComparer(definitions),
}
}
// AddProperties adds all the properties from the specified type to the graph.
func (repairer *skippingPropertyRepairer) AddProperties(
name astmodel.InternalTypeName,
properties ...*astmodel.PropertyDefinition,
) error {
var errs []error
for _, property := range properties {
if err := repairer.AddProperty(name, property); err != nil {
errs = append(errs, err)
}
}
err := kerrors.NewAggregate(errs)
if err != nil {
return errors.Wrapf(err, "adding properties from %s", name)
}
return nil
}
// AddProperty adds a single property from a type to the graph, marking it as observed
func (repairer *skippingPropertyRepairer) AddProperty(
name astmodel.InternalTypeName,
property *astmodel.PropertyDefinition,
) error {
ref := astmodel.MakePropertyReference(name, property.PropertyName())
if err := repairer.establishPropertyChain(ref); err != nil {
return errors.Wrapf(err, "adding property %s", property.PropertyName())
}
repairer.propertyObserved(ref)
return nil
}
// RepairSkippedProperties scans for properties that skip versions, and injects new types to repair the chain, returning
// a (possibly empty) set of new definitions.
func (repairer *skippingPropertyRepairer) RepairSkippedProperties() (astmodel.TypeDefinitionSet, error) {
result := make(astmodel.TypeDefinitionSet)
chains := repairer.findChains().AsSlice()
var errs []error
for _, ref := range chains {
defs, err := repairer.repairChain(ref)
if err != nil {
errs = append(errs, err)
continue
}
// If the repair added any new types (mostly it won't), include them in the result
result.AddTypes(defs)
}
if len(errs) > 0 {
return nil, errors.Wrapf(
kerrors.NewAggregate(errs),
"failed to repair skipping properties")
}
return result, nil
}
// establishPropertyChain ensures that a full property chain exists for the specified property. Any missing links in the
// chain will be created. If the required chain already exists, this is a no-op.
func (repairer *skippingPropertyRepairer) establishPropertyChain(ref astmodel.PropertyReference) error {
if ref.IsEmpty() || repairer.hasLinkFrom(ref) {
// Nothing to do
return nil
}
return repairer.createPropertyChain(ref)
}
// createPropertyChain creates a full property chain for the specified property.
// ref is the property reference that specifies the start of our chain.
// It recursively calls establishPropertyChain to avoid creating parts of the chain multiple times.
func (repairer *skippingPropertyRepairer) createPropertyChain(ref astmodel.PropertyReference) error {
next, err := repairer.conversionGraph.FindNextProperty(ref, repairer.definitions)
if err != nil {
return errors.Wrapf(err, "creating property chain link from %s", ref.String())
}
repairer.addLink(ref, next)
return repairer.establishPropertyChain(next)
}
// propertyObserved makes a record that a given property has been observedProperties
func (repairer *skippingPropertyRepairer) propertyObserved(ref astmodel.PropertyReference) {
repairer.observedProperties.Add(ref)
}
// addLink adds a link between two property references
func (repairer *skippingPropertyRepairer) addLink(ref astmodel.PropertyReference, next astmodel.PropertyReference) {
repairer.links[ref] = next
}
// findChains finds all the property references that are found only as the start of a chain
func (repairer *skippingPropertyRepairer) findChains() *astmodel.PropertyReferenceSet {
starts := astmodel.NewPropertyReferenceSet()
finishes := astmodel.NewPropertyReferenceSet()
for s, f := range repairer.links {
starts.Add(s)
finishes.Add(f)
}
return starts.Except(finishes)
}
// hasLinkFrom returns true if we already have a link from the specified property
func (repairer *skippingPropertyRepairer) hasLinkFrom(ref astmodel.PropertyReference) bool {
_, found := repairer.links[ref]
return found
}
// repairChain checks for a gap in the specified property chain.
// start is the first property reference in the chain
func (repairer *skippingPropertyRepairer) repairChain(
start astmodel.PropertyReference,
) (astmodel.TypeDefinitionSet, error) {
lastObserved, firstMissing := repairer.findBreak(start, repairer.wasPropertyObserved)
if firstMissing.IsEmpty() {
// Property was never discontinued
return nil, nil
}
lastMissing, reintroduced := repairer.findBreak(firstMissing, repairer.wasPropertyObserved)
if reintroduced.IsEmpty() {
// Property was never reintroduced
return nil, nil
}
// If the properties have the same type, we don't have a break here - so we check the remainder of the chain
// (This is Ok because the value serialized into the property bag from lastObserved will deserialize into the
// reintroduced property intact.)
typesSame := repairer.propertiesHaveStructurallyIdenticalType(lastObserved, reintroduced)
if typesSame {
return repairer.repairChain(reintroduced)
}
// We've found a skipping property with a different shape. If it's a TypeName we need to repair it.
// We do this by creating a new type that has the same shape as the missing property, injected just prior to
// reintroduction.
lastObservedPropertyType, ok := repairer.lookupPropertyType(lastObserved)
if !ok {
// Should never fail, given the way findBreak() works
panic(fmt.Sprintf("failed to find type for property %s", lastObserved))
}
tn, ok := astmodel.AsInternalTypeName(lastObservedPropertyType)
if !ok {
// If not a type name, defer to our existing property conversion logic
// Continue checking the rest of the chain
return repairer.repairChain(reintroduced)
}
// Find the type definition for when the object was last observed
def := repairer.definitions[tn]
// Find all the type definitions referenced by this definition (e.g. nested types)
defs, err := astmodel.FindConnectedDefinitions(repairer.definitions, astmodel.MakeTypeDefinitionSetFromDefinitions(def))
if err != nil {
return nil, errors.Wrapf(
err,
"failed to find connected definitions from %s",
tn)
}
// Move all of these types into a compatibility package
compatPkg := astmodel.MakeCompatPackageReference(lastMissing.DeclaringType().InternalPackageReference())
renamer := astmodel.NewRenamingVisitorFromLambda(
func(name astmodel.InternalTypeName) astmodel.InternalTypeName {
return name.WithPackageReference(compatPkg)
})
newDefs, err := renamer.RenameAll(defs)
if err != nil {
return nil, errors.Wrapf(
err,
"failed to rename definitions from %s",
tn)
}
return newDefs, nil
}
// wasPropertyObserved returns true if the property reference has been observedProperties; false otherwise.
func (repairer *skippingPropertyRepairer) wasPropertyObserved(ref astmodel.PropertyReference) bool {
return repairer.observedProperties.Contains(ref)
}
// findBreak finds a pair of consecutive references where the provided predicate gives a different answer for each.
// ref is the property reference from which to start scanning the chain.
// predicate is a test used to identify the pair of references to return.
// A break is always found at the end of the chain, returning <last>, <empty>.
// If ref is empty, will return <empty>, <empty>
func (repairer *skippingPropertyRepairer) findBreak(
ref astmodel.PropertyReference,
predicate func(astmodel.PropertyReference) bool,
) (astmodel.PropertyReference, astmodel.PropertyReference) {
next := repairer.lookupNext(ref)
if next.IsEmpty() || predicate(ref) != predicate(next) {
return ref, next
}
return repairer.findBreak(next, predicate)
}
// lookupNext returns the next property in the chain, if any.
// ref is the property reference to look up.
// returns the next property, if found; <empty>> if not.
func (repairer *skippingPropertyRepairer) lookupNext(ref astmodel.PropertyReference) astmodel.PropertyReference {
if next, ok := repairer.links[ref]; ok {
return next
}
return astmodel.EmptyPropertyReference
}
// propertiesHaveStructurallyIdenticalType returns true if both the passed property-references exist and have the same underlying type
func (repairer *skippingPropertyRepairer) propertiesHaveStructurallyIdenticalType(
left astmodel.PropertyReference,
right astmodel.PropertyReference,
) bool {
leftType, leftOk := repairer.lookupPropertyType(left)
rightType, rightOk := repairer.lookupPropertyType(right)
exactlyEqual := leftOk && rightOk && leftType.Equals(rightType, astmodel.EqualityOverrides{})
if exactlyEqual {
return true
}
// If the types aren't exactly equal, we need to check if they're structurally equal
return repairer.comparer.areTypesStructurallyEqual(leftType, rightType)
}
// lookupPropertyType accepts a PropertyReference and looks up the actual type of the property, returning true if found,
// or false if not.
func (repairer *skippingPropertyRepairer) lookupPropertyType(ref astmodel.PropertyReference) (astmodel.Type, bool) {
def, ok := repairer.definitions[ref.DeclaringType()]
if !ok {
// Type not found
return nil, false
}
container, ok := astmodel.AsPropertyContainer(def.Type())
if !ok {
// Not a property container
return nil, false
}
prop, ok := container.Property(ref.Property())
if !ok {
// Not a known property
return nil, false
}
return prop.PropertyType(), true
}
type structuralComparer struct {
definitions astmodel.TypeDefinitionSet
equalityOverrides astmodel.EqualityOverrides
}
func newStructuralComparer(definitions astmodel.TypeDefinitionSet) *structuralComparer {
result := &structuralComparer{
definitions: definitions,
}
result.equalityOverrides = astmodel.EqualityOverrides{
InternalTypeName: result.equalTypesReferencedByInternalTypeNames,
ObjectType: result.equalObjectTypeStructure,
}
return result
}
// areTypeSetsEqual returns true if the two sets of definitions are structurally equal, false otherwise.
func (comparer *structuralComparer) areTypeSetsEqual(
left astmodel.TypeDefinitionSet,
right astmodel.TypeDefinitionSet,
) bool {
if len(left) != len(right) {
return false
}
_, err := comparer.findCommonPackage(left)
if err != nil {
// Never expected to happen
panic(err)
}
rightPkg, err := comparer.findCommonPackage(right)
if err != nil {
// Never expected to happen
panic(err)
}
for leftName, leftDef := range left {
rightName := leftName.WithPackageReference(rightPkg)
rightDef, ok := right[rightName]
if !ok {
// Didn't find right-hand definition, types are not equal
return false
}
if !comparer.areTypesStructurallyEqual(leftDef.Type(), rightDef.Type()) {
return false
}
}
return true
}
func (comparer *structuralComparer) areTypesStructurallyEqual(left astmodel.Type, right astmodel.Type) bool {
return left.Equals(right, comparer.equalityOverrides)
}
func (comparer *structuralComparer) equalObjectTypeStructure(left *astmodel.ObjectType, right *astmodel.ObjectType) bool {
if left == right {
return true // short circuit
}
// Create a copy of the properties with description removed as we don't care if it matches
leftProperties := comparer.simplifyProperties(left)
rightProperties := comparer.simplifyProperties(right)
return leftProperties.Equals(rightProperties, comparer.equalityOverrides)
}
func (comparer *structuralComparer) equalTypesReferencedByInternalTypeNames(
left astmodel.InternalTypeName,
right astmodel.InternalTypeName,
) bool {
// Look up the definitions referenced by the names and compare them for structural equality
leftDef, ok := comparer.definitions[left]
if !ok {
return false
}
rightDef, ok := comparer.definitions[right]
if !ok {
return false
}
return leftDef.Type().Equals(rightDef.Type(), comparer.equalityOverrides)
}
// simplifyProperties is a helper function used to strip out details of the properties we don't care about
// when evaluating for structural integrity
func (comparer *structuralComparer) simplifyProperties(o *astmodel.ObjectType) astmodel.PropertySet {
result := astmodel.NewPropertySet()
o.Properties().ForEach(func(def *astmodel.PropertyDefinition) {
result.Add(def.WithDescription(""))
})
return result
}
func (comparer *structuralComparer) findCommonPackage(
defs astmodel.TypeDefinitionSet,
) (astmodel.InternalPackageReference, error) {
pkgs := set.Make[astmodel.InternalPackageReference]()
for _, def := range defs {
pkgs.Add(def.Name().InternalPackageReference())
}
if len(pkgs) != 1 {
return nil, errors.Errorf(
"expected all definitions to be from the same package, but found %d packages instead",
len(pkgs))
}
return pkgs.Values()[0], nil
}