/
xds_resolver.go
566 lines (497 loc) · 19.9 KB
/
xds_resolver.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
/*
* Copyright 2019 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
// Package resolver implements the xds resolver, that does LDS and RDS to find
// the cluster to use.
package resolver
import (
"context"
"fmt"
"sync/atomic"
"google.golang.org/grpc/internal"
"google.golang.org/grpc/internal/grpclog"
"google.golang.org/grpc/internal/grpcrand"
"google.golang.org/grpc/internal/grpcsync"
"google.golang.org/grpc/internal/pretty"
iresolver "google.golang.org/grpc/internal/resolver"
"google.golang.org/grpc/internal/wrr"
"google.golang.org/grpc/resolver"
rinternal "google.golang.org/grpc/xds/internal/resolver/internal"
"google.golang.org/grpc/xds/internal/xdsclient"
"google.golang.org/grpc/xds/internal/xdsclient/bootstrap"
"google.golang.org/grpc/xds/internal/xdsclient/xdsresource"
)
// Scheme is the xDS resolver's scheme.
//
// TODO(easwars): Rename this package as xdsresolver so that this is accessed as
// xdsresolver.Scheme
const Scheme = "xds"
// newBuilderForTesting creates a new xds resolver builder using a specific xds
// bootstrap config, so tests can use multiple xds clients in different
// ClientConns at the same time.
func newBuilderForTesting(config []byte) (resolver.Builder, error) {
return &xdsResolverBuilder{
newXDSClient: func() (xdsclient.XDSClient, func(), error) {
return xdsclient.NewWithBootstrapContentsForTesting(config)
},
}, nil
}
func init() {
resolver.Register(&xdsResolverBuilder{})
internal.NewXDSResolverWithConfigForTesting = newBuilderForTesting
rinternal.NewWRR = wrr.NewRandom
rinternal.NewXDSClient = xdsclient.New
}
type xdsResolverBuilder struct {
newXDSClient func() (xdsclient.XDSClient, func(), error)
}
// Build helps implement the resolver.Builder interface.
//
// The xds bootstrap process is performed (and a new xds client is built) every
// time an xds resolver is built.
func (b *xdsResolverBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (_ resolver.Resolver, retErr error) {
r := &xdsResolver{
cc: cc,
activeClusters: make(map[string]*clusterInfo),
channelID: grpcrand.Uint64(),
}
defer func() {
if retErr != nil {
r.Close()
}
}()
r.logger = prefixLogger(r)
r.logger.Infof("Creating resolver for target: %+v", target)
// Initialize the serializer used to synchronize the following:
// - updates from the xDS client. This could lead to generation of new
// service config if resolution is complete.
// - completion of an RPC to a removed cluster causing the associated ref
// count to become zero, resulting in generation of new service config.
// - stopping of a config selector that results in generation of new service
// config.
ctx, cancel := context.WithCancel(context.Background())
r.serializer = grpcsync.NewCallbackSerializer(ctx)
r.serializerCancel = cancel
// Initialize the xDS client.
newXDSClient := rinternal.NewXDSClient.(func() (xdsclient.XDSClient, func(), error))
if b.newXDSClient != nil {
newXDSClient = b.newXDSClient
}
client, close, err := newXDSClient()
if err != nil {
return nil, fmt.Errorf("xds: failed to create xds-client: %v", err)
}
r.xdsClient = client
r.xdsClientClose = close
// Determine the listener resource name and start a watcher for it.
template, err := r.sanityChecksOnBootstrapConfig(target, opts, r.xdsClient)
if err != nil {
return nil, err
}
r.dataplaneAuthority = opts.Authority
r.ldsResourceName = bootstrap.PopulateResourceTemplate(template, target.Endpoint())
r.listenerWatcher = newListenerWatcher(r.ldsResourceName, r)
return r, nil
}
// Performs the following sanity checks:
// - Verifies that the bootstrap configuration is not empty.
// - Verifies that if xDS credentials are specified by the user, the
// bootstrap configuration contains certificate providers.
// - Verifies that if the provided dial target contains an authority, the
// bootstrap configuration contains server config for that authority.
//
// Returns the listener resource name template to use. If any of the above
// validations fail, a non-nil error is returned.
func (r *xdsResolver) sanityChecksOnBootstrapConfig(target resolver.Target, opts resolver.BuildOptions, client xdsclient.XDSClient) (string, error) {
bootstrapConfig := client.BootstrapConfig()
if bootstrapConfig == nil {
// This is never expected to happen after a successful xDS client
// creation. Defensive programming.
return "", fmt.Errorf("xds: bootstrap configuration is empty")
}
// Find the client listener template to use from the bootstrap config:
// - If authority is not set in the target, use the top level template
// - If authority is set, use the template from the authority map.
template := bootstrapConfig.ClientDefaultListenerResourceNameTemplate
if authority := target.URL.Host; authority != "" {
a := bootstrapConfig.Authorities[authority]
if a == nil {
return "", fmt.Errorf("xds: authority %q specified in dial target %q is not found in the bootstrap file", authority, target)
}
if a.ClientListenerResourceNameTemplate != "" {
// This check will never be false, because
// ClientListenerResourceNameTemplate is required to start with
// xdstp://, and has a default value (not an empty string) if unset.
template = a.ClientListenerResourceNameTemplate
}
}
return template, nil
}
// Name helps implement the resolver.Builder interface.
func (*xdsResolverBuilder) Scheme() string {
return Scheme
}
// xdsResolver implements the resolver.Resolver interface.
//
// It registers a watcher for ServiceConfig updates with the xdsClient object
// (which performs LDS/RDS queries for the same), and passes the received
// updates to the ClientConn.
type xdsResolver struct {
cc resolver.ClientConn
logger *grpclog.PrefixLogger
// The underlying xdsClient which performs all xDS requests and responses.
xdsClient xdsclient.XDSClient
xdsClientClose func()
// A random number which uniquely identifies the channel which owns this
// resolver.
channelID uint64
// All methods on the xdsResolver type except for the ones invoked by gRPC,
// i.e ResolveNow() and Close(), are guaranteed to execute in the context of
// this serializer's callback. And since the serializer guarantees mutual
// exclusion among these callbacks, we can get by without any mutexes to
// access all of the below defined state. The only exception is Close(),
// which does access some of this shared state, but it does so after
// cancelling the context passed to the serializer.
serializer *grpcsync.CallbackSerializer
serializerCancel context.CancelFunc
// dataplaneAuthority is the authority used for the data plane connections,
// which is also used to select the VirtualHost within the xDS
// RouteConfiguration. This is %-encoded to match with VirtualHost Domain
// in xDS RouteConfiguration.
dataplaneAuthority string
ldsResourceName string
listenerWatcher *listenerWatcher
listenerUpdateRecvd bool
currentListener xdsresource.ListenerUpdate
rdsResourceName string
routeConfigWatcher *routeConfigWatcher
routeConfigUpdateRecvd bool
currentRouteConfig xdsresource.RouteConfigUpdate
currentVirtualHost *xdsresource.VirtualHost // Matched virtual host for quick access.
// activeClusters is a map from cluster name to information about the
// cluster that includes a ref count and load balancing configuration.
activeClusters map[string]*clusterInfo
curConfigSelector *configSelector
}
// ResolveNow is a no-op at this point.
func (*xdsResolver) ResolveNow(o resolver.ResolveNowOptions) {}
func (r *xdsResolver) Close() {
// Cancel the context passed to the serializer and wait for any scheduled
// callbacks to complete. Canceling the context ensures that no new
// callbacks will be scheduled.
r.serializerCancel()
<-r.serializer.Done()
// Note that Close needs to check for nils even if some of them are always
// set in the constructor. This is because the constructor defers Close() in
// error cases, and the fields might not be set when the error happens.
if r.listenerWatcher != nil {
r.listenerWatcher.stop()
}
if r.routeConfigWatcher != nil {
r.routeConfigWatcher.stop()
}
if r.xdsClientClose != nil {
r.xdsClientClose()
}
r.logger.Infof("Shutdown")
}
// sendNewServiceConfig prunes active clusters, generates a new service config
// based on the current set of active clusters, and sends an update to the
// channel with that service config and the provided config selector. Returns
// false if an error occurs while generating the service config and the update
// cannot be sent.
//
// Only executed in the context of a serializer callback.
func (r *xdsResolver) sendNewServiceConfig(cs *configSelector) bool {
// Delete entries from r.activeClusters with zero references;
// otherwise serviceConfigJSON will generate a config including
// them.
r.pruneActiveClusters()
if cs == nil && len(r.activeClusters) == 0 {
// There are no clusters and we are sending a failing configSelector.
// Send an empty config, which picks pick-first, with no address, and
// puts the ClientConn into transient failure.
r.cc.UpdateState(resolver.State{ServiceConfig: r.cc.ParseServiceConfig("{}")})
return true
}
sc, err := serviceConfigJSON(r.activeClusters)
if err != nil {
// JSON marshal error; should never happen.
r.logger.Errorf("For Listener resource %q and RouteConfiguration resource %q, failed to marshal newly built service config: %v", r.ldsResourceName, r.rdsResourceName, err)
r.cc.ReportError(err)
return false
}
r.logger.Infof("For Listener resource %q and RouteConfiguration resource %q, generated service config: %v", r.ldsResourceName, r.rdsResourceName, pretty.FormatJSON(sc))
// Send the update to the ClientConn.
state := iresolver.SetConfigSelector(resolver.State{
ServiceConfig: r.cc.ParseServiceConfig(string(sc)),
}, cs)
r.cc.UpdateState(xdsclient.SetClient(state, r.xdsClient))
return true
}
// newConfigSelector creates a new config selector using the most recently
// received listener and route config updates. May add entries to
// r.activeClusters for previously-unseen clusters.
//
// Only executed in the context of a serializer callback.
func (r *xdsResolver) newConfigSelector() (*configSelector, error) {
cs := &configSelector{
r: r,
virtualHost: virtualHost{
httpFilterConfigOverride: r.currentVirtualHost.HTTPFilterConfigOverride,
retryConfig: r.currentVirtualHost.RetryConfig,
},
routes: make([]route, len(r.currentVirtualHost.Routes)),
clusters: make(map[string]*clusterInfo),
httpFilterConfig: r.currentListener.HTTPFilters,
}
for i, rt := range r.currentVirtualHost.Routes {
clusters := rinternal.NewWRR.(func() wrr.WRR)()
if rt.ClusterSpecifierPlugin != "" {
clusterName := clusterSpecifierPluginPrefix + rt.ClusterSpecifierPlugin
clusters.Add(&routeCluster{
name: clusterName,
}, 1)
ci := r.addOrGetActiveClusterInfo(clusterName)
ci.cfg = xdsChildConfig{ChildPolicy: balancerConfig(r.currentRouteConfig.ClusterSpecifierPlugins[rt.ClusterSpecifierPlugin])}
cs.clusters[clusterName] = ci
} else {
for cluster, wc := range rt.WeightedClusters {
clusterName := clusterPrefix + cluster
clusters.Add(&routeCluster{
name: clusterName,
httpFilterConfigOverride: wc.HTTPFilterConfigOverride,
}, int64(wc.Weight))
ci := r.addOrGetActiveClusterInfo(clusterName)
ci.cfg = xdsChildConfig{ChildPolicy: newBalancerConfig(cdsName, cdsBalancerConfig{Cluster: cluster})}
cs.clusters[clusterName] = ci
}
}
cs.routes[i].clusters = clusters
var err error
cs.routes[i].m, err = xdsresource.RouteToMatcher(rt)
if err != nil {
return nil, err
}
cs.routes[i].actionType = rt.ActionType
if rt.MaxStreamDuration == nil {
cs.routes[i].maxStreamDuration = r.currentListener.MaxStreamDuration
} else {
cs.routes[i].maxStreamDuration = *rt.MaxStreamDuration
}
cs.routes[i].httpFilterConfigOverride = rt.HTTPFilterConfigOverride
cs.routes[i].retryConfig = rt.RetryConfig
cs.routes[i].hashPolicies = rt.HashPolicies
}
// Account for this config selector's clusters. Do this after no further
// errors may occur. Note: cs.clusters are pointers to entries in
// activeClusters.
for _, ci := range cs.clusters {
atomic.AddInt32(&ci.refCount, 1)
}
return cs, nil
}
// pruneActiveClusters deletes entries in r.activeClusters with zero
// references.
func (r *xdsResolver) pruneActiveClusters() {
for cluster, ci := range r.activeClusters {
if atomic.LoadInt32(&ci.refCount) == 0 {
delete(r.activeClusters, cluster)
}
}
}
func (r *xdsResolver) addOrGetActiveClusterInfo(name string) *clusterInfo {
ci := r.activeClusters[name]
if ci != nil {
return ci
}
ci = &clusterInfo{refCount: 0}
r.activeClusters[name] = ci
return ci
}
type clusterInfo struct {
// number of references to this cluster; accessed atomically
refCount int32
// cfg is the child configuration for this cluster, containing either the
// csp config or the cds cluster config.
cfg xdsChildConfig
}
// Determines if the xdsResolver has received all required configuration, i.e
// Listener and RouteConfiguration resources, from the management server, and
// whether a matching virtual host was found in the RouteConfiguration resource.
func (r *xdsResolver) resolutionComplete() bool {
return r.listenerUpdateRecvd && r.routeConfigUpdateRecvd && r.currentVirtualHost != nil
}
// onResolutionComplete performs the following actions when resolution is
// complete, i.e Listener and RouteConfiguration resources have been received
// from the management server and a matching virtual host is found in the
// latter.
// - creates a new config selector (this involves incrementing references to
// clusters owned by this config selector).
// - stops the old config selector (this involves decrementing references to
// clusters owned by this config selector).
// - prunes active clusters and pushes a new service config to the channel.
// - updates the current config selector used by the resolver.
//
// Only executed in the context of a serializer callback.
func (r *xdsResolver) onResolutionComplete() {
if !r.resolutionComplete() {
return
}
cs, err := r.newConfigSelector()
if err != nil {
r.logger.Warningf("Failed to build a config selector for resource %q: %v", r.ldsResourceName, err)
r.cc.ReportError(err)
return
}
if !r.sendNewServiceConfig(cs) {
// JSON error creating the service config (unexpected); erase
// this config selector and ignore this update, continuing with
// the previous config selector.
cs.stop()
return
}
r.curConfigSelector.stop()
r.curConfigSelector = cs
}
func (r *xdsResolver) applyRouteConfigUpdate(update xdsresource.RouteConfigUpdate) {
matchVh := xdsresource.FindBestMatchingVirtualHost(r.dataplaneAuthority, update.VirtualHosts)
if matchVh == nil {
r.onError(fmt.Errorf("no matching virtual host found for %q", r.dataplaneAuthority))
return
}
r.currentRouteConfig = update
r.currentVirtualHost = matchVh
r.routeConfigUpdateRecvd = true
r.onResolutionComplete()
}
// onError propagates the error up to the channel. And since this is invoked
// only for non resource-not-found errors, we don't have to update resolver
// state and we can keep using the old config.
//
// Only executed in the context of a serializer callback.
func (r *xdsResolver) onError(err error) {
r.cc.ReportError(err)
}
// Contains common functionality to be executed when resources of either type
// are removed.
//
// Only executed in the context of a serializer callback.
func (r *xdsResolver) onResourceNotFound() {
// We cannot remove clusters from the service config that have ongoing RPCs.
// Instead, what we can do is to send an erroring (nil) config selector
// along with normal service config. This will ensure that new RPCs will
// fail, and once the active RPCs complete, the reference counts on the
// clusters will come down to zero. At that point, we will send an empty
// service config with no addresses. This results in the pick-first
// LB policy being configured on the channel, and since there are no
// address, pick-first will put the channel in TRANSIENT_FAILURE.
r.sendNewServiceConfig(nil)
// Stop and dereference the active config selector, if one exists.
r.curConfigSelector.stop()
r.curConfigSelector = nil
}
// Only executed in the context of a serializer callback.
func (r *xdsResolver) onListenerResourceUpdate(update xdsresource.ListenerUpdate) {
if r.logger.V(2) {
r.logger.Infof("Received update for Listener resource %q: %v", r.ldsResourceName, pretty.ToJSON(update))
}
r.currentListener = update
r.listenerUpdateRecvd = true
if update.InlineRouteConfig != nil {
// If there was a previous route config watcher because of a non-inline
// route configuration, cancel it.
r.rdsResourceName = ""
if r.routeConfigWatcher != nil {
r.routeConfigWatcher.stop()
r.routeConfigWatcher = nil
}
r.applyRouteConfigUpdate(*update.InlineRouteConfig)
return
}
// We get here only if there was no inline route configuration.
// If the route config name has not changed, send an update with existing
// route configuration and the newly received listener configuration.
if r.rdsResourceName == update.RouteConfigName {
r.onResolutionComplete()
return
}
// If the route config name has changed, cancel the old watcher and start a
// new one. At this point, since we have not yet resolved the new route
// config name, we don't send an update to the channel, and therefore
// continue using the old route configuration (if received) until the new
// one is received.
r.rdsResourceName = update.RouteConfigName
if r.routeConfigWatcher != nil {
r.routeConfigWatcher.stop()
r.currentVirtualHost = nil
r.routeConfigUpdateRecvd = false
}
r.routeConfigWatcher = newRouteConfigWatcher(r.rdsResourceName, r)
}
func (r *xdsResolver) onListenerResourceError(err error) {
if r.logger.V(2) {
r.logger.Infof("Received error for Listener resource %q: %v", r.ldsResourceName, err)
}
r.onError(err)
}
// Only executed in the context of a serializer callback.
func (r *xdsResolver) onListenerResourceNotFound() {
if r.logger.V(2) {
r.logger.Infof("Received resource-not-found-error for Listener resource %q", r.ldsResourceName)
}
r.listenerUpdateRecvd = false
if r.routeConfigWatcher != nil {
r.routeConfigWatcher.stop()
}
r.rdsResourceName = ""
r.currentVirtualHost = nil
r.routeConfigUpdateRecvd = false
r.routeConfigWatcher = nil
r.onResourceNotFound()
}
// Only executed in the context of a serializer callback.
func (r *xdsResolver) onRouteConfigResourceUpdate(name string, update xdsresource.RouteConfigUpdate) {
if r.logger.V(2) {
r.logger.Infof("Received update for RouteConfiguration resource %q: %v", name, pretty.ToJSON(update))
}
if r.rdsResourceName != name {
// Drop updates from canceled watchers.
return
}
r.applyRouteConfigUpdate(update)
}
// Only executed in the context of a serializer callback.
func (r *xdsResolver) onRouteConfigResourceError(name string, err error) {
if r.logger.V(2) {
r.logger.Infof("Received error for RouteConfiguration resource %q: %v", name, err)
}
r.onError(err)
}
// Only executed in the context of a serializer callback.
func (r *xdsResolver) onRouteConfigResourceNotFound(name string) {
if r.logger.V(2) {
r.logger.Infof("Received resource-not-found-error for RouteConfiguration resource %q", name)
}
if r.rdsResourceName != name {
return
}
r.onResourceNotFound()
}
// Only executed in the context of a serializer callback.
func (r *xdsResolver) onClusterRefDownToZero() {
r.sendNewServiceConfig(r.curConfigSelector)
}