diff --git a/packages/grpc-js-xds/gulpfile.ts b/packages/grpc-js-xds/gulpfile.ts index 4ee6ac2c5..f2e77b8bd 100644 --- a/packages/grpc-js-xds/gulpfile.ts +++ b/packages/grpc-js-xds/gulpfile.ts @@ -61,6 +61,7 @@ const cleanAll = gulp.parallel(clean); const compile = checkTask(() => execNpmCommand('compile')); const runTests = checkTask(() => { + process.env.GRPC_EXPERIMENTAL_XDS_FEDERATION = 'true'; return gulp.src(`${outDir}/test/**/*.js`) .pipe(mocha({reporter: 'mocha-jenkins-reporter', require: ['ts-node/register']})); diff --git a/packages/grpc-js-xds/package.json b/packages/grpc-js-xds/package.json index 7c735b652..7fd7e700d 100644 --- a/packages/grpc-js-xds/package.json +++ b/packages/grpc-js-xds/package.json @@ -45,7 +45,8 @@ "dependencies": { "@grpc/proto-loader": "^0.6.0", "google-auth-library": "^7.0.2", - "re2-wasm": "^1.0.1" + "re2-wasm": "^1.0.1", + "vscode-uri": "^3.0.7" }, "peerDependencies": { "@grpc/grpc-js": "~1.8.0" diff --git a/packages/grpc-js-xds/src/csds.ts b/packages/grpc-js-xds/src/csds.ts index 95a3bf7b7..f9fac8569 100644 --- a/packages/grpc-js-xds/src/csds.ts +++ b/packages/grpc-js-xds/src/csds.ts @@ -15,19 +15,18 @@ * */ -import { Node } from "./generated/envoy/config/core/v3/Node"; import { ClientConfig, _envoy_service_status_v3_ClientConfig_GenericXdsConfig as GenericXdsConfig } from "./generated/envoy/service/status/v3/ClientConfig"; import { ClientStatusDiscoveryServiceHandlers } from "./generated/envoy/service/status/v3/ClientStatusDiscoveryService"; import { ClientStatusRequest__Output } from "./generated/envoy/service/status/v3/ClientStatusRequest"; import { ClientStatusResponse } from "./generated/envoy/service/status/v3/ClientStatusResponse"; import { Timestamp } from "./generated/google/protobuf/Timestamp"; -import { AdsTypeUrl, CDS_TYPE_URL, EDS_TYPE_URL, LDS_TYPE_URL, RDS_TYPE_URL } from "./resources"; -import { HandleResponseResult } from "./xds-stream-state/xds-stream-state"; +import { xdsResourceNameToString } from "./resources"; import { sendUnaryData, ServerDuplexStream, ServerUnaryCall, status, experimental, loadPackageDefinition, logVerbosity } from '@grpc/grpc-js'; import { loadSync } from "@grpc/proto-loader"; import { ProtoGrpcType as CsdsProtoGrpcType } from "./generated/csds"; import registerAdminService = experimental.registerAdminService; +import { XdsClient } from "./xds-client"; const TRACER_NAME = 'csds'; @@ -47,115 +46,47 @@ function dateToProtoTimestamp(date?: Date | null): Timestamp | null { } } -let clientNode: Node | null = null; +const registeredClients: XdsClient[] = []; -const configStatus = { - [EDS_TYPE_URL]: new Map(), - [CDS_TYPE_URL]: new Map(), - [RDS_TYPE_URL]: new Map(), - [LDS_TYPE_URL]: new Map() -}; - -/** - * This function only accepts a v3 Node message, because we are only supporting - * v3 CSDS and it only handles v3 Nodes. If the client is actually using v2 xDS - * APIs, it should just provide the equivalent v3 Node message. - * @param node The Node message for the client that is requesting resources - */ -export function setCsdsClientNode(node: Node) { - clientNode = node; +export function registerXdsClientWithCsds(client: XdsClient) { + registeredClients.push(client); } -/** - * Update the config status maps from the list of names of requested resources - * for a specific type URL. These lists are the source of truth for determining - * what resources will be listed in the CSDS response. Any resource that is not - * in this list will never actually be applied anywhere. - * @param typeUrl The resource type URL - * @param names The list of resource names that are being requested - */ -export function updateCsdsRequestedNameList(typeUrl: AdsTypeUrl, names: string[]) { - trace('Update type URL ' + typeUrl + ' with names [' + names + ']'); - const currentTime = dateToProtoTimestamp(new Date()); - const configMap = configStatus[typeUrl]; - for (const name of names) { - if (!configMap.has(name)) { - configMap.set(name, { - type_url: typeUrl, - name: name, - last_updated: currentTime, - client_status: 'REQUESTED' - }); - } - } - for (const name of configMap.keys()) { - if (!names.includes(name)) { - configMap.delete(name); - } - } -} - -/** - * Update the config status maps from the result of parsing a single ADS - * response. All resources that validated are considered "ACKED", and all - * resources that failed validation are considered "NACKED". - * @param typeUrl The type URL of resources in this response - * @param versionInfo The version info field from this response - * @param updates The lists of resources that passed and failed validation - */ -export function updateCsdsResourceResponse(typeUrl: AdsTypeUrl, versionInfo: string, updates: HandleResponseResult) { - const currentTime = dateToProtoTimestamp(new Date()); - const configMap = configStatus[typeUrl]; - for (const {name, raw} of updates.accepted) { - const mapEntry = configMap.get(name); - if (mapEntry) { - trace('Updated ' + typeUrl + ' resource ' + name + ' to state ACKED'); - mapEntry.client_status = 'ACKED'; - mapEntry.version_info = versionInfo; - mapEntry.xds_config = raw; - mapEntry.error_state = null; - mapEntry.last_updated = currentTime; +function getCurrentConfigList(): ClientConfig[] { + const result: ClientConfig[] = []; + for (const client of registeredClients) { + if (!client.adsNode) { + continue; } - } - for (const {name, error, raw} of updates.rejected) { - const mapEntry = configMap.get(name); - if (mapEntry) { - trace('Updated ' + typeUrl + ' resource ' + name + ' to state NACKED'); - mapEntry.client_status = 'NACKED'; - mapEntry.error_state = { - failed_configuration: raw, - last_update_attempt: currentTime, - details: error, - version_info: versionInfo - }; - } - } - for (const name of updates.missing) { - const mapEntry = configMap.get(name); - if (mapEntry) { - trace('Updated ' + typeUrl + ' resource ' + name + ' to state DOES_NOT_EXIST'); - mapEntry.client_status = 'DOES_NOT_EXIST'; - mapEntry.version_info = versionInfo; - mapEntry.xds_config = null; - mapEntry.error_state = null; - mapEntry.last_updated = currentTime; - } - } -} - -function getCurrentConfig(): ClientConfig { - const genericConfigList: GenericXdsConfig[] = []; - for (const configMap of Object.values(configStatus)) { - for (const configValue of configMap.values()) { - genericConfigList.push(configValue); + const genericConfigList: GenericXdsConfig[] = []; + for (const [authority, authorityState] of client.authorityStateMap) { + for (const [type, typeMap] of authorityState.resourceMap) { + for (const [key, resourceState] of typeMap) { + const typeUrl = type.getTypeUrl(); + const meta = resourceState.meta; + genericConfigList.push({ + name: xdsResourceNameToString({authority, key}, typeUrl), + type_url: typeUrl, + client_status: meta.clientStatus, + version_info: meta.version, + xds_config: meta.clientStatus === 'ACKED' ? meta.rawResource : undefined, + last_updated: meta.updateTime ? dateToProtoTimestamp(meta.updateTime) : undefined, + error_state: meta.clientStatus === 'NACKED' ? { + details: meta.failedDetails, + failed_configuration: meta.rawResource, + last_update_attempt: meta.failedUpdateTime ? dateToProtoTimestamp(meta.failedUpdateTime) : undefined, + version_info: meta.failedVersion + } : undefined + }); + } + } } + result.push({ + node: client.adsNode, + generic_xds_configs: genericConfigList + }); } - const config = { - node: clientNode, - generic_xds_configs: genericConfigList - }; - trace('Sending current config ' + JSON.stringify(config, undefined, 2)); - return config; + return result; } const csdsImplementation: ClientStatusDiscoveryServiceHandlers = { @@ -169,7 +100,7 @@ const csdsImplementation: ClientStatusDiscoveryServiceHandlers = { return; } callback(null, { - config: [getCurrentConfig()] + config: getCurrentConfigList() }); }, StreamClientStatus(call: ServerDuplexStream) { @@ -182,7 +113,7 @@ const csdsImplementation: ClientStatusDiscoveryServiceHandlers = { return; } call.write({ - config: [getCurrentConfig()] + config: getCurrentConfigList() }); }); call.on('end', () => { @@ -211,4 +142,4 @@ const csdsServiceDefinition = csdsGrpcObject.envoy.service.status.v3.ClientStatu export function setup() { registerAdminService(() => csdsServiceDefinition, () => csdsImplementation); -} \ No newline at end of file +} diff --git a/packages/grpc-js-xds/src/environment.ts b/packages/grpc-js-xds/src/environment.ts index 7ec0fd187..32c9f28ba 100644 --- a/packages/grpc-js-xds/src/environment.ts +++ b/packages/grpc-js-xds/src/environment.ts @@ -17,4 +17,5 @@ export const EXPERIMENTAL_FAULT_INJECTION = (process.env.GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION ?? 'true') === 'true'; export const EXPERIMENTAL_OUTLIER_DETECTION = (process.env.GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION ?? 'true') === 'true'; -export const EXPERIMENTAL_RETRY = (process.env.GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY ?? 'true') === 'true'; \ No newline at end of file +export const EXPERIMENTAL_RETRY = (process.env.GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY ?? 'true') === 'true'; +export const EXPERIMENTAL_FEDERATION = (process.env.GRPC_EXPERIMENTAL_XDS_FEDERATION ?? 'false') === 'true'; diff --git a/packages/grpc-js-xds/src/load-balancer-cds.ts b/packages/grpc-js-xds/src/load-balancer-cds.ts index 4c1c576f3..6f791299c 100644 --- a/packages/grpc-js-xds/src/load-balancer-cds.ts +++ b/packages/grpc-js-xds/src/load-balancer-cds.ts @@ -16,7 +16,7 @@ */ import { connectivityState, status, Metadata, logVerbosity, experimental } from '@grpc/grpc-js'; -import { getSingletonXdsClient, XdsClient } from './xds-client'; +import { getSingletonXdsClient, Watcher, XdsClient } from './xds-client'; import { Cluster__Output } from './generated/envoy/config/cluster/v3/Cluster'; import SubchannelAddress = experimental.SubchannelAddress; import UnavailablePicker = experimental.UnavailablePicker; @@ -29,12 +29,12 @@ import OutlierDetectionLoadBalancingConfig = experimental.OutlierDetectionLoadBa import SuccessRateEjectionConfig = experimental.SuccessRateEjectionConfig; import FailurePercentageEjectionConfig = experimental.FailurePercentageEjectionConfig; import QueuePicker = experimental.QueuePicker; -import { Watcher } from './xds-stream-state/xds-stream-state'; import { OutlierDetection__Output } from './generated/envoy/config/cluster/v3/OutlierDetection'; import { Duration__Output } from './generated/google/protobuf/Duration'; import { EXPERIMENTAL_OUTLIER_DETECTION } from './environment'; import { DiscoveryMechanism, XdsClusterResolverChildPolicyHandler, XdsClusterResolverLoadBalancingConfig } from './load-balancer-xds-cluster-resolver'; import { CLUSTER_CONFIG_TYPE_URL, decodeSingleResource } from './resources'; +import { CdsUpdate, ClusterResourceType, OutlierDetectionUpdate } from './xds-resource-type/cluster-resource-type'; const TRACER_NAME = 'cds_balancer'; @@ -76,7 +76,7 @@ function durationToMs(duration: Duration__Output): number { return (Number(duration.seconds) * 1_000 + duration.nanos / 1_000_000) | 0; } -function translateOutlierDetectionConfig(outlierDetection: OutlierDetection__Output | null): OutlierDetectionLoadBalancingConfig | undefined { +function translateOutlierDetectionConfig(outlierDetection: OutlierDetectionUpdate | undefined): OutlierDetectionLoadBalancingConfig | undefined { if (!EXPERIMENTAL_OUTLIER_DETECTION) { return undefined; } @@ -84,42 +84,20 @@ function translateOutlierDetectionConfig(outlierDetection: OutlierDetection__Out /* No-op outlier detection config, with all fields unset. */ return new OutlierDetectionLoadBalancingConfig(null, null, null, null, null, null, []); } - let successRateConfig: Partial | null = null; - /* Success rate ejection is enabled by default, so we only disable it if - * enforcing_success_rate is set and it has the value 0 */ - if (!outlierDetection.enforcing_success_rate || outlierDetection.enforcing_success_rate.value > 0) { - successRateConfig = { - enforcement_percentage: outlierDetection.enforcing_success_rate?.value, - minimum_hosts: outlierDetection.success_rate_minimum_hosts?.value, - request_volume: outlierDetection.success_rate_request_volume?.value, - stdev_factor: outlierDetection.success_rate_stdev_factor?.value - }; - } - let failurePercentageConfig: Partial | null = null; - /* Failure percentage ejection is disabled by default, so we only enable it - * if enforcing_failure_percentage is set and it has a value greater than 0 */ - if (outlierDetection.enforcing_failure_percentage && outlierDetection.enforcing_failure_percentage.value > 0) { - failurePercentageConfig = { - enforcement_percentage: outlierDetection.enforcing_failure_percentage.value, - minimum_hosts: outlierDetection.failure_percentage_minimum_hosts?.value, - request_volume: outlierDetection.failure_percentage_request_volume?.value, - threshold: outlierDetection.failure_percentage_threshold?.value - } - } return new OutlierDetectionLoadBalancingConfig( - outlierDetection.interval ? durationToMs(outlierDetection.interval) : null, - outlierDetection.base_ejection_time ? durationToMs(outlierDetection.base_ejection_time) : null, - outlierDetection.max_ejection_time ? durationToMs(outlierDetection.max_ejection_time) : null, - outlierDetection.max_ejection_percent?.value ?? null, - successRateConfig, - failurePercentageConfig, + outlierDetection.intervalMs, + outlierDetection.baseEjectionTimeMs, + outlierDetection.maxEjectionTimeMs, + outlierDetection.maxEjectionPercent, + outlierDetection.successRateConfig, + outlierDetection.failurePercentageConfig, [] ); } interface ClusterEntry { - watcher: Watcher; - latestUpdate?: Cluster__Output; + watcher: Watcher; + latestUpdate?: CdsUpdate; children: string[]; } @@ -144,34 +122,19 @@ function isClusterTreeFullyUpdated(tree: ClusterTree, root: string): boolean { return true; } -function generateDiscoveryMechanismForCluster(config: Cluster__Output): DiscoveryMechanism { - let maxConcurrentRequests: number | undefined = undefined; - for (const threshold of config.circuit_breakers?.thresholds ?? []) { - if (threshold.priority === 'DEFAULT') { - maxConcurrentRequests = threshold.max_requests?.value; - } - } - if (config.type === 'EDS') { - // EDS cluster - return { - cluster: config.name, - lrs_load_reporting_server_name: config.lrs_server?.self ? '' : undefined, - max_concurrent_requests: maxConcurrentRequests, - type: 'EDS', - eds_service_name: config.eds_cluster_config!.service_name === '' ? undefined : config.eds_cluster_config!.service_name, - outlier_detection: translateOutlierDetectionConfig(config.outlier_detection) - }; - } else { - // Logical DNS cluster - const socketAddress = config.load_assignment!.endpoints[0].lb_endpoints[0].endpoint!.address!.socket_address!; - return { - cluster: config.name, - lrs_load_reporting_server_name: config.lrs_server?.self ? '' : undefined, - max_concurrent_requests: maxConcurrentRequests, - type: 'LOGICAL_DNS', - dns_hostname: `${socketAddress.address}:${socketAddress.port_value}` - }; +function generateDiscoverymechanismForCdsUpdate(config: CdsUpdate): DiscoveryMechanism { + if (config.type === 'AGGREGATE') { + throw new Error('Cannot generate DiscoveryMechanism for AGGREGATE cluster'); } + return { + cluster: config.name, + lrs_load_reporting_server: config.lrsLoadReportingServer, + max_concurrent_requests: config.maxConcurrentRequests, + type: config.type, + eds_service_name: config.edsServiceName, + dns_hostname: config.dnsHostname, + outlier_detection: translateOutlierDetectionConfig(config.outlierDetectionUpdate) + }; } const RECURSION_DEPTH_LIMIT = 15; @@ -203,7 +166,7 @@ function getDiscoveryMechanismList(tree: ClusterTree, root: string): DiscoveryMe trace('Visit leaf ' + node); // individual cluster const config = tree[node].latestUpdate!; - return [generateDiscoveryMechanismForCluster(config)]; + return [generateDiscoverymechanismForCdsUpdate(config)]; } } return getDiscoveryMechanismListHelper(root, 0); @@ -231,11 +194,11 @@ export class CdsLoadBalancer implements LoadBalancer { return; } trace('Adding watcher for cluster ' + cluster); - const watcher: Watcher = { - onValidUpdate: (update) => { + const watcher: Watcher = new Watcher({ + onResourceChanged: (update) => { this.clusterTree[cluster].latestUpdate = update; - if (update.cluster_discovery_type === 'cluster_type') { - const children = decodeSingleResource(CLUSTER_CONFIG_TYPE_URL, update.cluster_type!.typed_config!.value).clusters; + if (update.type === 'AGGREGATE') { + const children = update.aggregateChildren trace('Received update for aggregate cluster ' + cluster + ' with children [' + children + ']'); this.clusterTree[cluster].children = children; children.forEach(child => this.addCluster(child)); @@ -263,6 +226,7 @@ export class CdsLoadBalancer implements LoadBalancer { } }, onResourceDoesNotExist: () => { + trace('Received onResourceDoesNotExist update for cluster ' + cluster); if (cluster in this.clusterTree) { this.clusterTree[cluster].latestUpdate = undefined; this.clusterTree[cluster].children = []; @@ -270,8 +234,9 @@ export class CdsLoadBalancer implements LoadBalancer { this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `CDS resource ${cluster} does not exist`, metadata: new Metadata()})); this.childBalancer.destroy(); }, - onTransientError: (statusObj) => { + onError: (statusObj) => { if (!this.updatedChild) { + trace('Transitioning to transient failure due to onError update for cluster' + cluster); this.channelControlHelper.updateState( connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({ @@ -282,19 +247,23 @@ export class CdsLoadBalancer implements LoadBalancer { ); } } - }; + }); this.clusterTree[cluster] = { watcher: watcher, children: [] }; - this.xdsClient?.addClusterWatcher(cluster, watcher); + if (this.xdsClient) { + ClusterResourceType.startWatch(this.xdsClient, cluster, watcher); + } } private removeCluster(cluster: string) { if (!(cluster in this.clusterTree)) { return; } - this.xdsClient?.removeClusterWatcher(cluster, this.clusterTree[cluster].watcher); + if (this.xdsClient) { + ClusterResourceType.cancelWatch(this.xdsClient, cluster, this.clusterTree[cluster].watcher); + } delete this.clusterTree[cluster]; } diff --git a/packages/grpc-js-xds/src/load-balancer-lrs.ts b/packages/grpc-js-xds/src/load-balancer-lrs.ts index 9610ea834..a2deb72c3 100644 --- a/packages/grpc-js-xds/src/load-balancer-lrs.ts +++ b/packages/grpc-js-xds/src/load-balancer-lrs.ts @@ -17,6 +17,7 @@ import { connectivityState as ConnectivityState, StatusObject, status as Status, experimental } from '@grpc/grpc-js'; import { Locality__Output } from './generated/envoy/config/core/v3/Locality'; +import { validateXdsServerConfig, XdsServerConfig } from './xds-bootstrap'; import { XdsClusterLocalityStats, XdsClient, getSingletonXdsClient } from './xds-client'; import LoadBalancer = experimental.LoadBalancer; import ChannelControlHelper = experimental.ChannelControlHelper; @@ -46,14 +47,14 @@ export class LrsLoadBalancingConfig implements LoadBalancingConfig { [TYPE_NAME]: { cluster_name: this.clusterName, eds_service_name: this.edsServiceName, - lrs_load_reporting_server_name: this.lrsLoadReportingServerName, + lrs_load_reporting_server_name: this.lrsLoadReportingServer, locality: this.locality, child_policy: this.childPolicy.map(policy => policy.toJsonObject()) } } } - constructor(private clusterName: string, private edsServiceName: string, private lrsLoadReportingServerName: string, private locality: Locality__Output, private childPolicy: LoadBalancingConfig[]) {} + constructor(private clusterName: string, private edsServiceName: string, private lrsLoadReportingServer: XdsServerConfig, private locality: Locality__Output, private childPolicy: LoadBalancingConfig[]) {} getClusterName() { return this.clusterName; @@ -63,8 +64,8 @@ export class LrsLoadBalancingConfig implements LoadBalancingConfig { return this.edsServiceName; } - getLrsLoadReportingServerName() { - return this.lrsLoadReportingServerName; + getLrsLoadReportingServer() { + return this.lrsLoadReportingServer; } getLocality() { @@ -82,9 +83,6 @@ export class LrsLoadBalancingConfig implements LoadBalancingConfig { if (!('eds_service_name' in obj && typeof obj.eds_service_name === 'string')) { throw new Error('lrs config must have a string field eds_service_name'); } - if (!('lrs_load_reporting_server_name' in obj && typeof obj.lrs_load_reporting_server_name === 'string')) { - throw new Error('lrs config must have a string field lrs_load_reporting_server_name'); - } if (!('locality' in obj && obj.locality !== null && typeof obj.locality === 'object')) { throw new Error('lrs config must have an object field locality'); } @@ -100,7 +98,7 @@ export class LrsLoadBalancingConfig implements LoadBalancingConfig { if (!('child_policy' in obj && Array.isArray(obj.child_policy))) { throw new Error('lrs config must have a child_policy array'); } - return new LrsLoadBalancingConfig(obj.cluster_name, obj.eds_service_name, obj.lrs_load_reporting_server_name, { + return new LrsLoadBalancingConfig(obj.cluster_name, obj.eds_service_name, validateXdsServerConfig(obj.lrs_load_reporting_server), { region: obj.locality.region ?? '', zone: obj.locality.zone ?? '', sub_zone: obj.locality.sub_zone ?? '' @@ -170,7 +168,7 @@ export class LrsLoadBalancer implements LoadBalancer { return; } this.localityStatsReporter = (attributes.xdsClient as XdsClient).addClusterLocalityStats( - lbConfig.getLrsLoadReportingServerName(), + lbConfig.getLrsLoadReportingServer(), lbConfig.getClusterName(), lbConfig.getEdsServiceName(), lbConfig.getLocality() diff --git a/packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts b/packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts index 16f2cc3c9..edfd52016 100644 --- a/packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts +++ b/packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts @@ -16,6 +16,7 @@ */ import { experimental, logVerbosity, status as Status, Metadata, connectivityState } from "@grpc/grpc-js"; +import { validateXdsServerConfig, XdsServerConfig } from "./xds-bootstrap"; import { getSingletonXdsClient, XdsClient, XdsClusterDropStats } from "./xds-client"; import LoadBalancingConfig = experimental.LoadBalancingConfig; @@ -72,15 +73,15 @@ export class XdsClusterImplLoadBalancingConfig implements LoadBalancingConfig { if (this.edsServiceName !== undefined) { jsonObj.eds_service_name = this.edsServiceName; } - if (this.lrsLoadReportingServerName !== undefined) { - jsonObj.lrs_load_reporting_server_name = this.lrsLoadReportingServerName; + if (this.lrsLoadReportingServer !== undefined) { + jsonObj.lrs_load_reporting_server_name = this.lrsLoadReportingServer; } return { [TYPE_NAME]: jsonObj }; } - constructor(private cluster: string, private dropCategories: DropCategory[], private childPolicy: LoadBalancingConfig[], private edsServiceName?: string, private lrsLoadReportingServerName?: string, maxConcurrentRequests?: number) { + constructor(private cluster: string, private dropCategories: DropCategory[], private childPolicy: LoadBalancingConfig[], private edsServiceName?: string, private lrsLoadReportingServer?: XdsServerConfig, maxConcurrentRequests?: number) { this.maxConcurrentRequests = maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS; } @@ -92,8 +93,8 @@ export class XdsClusterImplLoadBalancingConfig implements LoadBalancingConfig { return this.edsServiceName; } - getLrsLoadReportingServerName() { - return this.lrsLoadReportingServerName; + getLrsLoadReportingServer() { + return this.lrsLoadReportingServer; } getMaxConcurrentRequests() { @@ -115,9 +116,6 @@ export class XdsClusterImplLoadBalancingConfig implements LoadBalancingConfig { if ('eds_service_name' in obj && !(obj.eds_service_name === undefined || typeof obj.eds_service_name === 'string')) { throw new Error('xds_cluster_impl config eds_service_name field must be a string if provided'); } - if ('lrs_load_reporting_server_name' in obj && (!obj.lrs_load_reporting_server_name === undefined || typeof obj.lrs_load_reporting_server_name === 'string')) { - throw new Error('xds_cluster_impl config lrs_load_reporting_server_name must be a string if provided'); - } if ('max_concurrent_requests' in obj && (!obj.max_concurrent_requests === undefined || typeof obj.max_concurrent_requests === 'number')) { throw new Error('xds_cluster_impl config max_concurrent_requests must be a number if provided'); } @@ -127,7 +125,7 @@ export class XdsClusterImplLoadBalancingConfig implements LoadBalancingConfig { if (!('child_policy' in obj && Array.isArray(obj.child_policy))) { throw new Error('xds_cluster_impl config must have an array field child_policy'); } - return new XdsClusterImplLoadBalancingConfig(obj.cluster, obj.drop_categories.map(validateDropCategory), obj.child_policy.map(validateLoadBalancingConfig), obj.eds_service_name, obj.lrs_load_reporting_server_name, obj.max_concurrent_requests); + return new XdsClusterImplLoadBalancingConfig(obj.cluster, obj.drop_categories.map(validateDropCategory), obj.child_policy.map(validateLoadBalancingConfig), obj.eds_service_name, obj.lrs_load_reporting_server ? validateXdsServerConfig(obj.lrs_load_reporting_server) : undefined, obj.max_concurrent_requests); } } @@ -245,9 +243,9 @@ class XdsClusterImplBalancer implements LoadBalancer { this.latestConfig = lbConfig; this.xdsClient = attributes.xdsClient as XdsClient; - if (lbConfig.getLrsLoadReportingServerName()) { + if (lbConfig.getLrsLoadReportingServer()) { this.clusterDropStats = this.xdsClient.addClusterDropStats( - lbConfig.getLrsLoadReportingServerName()!, + lbConfig.getLrsLoadReportingServer()!, lbConfig.getCluster(), lbConfig.getEdsServiceName() ?? '' ); @@ -271,4 +269,4 @@ class XdsClusterImplBalancer implements LoadBalancer { export function setup() { registerLoadBalancerType(TYPE_NAME, XdsClusterImplBalancer, XdsClusterImplLoadBalancingConfig); -} \ No newline at end of file +} diff --git a/packages/grpc-js-xds/src/load-balancer-xds-cluster-resolver.ts b/packages/grpc-js-xds/src/load-balancer-xds-cluster-resolver.ts index 8c01138b5..40f67c476 100644 --- a/packages/grpc-js-xds/src/load-balancer-xds-cluster-resolver.ts +++ b/packages/grpc-js-xds/src/load-balancer-xds-cluster-resolver.ts @@ -23,9 +23,8 @@ import { ClusterLoadAssignment__Output } from "./generated/envoy/config/endpoint import { LrsLoadBalancingConfig } from "./load-balancer-lrs"; import { LocalitySubchannelAddress, PriorityChild, PriorityLoadBalancingConfig } from "./load-balancer-priority"; import { WeightedTarget, WeightedTargetLoadBalancingConfig } from "./load-balancer-weighted-target"; -import { getSingletonXdsClient, XdsClient } from "./xds-client"; +import { getSingletonXdsClient, Watcher, XdsClient } from "./xds-client"; import { DropCategory, XdsClusterImplLoadBalancingConfig } from "./load-balancer-xds-cluster-impl"; -import { Watcher } from "./xds-stream-state/xds-stream-state"; import LoadBalancingConfig = experimental.LoadBalancingConfig; import validateLoadBalancingConfig = experimental.validateLoadBalancingConfig; @@ -37,6 +36,8 @@ import createResolver = experimental.createResolver; import ChannelControlHelper = experimental.ChannelControlHelper; import OutlierDetectionLoadBalancingConfig = experimental.OutlierDetectionLoadBalancingConfig; import subchannelAddressToString = experimental.subchannelAddressToString; +import { serverConfigEqual, validateXdsServerConfig, XdsServerConfig } from "./xds-bootstrap"; +import { EndpointResourceType } from "./xds-resource-type/endpoint-resource-type"; const TRACER_NAME = 'xds_cluster_resolver'; @@ -46,7 +47,7 @@ function trace(text: string): void { export interface DiscoveryMechanism { cluster: string; - lrs_load_reporting_server_name?: string; + lrs_load_reporting_server?: XdsServerConfig; max_concurrent_requests?: number; type: 'EDS' | 'LOGICAL_DNS'; eds_service_name?: string; @@ -61,9 +62,6 @@ function validateDiscoveryMechanism(obj: any): DiscoveryMechanism { if (!('type' in obj && (obj.type === 'EDS' || obj.type === 'LOGICAL_DNS'))) { throw new Error('discovery_mechanisms entry must have a field "type" with the value "EDS" or "LOGICAL_DNS"'); } - if ('lrs_load_reporting_server_name' in obj && typeof obj.lrs_load_reporting_server_name !== 'string') { - throw new Error('discovery_mechanisms entry lrs_load_reporting_server_name field must be a string if provided'); - } if ('max_concurrent_requests' in obj && typeof obj.max_concurrent_requests !== "number") { throw new Error('discovery_mechanisms entry max_concurrent_requests field must be a number if provided'); } @@ -78,7 +76,7 @@ function validateDiscoveryMechanism(obj: any): DiscoveryMechanism { if (!(outlierDetectionConfig instanceof OutlierDetectionLoadBalancingConfig)) { throw new Error('eds config outlier_detection must be a valid outlier detection config if provided'); } - return {...obj, outlier_detection: outlierDetectionConfig}; + return {...obj, lrs_load_reporting_server: validateXdsServerConfig(obj.lrs_load_reporting_server), outlier_detection: outlierDetectionConfig}; } return obj; } @@ -313,8 +311,8 @@ export class XdsClusterResolver implements LoadBalancer { const childTargets = new Map(); for (const localityObj of priorityEntry.localities) { let childPolicy: LoadBalancingConfig[]; - if (entry.discoveryMechanism.lrs_load_reporting_server_name !== undefined) { - childPolicy = [new LrsLoadBalancingConfig(entry.discoveryMechanism.cluster, entry.discoveryMechanism.eds_service_name ?? '', entry.discoveryMechanism.lrs_load_reporting_server_name!, localityObj.locality, endpointPickingPolicy)]; + if (entry.discoveryMechanism.lrs_load_reporting_server !== undefined) { + childPolicy = [new LrsLoadBalancingConfig(entry.discoveryMechanism.cluster, entry.discoveryMechanism.eds_service_name ?? '', entry.discoveryMechanism.lrs_load_reporting_server, localityObj.locality, endpointPickingPolicy)]; } else { childPolicy = endpointPickingPolicy; } @@ -334,7 +332,7 @@ export class XdsClusterResolver implements LoadBalancer { newLocalityPriorities.set(localityToName(localityObj.locality), priority); } const weightedTargetConfig = new WeightedTargetLoadBalancingConfig(childTargets); - const xdsClusterImplConfig = new XdsClusterImplLoadBalancingConfig(entry.discoveryMechanism.cluster, priorityEntry.dropCategories, [weightedTargetConfig], entry.discoveryMechanism.eds_service_name, entry.discoveryMechanism.lrs_load_reporting_server_name, entry.discoveryMechanism.max_concurrent_requests); + const xdsClusterImplConfig = new XdsClusterImplLoadBalancingConfig(entry.discoveryMechanism.cluster, priorityEntry.dropCategories, [weightedTargetConfig], entry.discoveryMechanism.eds_service_name, entry.discoveryMechanism.lrs_load_reporting_server, entry.discoveryMechanism.max_concurrent_requests); let outlierDetectionConfig: OutlierDetectionLoadBalancingConfig | undefined; if (EXPERIMENTAL_OUTLIER_DETECTION) { outlierDetectionConfig = entry.discoveryMechanism.outlier_detection?.copyWithChildPolicy([xdsClusterImplConfig]); @@ -379,8 +377,8 @@ export class XdsClusterResolver implements LoadBalancer { }; if (mechanism.type === 'EDS') { const edsServiceName = mechanism.eds_service_name ?? mechanism.cluster; - const watcher: Watcher = { - onValidUpdate: update => { + const watcher: Watcher = new Watcher({ + onResourceChanged: update => { mechanismEntry.latestUpdate = getEdsPriorities(update); this.maybeUpdateChild(); }, @@ -388,15 +386,17 @@ export class XdsClusterResolver implements LoadBalancer { trace('Resource does not exist: ' + edsServiceName); mechanismEntry.latestUpdate = [{localities: [], dropCategories: []}]; }, - onTransientError: error => { + onError: error => { if (!mechanismEntry.latestUpdate) { trace('xDS request failed with error ' + error); mechanismEntry.latestUpdate = [{localities: [], dropCategories: []}]; } } - }; + }); mechanismEntry.watcher = watcher; - this.xdsClient?.addEndpointWatcher(edsServiceName, watcher); + if (this.xdsClient) { + EndpointResourceType.startWatch(this.xdsClient, edsServiceName, watcher); + } } else { const resolver = createResolver({scheme: 'dns', path: mechanism.dns_hostname!}, { onSuccessfulResolution: addressList => { @@ -436,7 +436,9 @@ export class XdsClusterResolver implements LoadBalancer { for (const mechanismEntry of this.discoveryMechanismList) { if (mechanismEntry.watcher) { const edsServiceName = mechanismEntry.discoveryMechanism.eds_service_name ?? mechanismEntry.discoveryMechanism.cluster; - this.xdsClient?.removeEndpointWatcher(edsServiceName, mechanismEntry.watcher); + if (this.xdsClient) { + EndpointResourceType.cancelWatch(this.xdsClient, edsServiceName, mechanismEntry.watcher); + } } mechanismEntry.resolver?.destroy(); } @@ -448,6 +450,14 @@ export class XdsClusterResolver implements LoadBalancer { } } +function maybeServerConfigEqual(config1: XdsServerConfig | undefined, config2: XdsServerConfig | undefined) { + if (config1 !== undefined && config2 !== undefined) { + return serverConfigEqual(config1, config2); + } else { + return config1 === config2; + } +} + export class XdsClusterResolverChildPolicyHandler extends ChildLoadBalancerHandler { protected configUpdateRequiresNewPolicyInstance(oldConfig: LoadBalancingConfig, newConfig: LoadBalancingConfig): boolean { if (!(oldConfig instanceof XdsClusterResolverLoadBalancingConfig && newConfig instanceof XdsClusterResolverLoadBalancingConfig)) { @@ -463,7 +473,7 @@ export class XdsClusterResolverChildPolicyHandler extends ChildLoadBalancerHandl oldDiscoveryMechanism.cluster !== newDiscoveryMechanism.cluster || oldDiscoveryMechanism.eds_service_name !== newDiscoveryMechanism.eds_service_name || oldDiscoveryMechanism.dns_hostname !== newDiscoveryMechanism.dns_hostname || - oldDiscoveryMechanism.lrs_load_reporting_server_name !== newDiscoveryMechanism.lrs_load_reporting_server_name) { + !maybeServerConfigEqual(oldDiscoveryMechanism.lrs_load_reporting_server, newDiscoveryMechanism.lrs_load_reporting_server)) { return true; } } @@ -473,4 +483,4 @@ export class XdsClusterResolverChildPolicyHandler extends ChildLoadBalancerHandl export function setup() { registerLoadBalancerType(TYPE_NAME, XdsClusterResolver, XdsClusterResolverLoadBalancingConfig); -} \ No newline at end of file +} diff --git a/packages/grpc-js-xds/src/resolver-xds.ts b/packages/grpc-js-xds/src/resolver-xds.ts index 6c2ba4325..a934e7285 100644 --- a/packages/grpc-js-xds/src/resolver-xds.ts +++ b/packages/grpc-js-xds/src/resolver-xds.ts @@ -18,7 +18,7 @@ import * as protoLoader from '@grpc/proto-loader'; import { RE2 } from 're2-wasm'; -import { getSingletonXdsClient, XdsClient } from './xds-client'; +import { getSingletonXdsClient, Watcher, XdsClient } from './xds-client'; import { StatusObject, status, logVerbosity, Metadata, experimental, ChannelOptions } from '@grpc/grpc-js'; import Resolver = experimental.Resolver; import GrpcUri = experimental.GrpcUri; @@ -27,7 +27,6 @@ import uriToString = experimental.uriToString; import ServiceConfig = experimental.ServiceConfig; import registerResolver = experimental.registerResolver; import { Listener__Output } from './generated/envoy/config/listener/v3/Listener'; -import { Watcher } from './xds-stream-state/xds-stream-state'; import { RouteConfiguration__Output } from './generated/envoy/config/route/v3/RouteConfiguration'; import { HttpConnectionManager__Output } from './generated/envoy/extensions/filters/network/http_connection_manager/v3/HttpConnectionManager'; import { CdsLoadBalancingConfig } from './load-balancer-cds'; @@ -44,11 +43,13 @@ import { decodeSingleResource, HTTP_CONNECTION_MANGER_TYPE_URL } from './resourc import Duration = experimental.Duration; import { Duration__Output } from './generated/google/protobuf/Duration'; import { createHttpFilter, HttpFilterConfig, parseOverrideFilterConfig, parseTopLevelFilterConfig } from './http-filter'; -import { EXPERIMENTAL_FAULT_INJECTION, EXPERIMENTAL_RETRY } from './environment'; +import { EXPERIMENTAL_FAULT_INJECTION, EXPERIMENTAL_FEDERATION, EXPERIMENTAL_RETRY } from './environment'; import Filter = experimental.Filter; import FilterFactory = experimental.FilterFactory; import RetryPolicy = experimental.RetryPolicy; -import { validateBootstrapConfig } from './xds-bootstrap'; +import { BootstrapInfo, loadBootstrapInfo, validateBootstrapConfig } from './xds-bootstrap'; +import { ListenerResourceType } from './xds-resource-type/listener-resource-type'; +import { RouteConfigurationResourceType } from './xds-resource-type/route-config-resource-type'; const TRACER_NAME = 'xds_resolver'; @@ -231,6 +232,35 @@ function getDefaultRetryMaxInterval(baseInterval: string): string { return `${Number.parseFloat(baseInterval.substring(0, baseInterval.length - 1)) * 10}s`; } +/** + * Encode a text string as a valid path of a URI, as specified in RFC-3986 section 3.3 + * @param uriPath A value representing an unencoded URI path + * @returns + */ +function encodeURIPath(uriPath: string): string { + return uriPath.replace(/[^A-Za-z0-9._~!$&^()*+,;=/-]/g, substring => encodeURIComponent(substring)); +} + +function formatTemplateString(templateString: string, value: string): string { + if (templateString.startsWith('xdstp:')) { + return templateString.replace(/%s/g, encodeURIPath(value)); + } else { + return templateString.replace(/%s/g, value); + } +} + +export function getListenerResourceName(bootstrapConfig: BootstrapInfo, target: GrpcUri): string { + if (target.authority && target.authority !== '') { + if (target.authority in bootstrapConfig.authorities) { + return formatTemplateString(bootstrapConfig.authorities[target.authority].clientListenerResourceNameTemplate, target.path); + } else { + throw new Error(`Authority ${target.authority} not found in bootstrap file`); + } + } else { + return formatTemplateString(bootstrapConfig.clientDefaultListenerResourceNameTemplate, target.path); + } +} + const BOOTSTRAP_CONFIG_KEY = 'grpc.TEST_ONLY_DO_NOT_USE_IN_PROD.xds_bootstrap_config'; const RETRY_CODES: {[key: string]: status} = { @@ -247,6 +277,7 @@ class XdsResolver implements Resolver { private ldsWatcher: Watcher; private rdsWatcher: Watcher private isLdsWatcherActive = false; + private listenerResourceName: string | null = null; /** * The latest route config name from an LDS response. The RDS watcher is * actively watching that name if and only if this is not null. @@ -261,6 +292,8 @@ class XdsResolver implements Resolver { private ldsHttpFilterConfigs: {name: string, config: HttpFilterConfig}[] = []; + private bootstrapInfo: BootstrapInfo | null = null; + private xdsClient: XdsClient; constructor( @@ -270,13 +303,13 @@ class XdsResolver implements Resolver { ) { if (channelOptions[BOOTSTRAP_CONFIG_KEY]) { const parsedConfig = JSON.parse(channelOptions[BOOTSTRAP_CONFIG_KEY]); - const validatedConfig = validateBootstrapConfig(parsedConfig); - this.xdsClient = new XdsClient(validatedConfig); + this.bootstrapInfo = validateBootstrapConfig(parsedConfig); + this.xdsClient = new XdsClient(this.bootstrapInfo); } else { this.xdsClient = getSingletonXdsClient(); } - this.ldsWatcher = { - onValidUpdate: (update: Listener__Output) => { + this.ldsWatcher = new Watcher({ + onResourceChanged: (update: Listener__Output) => { const httpConnectionManager = decodeSingleResource(HTTP_CONNECTION_MANGER_TYPE_URL, update.api_listener!.api_listener!.value); const defaultTimeout = httpConnectionManager.common_http_protocol_options?.idle_timeout; if (defaultTimeout === null || defaultTimeout === undefined) { @@ -299,16 +332,16 @@ class XdsResolver implements Resolver { const routeConfigName = httpConnectionManager.rds!.route_config_name; if (this.latestRouteConfigName !== routeConfigName) { if (this.latestRouteConfigName !== null) { - this.xdsClient.removeRouteWatcher(this.latestRouteConfigName, this.rdsWatcher); + RouteConfigurationResourceType.cancelWatch(this.xdsClient, this.latestRouteConfigName, this.rdsWatcher); } - this.xdsClient.addRouteWatcher(httpConnectionManager.rds!.route_config_name, this.rdsWatcher); + RouteConfigurationResourceType.startWatch(this.xdsClient, routeConfigName, this.rdsWatcher); this.latestRouteConfigName = routeConfigName; } break; } case 'route_config': if (this.latestRouteConfigName) { - this.xdsClient.removeRouteWatcher(this.latestRouteConfigName, this.rdsWatcher); + RouteConfigurationResourceType.cancelWatch(this.xdsClient, this.latestRouteConfigName, this.rdsWatcher); } this.handleRouteConfig(httpConnectionManager.route_config!); break; @@ -316,7 +349,7 @@ class XdsResolver implements Resolver { // This is prevented by the validation rules } }, - onTransientError: (error: StatusObject) => { + onError: (error: StatusObject) => { /* A transient error only needs to bubble up as a failure if we have * not already provided a ServiceConfig for the upper layer to use */ if (!this.hasReportedSuccess) { @@ -328,12 +361,12 @@ class XdsResolver implements Resolver { trace('Resolution error for target ' + uriToString(this.target) + ': LDS resource does not exist'); this.reportResolutionError(`Listener ${this.target} does not exist`); } - }; - this.rdsWatcher = { - onValidUpdate: (update: RouteConfiguration__Output) => { + }); + this.rdsWatcher = new Watcher({ + onResourceChanged: (update: RouteConfiguration__Output) => { this.handleRouteConfig(update); }, - onTransientError: (error: StatusObject) => { + onError: (error: StatusObject) => { /* A transient error only needs to bubble up as a failure if we have * not already provided a ServiceConfig for the upper layer to use */ if (!this.hasReportedSuccess) { @@ -345,7 +378,7 @@ class XdsResolver implements Resolver { trace('Resolution error for target ' + uriToString(this.target) + ' and route config ' + this.latestRouteConfigName + ': RDS resource does not exist'); this.reportResolutionError(`Route config ${this.latestRouteConfigName} does not exist`); } - } + }); } private refCluster(clusterName: string) { @@ -591,19 +624,49 @@ class XdsResolver implements Resolver { }); } - updateResolution(): void { - // Wait until updateResolution is called once to start the xDS requests + private startResolution(): void { if (!this.isLdsWatcherActive) { trace('Starting resolution for target ' + uriToString(this.target)); - this.xdsClient.addListenerWatcher(this.target.path, this.ldsWatcher); - this.isLdsWatcherActive = true; + try { + this.listenerResourceName = getListenerResourceName(this.bootstrapInfo!, this.target); + trace('Resolving target ' + uriToString(this.target) + ' with Listener resource name ' + this.listenerResourceName); + ListenerResourceType.startWatch(this.xdsClient, this.listenerResourceName, this.ldsWatcher); + this.isLdsWatcherActive = true; + + } catch (e) { + this.reportResolutionError(e.message); + } + } + } + + updateResolution(): void { + if (EXPERIMENTAL_FEDERATION) { + if (this.bootstrapInfo) { + this.startResolution(); + } else { + try { + this.bootstrapInfo = loadBootstrapInfo(); + } catch (e) { + this.reportResolutionError(e.message); + } + this.startResolution(); + } + } else { + if (!this.isLdsWatcherActive) { + trace('Starting resolution for target ' + uriToString(this.target)); + ListenerResourceType.startWatch(this.xdsClient, this.target.path, this.ldsWatcher); + this.listenerResourceName = this.target.path; + this.isLdsWatcherActive = true; + } } } destroy() { - this.xdsClient.removeListenerWatcher(this.target.path, this.ldsWatcher); + if (this.listenerResourceName) { + ListenerResourceType.cancelWatch(this.xdsClient, this.listenerResourceName, this.ldsWatcher); + } if (this.latestRouteConfigName) { - this.xdsClient.removeRouteWatcher(this.latestRouteConfigName, this.rdsWatcher); + RouteConfigurationResourceType.cancelWatch(this.xdsClient, this.latestRouteConfigName, this.rdsWatcher); } } diff --git a/packages/grpc-js-xds/src/resources.ts b/packages/grpc-js-xds/src/resources.ts index e4d464b6d..4542c5fd6 100644 --- a/packages/grpc-js-xds/src/resources.ts +++ b/packages/grpc-js-xds/src/resources.ts @@ -15,6 +15,10 @@ * */ +import { URI } from 'vscode-uri'; +/* Since we are using an internal function from @grpc/proto-loader, we also + * need the top-level import to perform some setup operations. */ +import '@grpc/proto-loader'; // This is a non-public, unstable API, but it's very convenient import { loadProtosWithOptionsSync } from '@grpc/proto-loader/build/src/util'; import { Cluster__Output } from './generated/envoy/config/cluster/v3/Cluster'; @@ -23,6 +27,7 @@ import { Listener__Output } from './generated/envoy/config/listener/v3/Listener' import { RouteConfiguration__Output } from './generated/envoy/config/route/v3/RouteConfiguration'; import { ClusterConfig__Output } from './generated/envoy/extensions/clusters/aggregate/v3/ClusterConfig'; import { HttpConnectionManager__Output } from './generated/envoy/extensions/filters/network/http_connection_manager/v3/HttpConnectionManager'; +import { EXPERIMENTAL_FEDERATION } from './environment'; export const EDS_TYPE_URL = 'type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment'; export const CDS_TYPE_URL = 'type.googleapis.com/envoy.config.cluster.v3.Cluster'; @@ -60,6 +65,8 @@ export type AdsOutputType 0) { + const queryParams = uri.query.split('&'); + queryParams.sort(); + queryString = '?' + queryParams.join('&'); + } else { + queryString = ''; + } + return { + authority: uri.authority, + key: `${pathComponents.slice(1).join('/')}${queryString}` + }; +} + +export function xdsResourceNameToString(name: XdsResourceName, typeUrl: string): string { + if (name.authority === 'old:') { + return name.key; + } + return `xdstp://${name.authority}/${typeUrl}/${name.key}`; +} diff --git a/packages/grpc-js-xds/src/xds-bootstrap.ts b/packages/grpc-js-xds/src/xds-bootstrap.ts index 72a0ca375..327ee1b06 100644 --- a/packages/grpc-js-xds/src/xds-bootstrap.ts +++ b/packages/grpc-js-xds/src/xds-bootstrap.ts @@ -16,6 +16,7 @@ */ import * as fs from 'fs'; +import { EXPERIMENTAL_FEDERATION } from './environment'; import { Struct } from './generated/google/protobuf/Struct'; import { Value } from './generated/google/protobuf/Value'; @@ -45,9 +46,42 @@ export interface XdsServerConfig { serverFeatures: string[]; } +export interface Authority { + clientListenerResourceNameTemplate: string; + xdsServers?: XdsServerConfig[]; +} + export interface BootstrapInfo { xdsServers: XdsServerConfig[]; node: Node; + authorities: {[authorityName: string]: Authority}; + clientDefaultListenerResourceNameTemplate: string; +} + +const KNOWN_SERVER_FEATURES = ['ignore_resource_deletion']; + +export function serverConfigEqual(config1: XdsServerConfig, config2: XdsServerConfig): boolean { + if (config1.serverUri !== config2.serverUri) { + return false; + } + for (const feature of KNOWN_SERVER_FEATURES) { + if ((feature in config1.serverFeatures) !== (feature in config2.serverFeatures)) { + return false; + } + } + if (config1.channelCreds.length !== config2.channelCreds.length) { + return false; + } + for (const [index, creds1] of config1.channelCreds.entries()) { + const creds2 = config2.channelCreds[index]; + if (creds1.type !== creds2.type) { + return false; + } + if (JSON.stringify(creds1) !== JSON.stringify(creds2)) { + return false; + } + } + return true; } function validateChannelCredsConfig(obj: any): ChannelCredsConfig { @@ -72,7 +106,12 @@ function validateChannelCredsConfig(obj: any): ChannelCredsConfig { }; } -function validateXdsServerConfig(obj: any): XdsServerConfig { +const SUPPORTED_CHANNEL_CREDS_TYPES = [ + 'google_default', + 'insecure' +]; + +export function validateXdsServerConfig(obj: any): XdsServerConfig { if (!('server_uri' in obj)) { throw new Error('server_uri field missing in xds_servers element'); } @@ -89,9 +128,15 @@ function validateXdsServerConfig(obj: any): XdsServerConfig { `xds_servers.channel_creds field: expected array, got ${typeof obj.channel_creds}` ); } - if (obj.channel_creds.length === 0) { + let foundSupported = false; + for (const cred of obj.channel_creds) { + if (SUPPORTED_CHANNEL_CREDS_TYPES.includes(cred.type)) { + foundSupported = true; + } + } + if (!foundSupported) { throw new Error( - 'xds_servers.channel_creds field: at least one entry is required' + `xds_servers.channel_creds field: must contain at least one entry with a type in [${SUPPORTED_CHANNEL_CREDS_TYPES}]` ); } if ('server_features' in obj) { @@ -231,16 +276,67 @@ function validateNode(obj: any): Node { return result; } -export function validateBootstrapConfig(obj: any): BootstrapInfo { +function validateAuthority(obj: any, authorityName: string): Authority { + if ('client_listener_resource_name_template' in obj) { + if (typeof obj.client_listener_resource_name_template !== 'string') { + throw new Error(`authorities[${authorityName}].client_listener_resource_name_template: expected string, got ${typeof obj.client_listener_resource_name_template}`); + } + if (!obj.client_listener_resource_name_template.startsWith(`xdstp://${authorityName}/`)) { + throw new Error(`authorities[${authorityName}].client_listener_resource_name_template must start with "xdstp://${authorityName}/"`); + } + } return { - xdsServers: obj.xds_servers.map(validateXdsServerConfig), - node: validateNode(obj.node), + clientListenerResourceNameTemplate: obj.client_listener_resource_name_template ?? `xdstp://${authorityName}/envoy.config.listener.v3.Listener/%s`, + xdsServers: obj.xds_servers?.map(validateXdsServerConfig) }; } -let loadedBootstrapInfo: Promise | null = null; +function validateAuthoritiesMap(obj: any): {[authorityName: string]: Authority} { + if (!obj) { + return {}; + } + const result: {[authorityName: string]: Authority} = {}; + for (const [name, authority] of Object.entries(obj)) { + result[name] = validateAuthority(authority, name); + } + return result; +} + +export function validateBootstrapConfig(obj: any): BootstrapInfo { + const xdsServers = obj.xds_servers.map(validateXdsServerConfig); + const node = validateNode(obj.node); + if (EXPERIMENTAL_FEDERATION) { + if ('client_default_listener_resource_name_template' in obj) { + if (typeof obj.client_default_listener_resource_name_template !== 'string') { + throw new Error(`client_default_listener_resource_name_template: expected string, got ${typeof obj.client_default_listener_resource_name_template}`); + } + } + return { + xdsServers: xdsServers, + node: node, + authorities: validateAuthoritiesMap(obj.authorities), + clientDefaultListenerResourceNameTemplate: obj.client_default_listener_resource_name_template ?? '%s' + }; + } else { + return { + xdsServers: xdsServers, + node: node, + authorities: {}, + clientDefaultListenerResourceNameTemplate: '%s' + }; + } +} + +let loadedBootstrapInfo: BootstrapInfo | null = null; -export async function loadBootstrapInfo(): Promise { +/** + * Load the bootstrap information from the location determined by the + * GRPC_XDS_BOOTSTRAP environment variable, or if that is unset, from the + * GRPC_XDS_BOOTSTRAP_CONFIG environment variable. The value is cached, so any + * calls after the first will just return the cached value. + * @returns + */ +export function loadBootstrapInfo(): BootstrapInfo { if (loadedBootstrapInfo !== null) { return loadedBootstrapInfo; } @@ -254,28 +350,19 @@ export async function loadBootstrapInfo(): Promise { */ const bootstrapPath = process.env.GRPC_XDS_BOOTSTRAP; if (bootstrapPath) { - loadedBootstrapInfo = new Promise((resolve, reject) => { - fs.readFile(bootstrapPath, { encoding: 'utf8' }, (err, data) => { - if (err) { - reject( - new Error( - `Failed to read xDS bootstrap file from path ${bootstrapPath} with error ${err.message}` - ) - ); - } - try { - const parsedFile = JSON.parse(data); - resolve(validateBootstrapConfig(parsedFile)); - } catch (e) { - reject( - new Error( - `Failed to parse xDS bootstrap file at path ${bootstrapPath} with error ${e.message}` - ) - ); - } - }); - }); - return loadedBootstrapInfo; + let rawBootstrap: string; + try { + rawBootstrap = fs.readFileSync(bootstrapPath, { encoding: 'utf8'}); + } catch (e) { + throw new Error(`Failed to read xDS bootstrap file from path ${bootstrapPath} with error ${e.message}`); + } + try { + const parsedFile = JSON.parse(rawBootstrap); + loadedBootstrapInfo = validateBootstrapConfig(parsedFile); + return loadedBootstrapInfo; + } catch (e) { + throw new Error(`Failed to parse xDS bootstrap file at path ${bootstrapPath} with error ${e.message}`) + } } /** @@ -290,8 +377,7 @@ export async function loadBootstrapInfo(): Promise { if (bootstrapConfig) { try { const parsedConfig = JSON.parse(bootstrapConfig); - const loadedBootstrapInfoValue = validateBootstrapConfig(parsedConfig); - loadedBootstrapInfo = Promise.resolve(loadedBootstrapInfoValue); + loadedBootstrapInfo = validateBootstrapConfig(parsedConfig); } catch (e) { throw new Error( `Failed to parse xDS bootstrap config from environment variable GRPC_XDS_BOOTSTRAP_CONFIG with error ${e.message}` @@ -301,9 +387,8 @@ export async function loadBootstrapInfo(): Promise { return loadedBootstrapInfo; } - return Promise.reject( - new Error( - 'The GRPC_XDS_BOOTSTRAP or GRPC_XDS_BOOTSTRAP_CONFIG environment variables need to be set to the path to the bootstrap file to use xDS' - ) + + throw new Error( + 'The GRPC_XDS_BOOTSTRAP or GRPC_XDS_BOOTSTRAP_CONFIG environment variables need to be set to the path to the bootstrap file to use xDS' ); } diff --git a/packages/grpc-js-xds/src/xds-client.ts b/packages/grpc-js-xds/src/xds-client.ts index 20c914a1c..2ae9618b3 100644 --- a/packages/grpc-js-xds/src/xds-client.ts +++ b/packages/grpc-js-xds/src/xds-client.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 gRPC authors. + * Copyright 2023 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,38 +15,26 @@ * */ -import * as protoLoader from '@grpc/proto-loader'; -// This is a non-public, unstable API, but it's very convenient -import { loadProtosWithOptionsSync } from '@grpc/proto-loader/build/src/util'; -import { loadPackageDefinition, StatusObject, status, logVerbosity, Metadata, experimental, ChannelOptions, ClientDuplexStream, ServiceError, ChannelCredentials, Channel, connectivityState } from '@grpc/grpc-js'; +import { Channel, ChannelCredentials, ClientDuplexStream, Metadata, StatusObject, connectivityState, experimental, loadPackageDefinition, logVerbosity, status } from "@grpc/grpc-js"; +import { XdsDecodeContext, XdsDecodeResult, XdsResourceType } from "./xds-resource-type/xds-resource-type"; +import { XdsResourceName, parseXdsResourceName, xdsResourceNameToString } from "./resources"; +import { Node } from "./generated/envoy/config/core/v3/Node"; +import { BootstrapInfo, XdsServerConfig, loadBootstrapInfo, serverConfigEqual } from "./xds-bootstrap"; +import BackoffTimeout = experimental.BackoffTimeout; +import { DiscoveryRequest } from "./generated/envoy/service/discovery/v3/DiscoveryRequest"; +import { DiscoveryResponse__Output } from "./generated/envoy/service/discovery/v3/DiscoveryResponse"; import * as adsTypes from './generated/ads'; import * as lrsTypes from './generated/lrs'; -import { BootstrapInfo, loadBootstrapInfo } from './xds-bootstrap'; -import { Node } from './generated/envoy/config/core/v3/Node'; -import { AggregatedDiscoveryServiceClient } from './generated/envoy/service/discovery/v3/AggregatedDiscoveryService'; -import { DiscoveryRequest } from './generated/envoy/service/discovery/v3/DiscoveryRequest'; -import { DiscoveryResponse__Output } from './generated/envoy/service/discovery/v3/DiscoveryResponse'; -import { LoadReportingServiceClient } from './generated/envoy/service/load_stats/v3/LoadReportingService'; -import { LoadStatsRequest } from './generated/envoy/service/load_stats/v3/LoadStatsRequest'; -import { LoadStatsResponse__Output } from './generated/envoy/service/load_stats/v3/LoadStatsResponse'; -import { Locality, Locality__Output } from './generated/envoy/config/core/v3/Locality'; -import { Listener__Output } from './generated/envoy/config/listener/v3/Listener'; -import { Any__Output } from './generated/google/protobuf/Any'; -import BackoffTimeout = experimental.BackoffTimeout; -import ServiceConfig = experimental.ServiceConfig; -import { createGoogleDefaultCredentials } from './google-default-credentials'; -import { CdsLoadBalancingConfig } from './load-balancer-cds'; -import { EdsState } from './xds-stream-state/eds-state'; -import { CdsState } from './xds-stream-state/cds-state'; -import { RdsState } from './xds-stream-state/rds-state'; -import { LdsState } from './xds-stream-state/lds-state'; -import { HandleResponseResult, ResourcePair, Watcher } from './xds-stream-state/xds-stream-state'; -import { ClusterLoadAssignment__Output } from './generated/envoy/config/endpoint/v3/ClusterLoadAssignment'; -import { Cluster__Output } from './generated/envoy/config/cluster/v3/Cluster'; -import { RouteConfiguration__Output } from './generated/envoy/config/route/v3/RouteConfiguration'; -import { Duration } from './generated/google/protobuf/Duration'; -import { AdsOutputType, AdsTypeUrl, CDS_TYPE_URL, decodeSingleResource, EDS_TYPE_URL, LDS_TYPE_URL, RDS_TYPE_URL } from './resources'; -import { setCsdsClientNode, updateCsdsRequestedNameList, updateCsdsResourceResponse } from './csds'; +import * as protoLoader from '@grpc/proto-loader'; +import { AggregatedDiscoveryServiceClient } from "./generated/envoy/service/discovery/v3/AggregatedDiscoveryService"; +import { LoadReportingServiceClient } from "./generated/envoy/service/load_stats/v3/LoadReportingService"; +import { createGoogleDefaultCredentials } from "./google-default-credentials"; +import { Any__Output } from "./generated/google/protobuf/Any"; +import { LoadStatsRequest } from "./generated/envoy/service/load_stats/v3/LoadStatsRequest"; +import { LoadStatsResponse__Output } from "./generated/envoy/service/load_stats/v3/LoadStatsResponse"; +import { Locality, Locality__Output } from "./generated/envoy/config/core/v3/Locality"; +import { Duration } from "./generated/google/protobuf/Duration"; +import { registerXdsClientWithCsds } from "./csds"; const TRACER_NAME = 'xds_client'; @@ -54,20 +42,14 @@ function trace(text: string): void { experimental.trace(logVerbosity.DEBUG, TRACER_NAME, text); } -const clientVersion = require('../../package.json').version; - -let loadedProtos: Promise< - adsTypes.ProtoGrpcType & lrsTypes.ProtoGrpcType -> | null = null; +let loadedProtos: adsTypes.ProtoGrpcType & lrsTypes.ProtoGrpcType | null = null; -function loadAdsProtos(): Promise< - adsTypes.ProtoGrpcType & lrsTypes.ProtoGrpcType -> { +function loadAdsProtos(): adsTypes.ProtoGrpcType & lrsTypes.ProtoGrpcType { if (loadedProtos !== null) { return loadedProtos; } - loadedProtos = protoLoader - .load( + return (loadPackageDefinition(protoLoader + .loadSync( [ 'envoy/service/discovery/v3/ads.proto', 'envoy/service/load_stats/v3/lrs.proto', @@ -87,16 +69,447 @@ function loadAdsProtos(): Promise< __dirname + '/../../deps/protoc-gen-validate/', ], } - ) - .then( - (packageDefinition) => - (loadPackageDefinition( - packageDefinition - ) as unknown) as adsTypes.ProtoGrpcType & lrsTypes.ProtoGrpcType + )) as unknown) as adsTypes.ProtoGrpcType & lrsTypes.ProtoGrpcType; +} + +const clientVersion = require('../../package.json').version; + +export interface ResourceWatcherInterface { + onGenericResourceChanged(resource: object): void; + onError(status: StatusObject): void; + onResourceDoesNotExist(): void; +} + +export interface BasicWatcher { + onResourceChanged(resource: UpdateType): void; + onError(status: StatusObject): void; + onResourceDoesNotExist(): void; +} + +export class Watcher implements ResourceWatcherInterface { + constructor(private internalWatcher: BasicWatcher) {} + onGenericResourceChanged(resource: object): void { + this.internalWatcher.onResourceChanged(resource as unknown as UpdateType); + } + onError(status: StatusObject) { + this.internalWatcher.onError(status); + } + onResourceDoesNotExist() { + this.internalWatcher.onResourceDoesNotExist(); + } +} + +const RESOURCE_TIMEOUT_MS = 15_000; + +class ResourceTimer { + private timer: NodeJS.Timer | null = null; + private resourceSeen = false; + constructor(private callState: AdsCallState, private type: XdsResourceType, private name: XdsResourceName) {} + + maybeCancelTimer() { + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + } + + markSeen() { + this.resourceSeen = true; + this.maybeCancelTimer(); + } + + markAdsStreamStarted() { + this.maybeStartTimer(); + } + + private maybeStartTimer() { + if (this.resourceSeen) { + return; + } + if (this.timer) { + return; + } + const authorityState = this.callState.client.xdsClient.authorityStateMap.get(this.name.authority); + if (!authorityState) { + return; + } + const resourceState = authorityState.resourceMap.get(this.type)?.get(this.name.key); + if (resourceState?.cachedResource) { + return; + } + this.timer = setTimeout(() => { + this.onTimer(); + }, RESOURCE_TIMEOUT_MS); + } + + private onTimer() { + const authorityState = this.callState.client.xdsClient.authorityStateMap.get(this.name.authority); + const resourceState = authorityState?.resourceMap.get(this.type)?.get(this.name.key); + if (!resourceState) { + return; + } + resourceState.meta.clientStatus = 'DOES_NOT_EXIST'; + for (const watcher of resourceState.watchers) { + watcher.onResourceDoesNotExist(); + } + } +} + +interface AdsParseResult { + type?: XdsResourceType; + typeUrl?: string; + version?: string; + nonce?: string; + errors: string[]; + /** + * authority -> set of keys + */ + resourcesSeen: Map>; + haveValidResources: boolean; +} + +/** + * Responsible for parsing a single ADS response, one resource at a time + */ +class AdsResponseParser { + private result: AdsParseResult = { + errors: [], + resourcesSeen: new Map(), + haveValidResources: false + }; + private updateTime = new Date(); + + constructor(private adsCallState: AdsCallState) {} + + processAdsResponseFields(message: DiscoveryResponse__Output) { + const type = this.adsCallState.client.xdsClient.getResourceType(message.type_url); + if (!type) { + throw new Error(`Unexpected type URL ${message.type_url}`); + } + this.result.type = type; + this.result.typeUrl = message.type_url; + this.result.nonce = message.nonce; + this.result.version = message.version_info; + } + + parseResource(index: number, resource: Any__Output) { + const errorPrefix = `resource index ${index}:`; + if (resource.type_url !== this.result.typeUrl) { + this.result.errors.push(`${errorPrefix} incorrect resource type "${resource.type_url}" (should be "${this.result.typeUrl}")`); + return; + } + if (!this.result.type) { + return; + } + const decodeContext: XdsDecodeContext = { + server: this.adsCallState.client.xdsServerConfig + }; + let decodeResult: XdsDecodeResult; + try { + decodeResult = this.result.type.decode(decodeContext, resource); + } catch (e) { + this.result.errors.push(`${errorPrefix} ${e.message}`); + return; + } + let parsedName: XdsResourceName; + try { + parsedName = parseXdsResourceName(decodeResult.name, this.result.type!.getTypeUrl()); + } catch (e) { + this.result.errors.push(`${errorPrefix} ${e.message}`); + return; + } + this.adsCallState.typeStates.get(this.result.type!)?.subscribedResources.get(parsedName.authority)?.get(parsedName.key)?.markSeen(); + if (this.result.type.allResourcesRequiredInSotW()) { + if (!this.result.resourcesSeen.has(parsedName.authority)) { + this.result.resourcesSeen.set(parsedName.authority, new Set()); + } + this.result.resourcesSeen.get(parsedName.authority)!.add(parsedName.key); + } + const resourceState = this.adsCallState.client.xdsClient.authorityStateMap.get(parsedName.authority)?.resourceMap.get(this.result.type)?.get(parsedName.key); + if (!resourceState) { + // No subscription for this resource + return; + } + if (resourceState.deletionIgnored) { + experimental.log(logVerbosity.INFO, `Received resource with previously ignored deletion: ${decodeResult.name}`); + resourceState.deletionIgnored = false; + } + if (decodeResult.error) { + this.result.errors.push(`${errorPrefix} ${decodeResult.error}`); + process.nextTick(() => { + for (const watcher of resourceState.watchers) { + watcher.onError({code: status.UNAVAILABLE, details: decodeResult.error!, metadata: new Metadata()}); + } + }); + resourceState.meta.clientStatus = 'NACKED'; + resourceState.meta.failedVersion = this.result.version!; + resourceState.meta.failedDetails = decodeResult.error; + resourceState.meta.failedUpdateTime = this.updateTime; + return; + } + if (!decodeResult.value) { + return; + } + this.adsCallState.client.trace('Parsed resource of type ' + this.result.type.getTypeUrl() + ': ' + JSON.stringify(decodeResult.value, undefined, 2)); + this.result.haveValidResources = true; + if (this.result.type.resourcesEqual(resourceState.cachedResource, decodeResult.value)) { + return; + } + resourceState.cachedResource = decodeResult.value; + resourceState.meta = { + clientStatus: 'ACKED', + rawResource: resource, + updateTime: this.updateTime, + version: this.result.version! + }; + process.nextTick(() => { + for (const watcher of resourceState.watchers) { + watcher.onGenericResourceChanged(decodeResult.value!); + } + }); + } + + getResult() { + return this.result; + } +} + +type AdsCall = ClientDuplexStream; + +interface ResourceTypeState { + nonce?: string; + error?: string; + /** + * authority -> key -> timer + */ + subscribedResources: Map>; +} + +class AdsCallState { + public typeStates: Map = new Map(); + private receivedAnyResponse = false; + private sentInitialMessage = false; + constructor(public client: XdsSingleServerClient, private call: AdsCall, private node: Node) { + // Populate subscription map with existing subscriptions + for (const [authority, authorityState] of client.xdsClient.authorityStateMap) { + if (authorityState.client !== client) { + continue; + } + for (const [type, typeMap] of authorityState.resourceMap) { + for (const key of typeMap.keys()) { + this.subscribe(type, {authority, key}, true); + } + } + } + for (const type of this.typeStates.keys()) { + this.updateNames(type); + } + call.on('data', (message: DiscoveryResponse__Output) => { + this.handleResponseMessage(message); + }) + call.on('status', (status: StatusObject) => { + this.handleStreamStatus(status); + }); + call.on('error', () => {}); + } + + private trace(text: string) { + this.client.trace(text); + } + + private handleResponseMessage(message: DiscoveryResponse__Output) { + const parser = new AdsResponseParser(this); + let handledAdsResponseFields: boolean; + try { + parser.processAdsResponseFields(message); + handledAdsResponseFields = true; + } catch (e) { + this.trace('ADS response field parsing failed for type ' + message.type_url); + handledAdsResponseFields = false; + } + if (handledAdsResponseFields) { + for (const [index, resource] of message.resources.entries()) { + parser.parseResource(index, resource); + } + const result = parser.getResult(); + const typeState = this.typeStates.get(result.type!); + if (!typeState) { + this.trace('Type state not found for type ' + result.type!.getTypeUrl()); + return; + } + typeState.nonce = result.nonce; + if (result.errors.length > 0) { + typeState.error = `xDS response validation errors: [${result.errors.join('; ')}]`; + } else { + delete typeState.error; + } + // Delete resources not seen in update if needed + if (result.type!.allResourcesRequiredInSotW()) { + for (const [authority, authorityState] of this.client.xdsClient.authorityStateMap) { + if (authorityState.client !== this.client) { + continue; + } + const typeMap = authorityState.resourceMap.get(result.type!); + if (!typeMap) { + continue; + } + for (const [key, resourceState] of typeMap) { + if (!result.resourcesSeen.get(authority)?.has(key)) { + /* Do nothing for resources that have no cached value. Those are + * handled by the resource timer. */ + if (!resourceState.cachedResource) { + continue; + } + if (this.client.ignoreResourceDeletion) { + experimental.log(logVerbosity.ERROR, 'Ignoring nonexistent resource ' + xdsResourceNameToString({authority, key}, result.type!.getTypeUrl())); + resourceState.deletionIgnored = true; + } else { + resourceState.meta.clientStatus = 'DOES_NOT_EXIST'; + process.nextTick(() => { + for (const watcher of resourceState.watchers) { + watcher.onResourceDoesNotExist(); + } + }); + } + } + } + } + } + if (result.haveValidResources || result.errors.length === 0) { + this.client.resourceTypeVersionMap.set(result.type!, result.version!); + } + this.updateNames(result.type!); + } + } + + private* allWatchers() { + for (const [type, typeState] of this.typeStates) { + for (const [authority, authorityMap] of typeState.subscribedResources) { + for (const key of authorityMap.keys()) { + yield* this.client.xdsClient.authorityStateMap.get(authority)?.resourceMap.get(type)?.get(key)?.watchers ?? []; + } + } + } + } + + private handleStreamStatus(streamStatus: StatusObject) { + this.trace( + 'ADS stream ended. code=' + streamStatus.code + ' details= ' + streamStatus.details ); - return loadedProtos; + if (streamStatus.code !== status.OK && !this.receivedAnyResponse) { + for (const watcher of this.allWatchers()) { + watcher.onError(streamStatus); + } + } + this.client.handleAdsStreamEnd(); + } + + hasSubscribedResources(): boolean { + for (const typeState of this.typeStates.values()) { + for (const authorityMap of typeState.subscribedResources.values()) { + if (authorityMap.size > 0) { + return true; + } + } + } + return false; + } + + subscribe(type: XdsResourceType, name: XdsResourceName, delaySend: boolean = false) { + let typeState = this.typeStates.get(type); + if (!typeState) { + typeState = { + nonce: '', + subscribedResources: new Map() + }; + this.typeStates.set(type, typeState); + } + let authorityMap = typeState.subscribedResources.get(name.authority); + if (!authorityMap) { + authorityMap = new Map(); + typeState.subscribedResources.set(name.authority, authorityMap); + } + if (!authorityMap.has(name.key)) { + const timer = new ResourceTimer(this, type, name); + authorityMap.set(name.key, timer); + if (!delaySend) { + this.updateNames(type); + } + } + } + + unsubscribe(type: XdsResourceType, name: XdsResourceName) { + const typeState = this.typeStates.get(type); + if (!typeState) { + return; + } + const authorityMap = typeState.subscribedResources.get(name.authority); + if (!authorityMap) { + return; + } + authorityMap.delete(name.key); + if (authorityMap.size === 0) { + typeState.subscribedResources.delete(name.authority); + } + if (typeState.subscribedResources.size === 0) { + this.typeStates.delete(type); + } + this.updateNames(type); + } + + resourceNamesForRequest(type: XdsResourceType): string[] { + const typeState = this.typeStates.get(type); + if (!typeState) { + return []; + } + const result: string[] = []; + for (const [authority, authorityMap] of typeState.subscribedResources) { + for (const [key, timer] of authorityMap) { + result.push(xdsResourceNameToString({authority, key}, type.getTypeUrl())); + } + } + return result; + } + + updateNames(type: XdsResourceType) { + const typeState = this.typeStates.get(type); + if (!typeState) { + return; + } + const request: DiscoveryRequest = { + node: this.sentInitialMessage ? null : this.node, + type_url: type.getFullTypeUrl(), + response_nonce: typeState.nonce, + resource_names: this.resourceNamesForRequest(type), + version_info: this.client.resourceTypeVersionMap.get(type), + error_detail: typeState.error ? { code: status.UNAVAILABLE, message: typeState.error} : null + }; + this.trace('Sending discovery request: ' + JSON.stringify(request, undefined, 2)); + this.call.write(request); + this.sentInitialMessage = true; + } + + end() { + this.call.end(); + } + + /** + * Should be called when the channel state is READY after starting the + * stream. + */ + markStreamStarted() { + for (const [type, typeState] of this.typeStates) { + for (const [authority, authorityMap] of typeState.subscribedResources) { + for (const resourceTimer of authorityMap.values()) { + resourceTimer.markAdsStreamStarted(); + } + } + } + } } +type LrsCall = ClientDuplexStream; + function localityEqual( loc1: Locality__Output, loc2: Locality__Output @@ -150,21 +563,25 @@ interface ClusterLocalityStats { callsSucceeded: number; callsFailed: number; callsInProgress: number; + refcount: number; } interface ClusterLoadReport { callsDropped: Map; uncategorizedCallsDropped: number; - localityStats: ClusterLocalityStats[]; + localityStats: Set; intervalStart: [number, number]; } +interface StatsMapEntry { + clusterName: string; + edsServiceName: string; + refCount: number; + stats: ClusterLoadReport; +} + class ClusterLoadReportMap { - private statsMap: { - clusterName: string; - edsServiceName: string; - stats: ClusterLoadReport; - }[] = []; + private statsMap: Set = new Set(); get( clusterName: string, @@ -181,24 +598,34 @@ class ClusterLoadReportMap { return undefined; } + /** + * Get the indicated map entry if it exists, or create a new one if it does + * not. Increments the refcount of that entry, so a call to this method + * should correspond to a later call to unref + * @param clusterName + * @param edsServiceName + * @returns + */ getOrCreate(clusterName: string, edsServiceName: string): ClusterLoadReport { for (const statsObj of this.statsMap) { if ( statsObj.clusterName === clusterName && statsObj.edsServiceName === edsServiceName ) { + statsObj.refCount += 1; return statsObj.stats; } } const newStats: ClusterLoadReport = { callsDropped: new Map(), uncategorizedCallsDropped: 0, - localityStats: [], + localityStats: new Set(), intervalStart: process.hrtime(), }; - this.statsMap.push({ + this.statsMap.add({ clusterName, edsServiceName, + refCount: 1, stats: newStats, }); return newStats; @@ -217,576 +644,100 @@ class ClusterLoadReportMap { ]; } } -} - -type AdsServiceKind = 'eds' | 'cds' | 'rds' | 'lds'; - -interface AdsState { - eds: EdsState; - cds: CdsState; - rds: RdsState; - lds: LdsState; -} - -function getResponseMessages( - targetTypeUrl: T, - resources: Any__Output[] -): ResourcePair>[] { - const result: ResourcePair>[] = []; - for (const resource of resources) { - if (resource.type_url !== targetTypeUrl) { - throw new Error( - `ADS Error: Invalid resource type ${resource.type_url}, expected ${targetTypeUrl}` - ); - } - result.push({ - resource: decodeSingleResource(targetTypeUrl, resource.value), - raw: resource - }); - } - return result; -} - -export class XdsClient { - - private adsNode: Node | null = null; - private adsClient: AggregatedDiscoveryServiceClient | null = null; - private adsCall: ClientDuplexStream< - DiscoveryRequest, - DiscoveryResponse__Output - > | null = null; - private receivedAdsResponseOnCurrentStream = false; - - private lrsNode: Node | null = null; - private lrsClient: LoadReportingServiceClient | null = null; - private lrsCall: ClientDuplexStream< - LoadStatsRequest, - LoadStatsResponse__Output - > | null = null; - private latestLrsSettings: LoadStatsResponse__Output | null = null; - private receivedLrsSettingsForCurrentStream = false; - - private clusterStatsMap: ClusterLoadReportMap = new ClusterLoadReportMap(); - private statsTimer: NodeJS.Timer; - - private hasShutdown = false; - - private adsState: AdsState; - - private adsBackoff: BackoffTimeout; - private lrsBackoff: BackoffTimeout; - - constructor(bootstrapInfoOverride?: BootstrapInfo) { - const edsState = new EdsState(() => { - this.updateNames('eds'); - }); - const cdsState = new CdsState(() => { - this.updateNames('cds'); - }); - const rdsState = new RdsState(() => { - this.updateNames('rds'); - }); - const ldsState = new LdsState(rdsState, () => { - this.updateNames('lds'); - }); - this.adsState = { - eds: edsState, - cds: cdsState, - rds: rdsState, - lds: ldsState, - }; - - const channelArgs = { - // 5 minutes - 'grpc.keepalive_time_ms': 5 * 60 * 1000 - } - - this.adsBackoff = new BackoffTimeout(() => { - this.maybeStartAdsStream(); - }); - this.adsBackoff.unref(); - this.lrsBackoff = new BackoffTimeout(() => { - this.maybeStartLrsStream(); - }); - this.lrsBackoff.unref(); - - async function getBootstrapInfo(): Promise { - if (bootstrapInfoOverride) { - return bootstrapInfoOverride; - } else { - return loadBootstrapInfo(); - } - } - - Promise.all([getBootstrapInfo(), loadAdsProtos()]).then( - ([bootstrapInfo, protoDefinitions]) => { - if (this.hasShutdown) { - return; - } - trace('Loaded bootstrap info: ' + JSON.stringify(bootstrapInfo, undefined, 2)); - if (bootstrapInfo.xdsServers.length < 1) { - trace('Failed to initialize xDS Client. No servers provided in bootstrap info.'); - // Bubble this error up to any listeners - this.reportStreamError({ - code: status.INTERNAL, - details: 'Failed to initialize xDS Client. No servers provided in bootstrap info.', - metadata: new Metadata(), - }); - return; - } - if (bootstrapInfo.xdsServers[0].serverFeatures.indexOf('ignore_resource_deletion') >= 0) { - this.adsState.lds.enableIgnoreResourceDeletion(); - this.adsState.cds.enableIgnoreResourceDeletion(); - } - const userAgentName = 'gRPC Node Pure JS'; - this.adsNode = { - ...bootstrapInfo.node, - user_agent_name: userAgentName, - user_agent_version: clientVersion, - client_features: ['envoy.lb.does_not_support_overprovisioning'], - }; - this.lrsNode = { - ...bootstrapInfo.node, - user_agent_name: userAgentName, - user_agent_version: clientVersion, - client_features: ['envoy.lrs.supports_send_all_clusters'], - }; - setCsdsClientNode(this.adsNode); - trace('ADS Node: ' + JSON.stringify(this.adsNode, undefined, 2)); - trace('LRS Node: ' + JSON.stringify(this.lrsNode, undefined, 2)); - const credentialsConfigs = bootstrapInfo.xdsServers[0].channelCreds; - let channelCreds: ChannelCredentials | null = null; - for (const config of credentialsConfigs) { - if (config.type === 'google_default') { - channelCreds = createGoogleDefaultCredentials(); - break; - } else if (config.type === 'insecure') { - channelCreds = ChannelCredentials.createInsecure(); - break; - } - } - if (channelCreds === null) { - trace('Failed to initialize xDS Client. No valid credentials types found.'); - // Bubble this error up to any listeners - this.reportStreamError({ - code: status.INTERNAL, - details: 'Failed to initialize xDS Client. No valid credentials types found.', - metadata: new Metadata(), - }); - return; - } - const serverUri = bootstrapInfo.xdsServers[0].serverUri - trace('Starting xDS client connected to server URI ' + bootstrapInfo.xdsServers[0].serverUri); - const channel = new Channel(serverUri, channelCreds, channelArgs); - this.adsClient = new protoDefinitions.envoy.service.discovery.v3.AggregatedDiscoveryService( - serverUri, - channelCreds, - {channelOverride: channel} - ); - this.maybeStartAdsStream(); - channel.watchConnectivityState(channel.getConnectivityState(false), Infinity, () => { - this.handleAdsConnectivityStateUpdate(); - }) - - this.lrsClient = new protoDefinitions.envoy.service.load_stats.v3.LoadReportingService( - serverUri, - channelCreds, - {channelOverride: channel} - ); - this.maybeStartLrsStream(); - }).catch((error) => { - trace('Failed to initialize xDS Client. ' + error.message); - // Bubble this error up to any listeners - this.reportStreamError({ - code: status.INTERNAL, - details: `Failed to initialize xDS Client. ${error.message}`, - metadata: new Metadata(), - }); - } - ); - this.statsTimer = setInterval(() => {}, 0); - clearInterval(this.statsTimer); - } - - private handleAdsConnectivityStateUpdate() { - if (!this.adsClient) { - return; - } - const state = this.adsClient.getChannel().getConnectivityState(false); - if (state === connectivityState.READY && this.adsCall) { - this.reportAdsStreamStarted(); - } - if (state === connectivityState.TRANSIENT_FAILURE) { - this.reportStreamError({ - code: status.UNAVAILABLE, - details: 'No connection established to xDS server', - metadata: new Metadata() - }); - } - this.adsClient.getChannel().watchConnectivityState(state, Infinity, () => { - this.handleAdsConnectivityStateUpdate(); - }); - } - - private handleAdsResponse(message: DiscoveryResponse__Output) { - this.receivedAdsResponseOnCurrentStream = true; - this.adsBackoff.reset(); - let handleResponseResult: { - result: HandleResponseResult; - serviceKind: AdsServiceKind; - } | null = null; - try { - switch (message.type_url) { - case EDS_TYPE_URL: - handleResponseResult = { - result: this.adsState.eds.handleResponses( - getResponseMessages(EDS_TYPE_URL, message.resources) - ), - serviceKind: 'eds' - }; - break; - case CDS_TYPE_URL: - handleResponseResult = { - result: this.adsState.cds.handleResponses( - getResponseMessages(CDS_TYPE_URL, message.resources) - ), - serviceKind: 'cds' - }; - break; - case RDS_TYPE_URL: - handleResponseResult = { - result: this.adsState.rds.handleResponses( - getResponseMessages(RDS_TYPE_URL, message.resources) - ), - serviceKind: 'rds' - }; - break; - case LDS_TYPE_URL: - handleResponseResult = { - result: this.adsState.lds.handleResponses( - getResponseMessages(LDS_TYPE_URL, message.resources) - ), - serviceKind: 'lds' - } - break; - } - } catch (e) { - trace('Nacking message with protobuf parsing error: ' + e.message); - this.nack(message.type_url, e.message); - return; - } - if (handleResponseResult === null) { - // Null handleResponseResult means that the type_url was unrecognized - trace('Nacking message with unknown type URL ' + message.type_url); - this.nack(message.type_url, `Unknown type_url ${message.type_url}`); - } else { - updateCsdsResourceResponse(message.type_url as AdsTypeUrl, message.version_info, handleResponseResult.result); - if (handleResponseResult.result.rejected.length > 0) { - // rejected.length > 0 means that at least one message validation failed - const errorString = `${handleResponseResult.serviceKind.toUpperCase()} Error: ${handleResponseResult.result.rejected[0].error}`; - trace('Nacking message with type URL ' + message.type_url + ': ' + errorString); - this.nack(message.type_url, errorString); - } else { - // If we get here, all message validation succeeded - trace('Acking message with type URL ' + message.type_url); - const serviceKind = handleResponseResult.serviceKind; - this.adsState[serviceKind].nonce = message.nonce; - this.adsState[serviceKind].versionInfo = message.version_info; - this.ack(serviceKind); - } - } - } - - private handleAdsCallStatus(streamStatus: StatusObject) { - trace( - 'ADS stream ended. code=' + streamStatus.code + ' details= ' + streamStatus.details - ); - this.adsCall = null; - if (streamStatus.code !== status.OK && !this.receivedAdsResponseOnCurrentStream) { - this.reportStreamError(streamStatus); - } - /* If the backoff timer is no longer running, we do not need to wait any - * more to start the new call. */ - if (!this.adsBackoff.isRunning()) { - this.maybeStartAdsStream(); - } - } - - /** - * Start the ADS stream if the client exists and there is not already an - * existing stream, and there are resources to request. - */ - private maybeStartAdsStream() { - if (this.hasShutdown) { - return; - } - if (this.adsState.eds.getResourceNames().length === 0 && - this.adsState.cds.getResourceNames().length === 0 && - this.adsState.rds.getResourceNames().length === 0 && - this.adsState.lds.getResourceNames().length === 0) { - return; - } - if (this.adsClient === null) { - return; - } - if (this.adsCall !== null) { - return; - } - this.receivedAdsResponseOnCurrentStream = false; - const metadata = new Metadata({waitForReady: true}); - this.adsCall = this.adsClient.StreamAggregatedResources(metadata); - this.adsCall.on('data', (message: DiscoveryResponse__Output) => { - this.handleAdsResponse(message); - }); - this.adsCall.on('status', (status: StatusObject) => { - this.handleAdsCallStatus(status); - }); - this.adsCall.on('error', () => {}); - trace('Started ADS stream'); - // Backoff relative to when we start the request - this.adsBackoff.runOnce(); - - const allServiceKinds: AdsServiceKind[] = ['eds', 'cds', 'rds', 'lds']; - for (const service of allServiceKinds) { - const state = this.adsState[service]; - if (state.getResourceNames().length > 0) { - this.updateNames(service); - } - } - if (this.adsClient.getChannel().getConnectivityState(false) === connectivityState.READY) { - this.reportAdsStreamStarted(); - } - } - private maybeSendAdsMessage(typeUrl: string, resourceNames: string[], responseNonce: string, versionInfo: string, errorMessage?: string) { - this.adsCall?.write({ - node: this.adsNode!, - type_url: typeUrl, - resource_names: resourceNames, - response_nonce: responseNonce, - version_info: versionInfo, - error_detail: errorMessage ? { message: errorMessage } : undefined - }); - } - - private getTypeUrl(serviceKind: AdsServiceKind): AdsTypeUrl { - switch (serviceKind) { - case 'eds': - return EDS_TYPE_URL; - case 'cds': - return CDS_TYPE_URL; - case 'rds': - return RDS_TYPE_URL; - case 'lds': - return LDS_TYPE_URL; + unref(clusterName: string, edsServiceName: string) { + for (const statsObj of this.statsMap) { + if ( + statsObj.clusterName === clusterName && + statsObj.edsServiceName === edsServiceName + ) { + statsObj.refCount -=1; + if (statsObj.refCount === 0) { + this.statsMap.delete(statsObj); + } + return; + } } } - /** - * Acknowledge an update. This should be called after the local nonce and - * version info are updated so that it sends the post-update values. - */ - ack(serviceKind: AdsServiceKind) { - this.updateNames(serviceKind); - } - - /** - * Reject an update. This should be called without updating the local - * nonce and version info. - */ - private nack(typeUrl: string, message: string) { - let resourceNames: string[]; - let nonce: string; - let versionInfo: string; - let serviceKind: AdsServiceKind | null; - switch (typeUrl) { - case EDS_TYPE_URL: - serviceKind = 'eds'; - break; - case CDS_TYPE_URL: - serviceKind = 'cds'; - break; - case RDS_TYPE_URL: - serviceKind = 'rds'; - break; - case LDS_TYPE_URL: - serviceKind = 'lds'; - break; - default: - serviceKind = null; - break; - } - if (serviceKind) { - this.adsState[serviceKind].reportStreamError({ - code: status.UNAVAILABLE, - details: message + ' Node ID=' + this.adsNode!.id, - metadata: new Metadata() - }); - resourceNames = this.adsState[serviceKind].getResourceNames(); - nonce = this.adsState[serviceKind].nonce; - versionInfo = this.adsState[serviceKind].versionInfo; - } else { - resourceNames = []; - nonce = ''; - versionInfo = ''; - } - this.maybeSendAdsMessage(typeUrl, resourceNames, nonce, versionInfo, message); - } - - private updateNames(serviceKind: AdsServiceKind) { - if (this.adsState.eds.getResourceNames().length === 0 && - this.adsState.cds.getResourceNames().length === 0 && - this.adsState.rds.getResourceNames().length === 0 && - this.adsState.lds.getResourceNames().length === 0) { - this.adsCall?.end(); - this.adsCall = null; - this.lrsCall?.end(); - this.lrsCall = null; - return; - } - this.maybeStartAdsStream(); - this.maybeStartLrsStream(); - if (!this.adsCall) { - /* If the stream is not set up yet at this point, shortcut the rest - * becuase nothing will actually be sent. This would mainly happen if - * the bootstrap file has not been read yet. In that case, the output - * of getTypeUrl is garbage and everything after that is invalid. */ - return; - } - trace('Sending update for ' + serviceKind + ' with names ' + this.adsState[serviceKind].getResourceNames()); - const typeUrl = this.getTypeUrl(serviceKind); - updateCsdsRequestedNameList(typeUrl, this.adsState[serviceKind].getResourceNames()); - this.maybeSendAdsMessage(typeUrl, this.adsState[serviceKind].getResourceNames(), this.adsState[serviceKind].nonce, this.adsState[serviceKind].versionInfo); + get size() { + return this.statsMap.size; } +} - private reportStreamError(status: StatusObject) { - status = {...status, details: status.details + ' Node ID=' + this.adsNode!.id}; - this.adsState.eds.reportStreamError(status); - this.adsState.cds.reportStreamError(status); - this.adsState.rds.reportStreamError(status); - this.adsState.lds.reportStreamError(status); +class LrsCallState { + private statsTimer: NodeJS.Timer | null = null; + private sentInitialMessage = false; + constructor(private client: XdsSingleServerClient, private call: LrsCall, private node: Node) { + call.on('data', (message: LoadStatsResponse__Output) => { + this.handleResponseMessage(message); + }) + call.on('status', (status: StatusObject) => { + this.handleStreamStatus(status); + }); + call.on('error', () => {}); + this.sendStats(); } - private reportAdsStreamStarted() { - this.adsState.eds.reportAdsStreamStart(); - this.adsState.cds.reportAdsStreamStart(); - this.adsState.rds.reportAdsStreamStart(); - this.adsState.lds.reportAdsStreamStart(); + private handleStreamStatus(status: StatusObject) { + this.client.trace( + 'LRS stream ended. code=' + status.code + ' details= ' + status.details + ); + this.client.handleLrsStreamEnd(); } - private handleLrsResponse(message: LoadStatsResponse__Output) { - trace('Received LRS response'); - /* Once we get any response from the server, we assume that the stream is - * in a good state, so we can reset the backoff timer. */ - this.lrsBackoff.reset(); + private handleResponseMessage(message: LoadStatsResponse__Output) { + this.client.trace('Received LRS response'); + this.client.onLrsStreamReceivedMessage(); if ( - !this.receivedLrsSettingsForCurrentStream || + !this.statsTimer || message.load_reporting_interval?.seconds !== - this.latestLrsSettings?.load_reporting_interval?.seconds || + this.client.latestLrsSettings?.load_reporting_interval?.seconds || message.load_reporting_interval?.nanos !== - this.latestLrsSettings?.load_reporting_interval?.nanos + this.client.latestLrsSettings?.load_reporting_interval?.nanos ) { /* Only reset the timer if the interval has changed or was not set * before. */ - clearInterval(this.statsTimer); + if (this.statsTimer) { + clearInterval(this.statsTimer); + } /* Convert a google.protobuf.Duration to a number of milliseconds for * use with setInterval. */ const loadReportingIntervalMs = Number.parseInt(message.load_reporting_interval!.seconds) * 1000 + message.load_reporting_interval!.nanos / 1_000_000; - trace('Received LRS response with load reporting interval ' + loadReportingIntervalMs + ' ms'); + this.client.trace('Received LRS response with load reporting interval ' + loadReportingIntervalMs + ' ms'); this.statsTimer = setInterval(() => { this.sendStats(); }, loadReportingIntervalMs); } - this.latestLrsSettings = message; - this.receivedLrsSettingsForCurrentStream = true; - } - - private handleLrsCallStatus(streamStatus: StatusObject) { - trace( - 'LRS stream ended. code=' + streamStatus.code + ' details= ' + streamStatus.details - ); - this.lrsCall = null; - clearInterval(this.statsTimer); - /* If the backoff timer is no longer running, we do not need to wait any - * more to start the new call. */ - if (!this.lrsBackoff.isRunning()) { - this.maybeStartLrsStream(); - } - } - - private maybeStartLrsStreamV3(): boolean { - if (!this.lrsClient) { - return false; - } - if (this.lrsCall) { - return false; - } - this.lrsCall = this.lrsClient.streamLoadStats(); - this.receivedLrsSettingsForCurrentStream = false; - this.lrsCall.on('data', (message: LoadStatsResponse__Output) => { - this.handleLrsResponse(message); - }); - this.lrsCall.on('status', (status: StatusObject) => { - this.handleLrsCallStatus(status); - }); - this.lrsCall.on('error', () => {}); - return true; + this.client.latestLrsSettings = message; } - private maybeStartLrsStream() { - if (this.hasShutdown) { - return; - } - if (this.adsState.eds.getResourceNames().length === 0 && - this.adsState.cds.getResourceNames().length === 0 && - this.adsState.rds.getResourceNames().length === 0 && - this.adsState.lds.getResourceNames().length === 0) { - return; - } - if (!this.lrsClient) { - return; - } - if (this.lrsCall) { - return; - } - this.lrsCall = this.lrsClient.streamLoadStats(); - this.receivedLrsSettingsForCurrentStream = false; - this.lrsCall.on('data', (message: LoadStatsResponse__Output) => { - this.handleLrsResponse(message); - }); - this.lrsCall.on('status', (status: StatusObject) => { - this.handleLrsCallStatus(status); - }); - this.lrsCall.on('error', () => {}); - trace('Starting LRS stream'); - this.lrsBackoff.runOnce(); - /* Send buffered stats information when starting LRS stream. If there is no - * buffered stats information, it will still send the node field. */ - this.sendStats(); + private sendLrsMessage(clusterStats: ClusterStats[]) { + const request: LoadStatsRequest = { + node: this.sentInitialMessage ? null : this.node, + cluster_stats: clusterStats + }; + this.client.trace('Sending LRS message ' + JSON.stringify(request, undefined, 2)); + this.call.write(request); + this.sentInitialMessage = true; } - private maybeSendLrsMessage(clusterStats: ClusterStats[]) { - this.lrsCall?.write({ - node: this.lrsNode!, - cluster_stats: clusterStats - }); + private get latestLrsSettings() { + return this.client.latestLrsSettings; } private sendStats() { - if (this.lrsCall === null) { - return; - } if (!this.latestLrsSettings) { - this.maybeSendLrsMessage([]); + this.sendLrsMessage([]); return; } const clusterStats: ClusterStats[] = []; for (const [ { clusterName, edsServiceName }, stats, - ] of this.clusterStatsMap.entries()) { + ] of this.client.clusterStatsMap.entries()) { if ( this.latestLrsSettings.send_all_clusters || this.latestLrsSettings.clusters.indexOf(clusterName) > 0 @@ -844,80 +795,197 @@ export class XdsClient { } } } - trace('Sending LRS stats ' + JSON.stringify(clusterStats, undefined, 2)); - this.maybeSendLrsMessage(clusterStats); + this.sendLrsMessage(clusterStats); + } +} - addEndpointWatcher( - edsServiceName: string, - watcher: Watcher - ) { - trace('Watcher added for endpoint ' + edsServiceName); - this.adsState.eds.addWatcher(edsServiceName, watcher); +class XdsSingleServerClient { + public ignoreResourceDeletion: boolean; + + private adsBackoff: BackoffTimeout; + private lrsBackoff: BackoffTimeout; + + private adsClient: AggregatedDiscoveryServiceClient; + private adsCallState: AdsCallState | null = null; + + private lrsClient: LoadReportingServiceClient; + private lrsCallState: LrsCallState | null = null; + public clusterStatsMap = new ClusterLoadReportMap(); + public latestLrsSettings: LoadStatsResponse__Output | null = null; + + /** + * The number of authorities that are using this client. Streams should only + * be started if refcount > 0 + */ + private refcount = 0; + + /** + * Map of type to latest accepted version string for that type + */ + public resourceTypeVersionMap: Map = new Map(); + constructor(public xdsClient: XdsClient, bootstrapNode: Node, public xdsServerConfig: XdsServerConfig) { + this.adsBackoff = new BackoffTimeout(() => { + this.maybeStartAdsStream(); + }); + this.adsBackoff.unref(); + this.lrsBackoff = new BackoffTimeout(() => { + this.maybeStartLrsStream(); + }); + this.lrsBackoff.unref(); + this.ignoreResourceDeletion = xdsServerConfig.serverFeatures.includes('ignore_resource_deletion'); + const channelArgs = { + // 5 minutes + 'grpc.keepalive_time_ms': 5 * 60 * 1000 + } + const credentialsConfigs = xdsServerConfig.channelCreds; + let channelCreds: ChannelCredentials | null = null; + for (const config of credentialsConfigs) { + if (config.type === 'google_default') { + channelCreds = createGoogleDefaultCredentials(); + break; + } else if (config.type === 'insecure') { + channelCreds = ChannelCredentials.createInsecure(); + break; + } + } + const serverUri = this.xdsServerConfig.serverUri + this.trace('Starting xDS client connected to server URI ' + this.xdsServerConfig.serverUri); + /* Bootstrap validation rules guarantee that a matching channel credentials + * config exists in the list. */ + const channel = new Channel(serverUri, channelCreds!, channelArgs); + const protoDefinitions = loadAdsProtos(); + this.adsClient = new protoDefinitions.envoy.service.discovery.v3.AggregatedDiscoveryService( + serverUri, + channelCreds!, + {channelOverride: channel} + ); + channel.watchConnectivityState(channel.getConnectivityState(false), Infinity, () => { + this.handleAdsConnectivityStateUpdate(); + }); + this.lrsClient = new protoDefinitions.envoy.service.load_stats.v3.LoadReportingService( + serverUri, + channelCreds!, + {channelOverride: channel} + ); } - removeEndpointWatcher( - edsServiceName: string, - watcher: Watcher - ) { - trace('Watcher removed for endpoint ' + edsServiceName); - this.adsState.eds.removeWatcher(edsServiceName, watcher); + private handleAdsConnectivityStateUpdate() { + const state = this.adsClient.getChannel().getConnectivityState(false); + if (state === connectivityState.READY) { + this.adsCallState?.markStreamStarted(); + } + if (state === connectivityState.TRANSIENT_FAILURE) { + for (const authorityState of this.xdsClient.authorityStateMap.values()) { + if (authorityState.client !== this) { + continue; + } + for (const typeMap of authorityState.resourceMap.values()) { + for (const resourceState of typeMap.values()) { + for (const watcher of resourceState.watchers) { + watcher.onError({ + code: status.UNAVAILABLE, + details: 'No connection established to xDS server', + metadata: new Metadata() + }); + } + } + } + } + } + this.adsClient.getChannel().watchConnectivityState(state, Infinity, () => { + this.handleAdsConnectivityStateUpdate(); + }); } - addClusterWatcher(clusterName: string, watcher: Watcher) { - trace('Watcher added for cluster ' + clusterName); - this.adsState.cds.addWatcher(clusterName, watcher); + onAdsStreamReceivedMessage() { + this.adsBackoff.stop(); + this.adsBackoff.reset(); } - removeClusterWatcher(clusterName: string, watcher: Watcher) { - trace('Watcher removed for cluster ' + clusterName); - this.adsState.cds.removeWatcher(clusterName, watcher); + handleAdsStreamEnd() { + this.adsCallState = null; + /* The backoff timer would start the stream when it finishes. If it is not + * running, restart the stream immediately. */ + if (!this.adsBackoff.isRunning()) { + this.maybeStartAdsStream(); + } } - addRouteWatcher(routeConfigName: string, watcher: Watcher) { - trace('Watcher added for route ' + routeConfigName); - this.adsState.rds.addWatcher(routeConfigName, watcher); + private maybeStartAdsStream() { + if (this.adsCallState || this.refcount < 1) { + return; + } + this.trace('Starting ADS stream'); + const metadata = new Metadata({waitForReady: true}); + const call = this.adsClient.StreamAggregatedResources(metadata); + this.adsCallState = new AdsCallState(this, call, this.xdsClient.adsNode!); + this.adsBackoff.runOnce(); } - removeRouteWatcher(routeConfigName: string, watcher: Watcher) { - trace('Watcher removed for route ' + routeConfigName); - this.adsState.rds.removeWatcher(routeConfigName, watcher); + onLrsStreamReceivedMessage() { + this.adsBackoff.stop(); + this.adsBackoff.reset(); } - addListenerWatcher(targetName: string, watcher: Watcher) { - trace('Watcher added for listener ' + targetName); - this.adsState.lds.addWatcher(targetName, watcher); + handleLrsStreamEnd() { + this.lrsCallState = null; + /* The backoff timer would start the stream when it finishes. If it is not + * running, restart the stream immediately. */ + if (!this.lrsBackoff.isRunning()) { + this.maybeStartLrsStream(); + } } - removeListenerWatcher(targetName: string, watcher: Watcher) { - trace('Watcher removed for listener ' + targetName); - this.adsState.lds.removeWatcher(targetName, watcher); + private maybeStartLrsStream() { + if (this.lrsCallState || this.refcount < 1 || this.clusterStatsMap.size < 1) { + return; + } + this.trace('Starting LRS stream'); + const metadata = new Metadata({waitForReady: true}); + const call = this.lrsClient.StreamLoadStats(metadata); + this.lrsCallState = new LrsCallState(this, call, this.xdsClient.lrsNode!); + this.lrsBackoff.runOnce(); + } + + trace(text: string) { + trace(this.xdsServerConfig.serverUri + ' ' + text); + } + + subscribe(type: XdsResourceType, name: XdsResourceName) { + this.trace('subscribe(type=' + type.getTypeUrl() + ', name=' + xdsResourceNameToString(name, type.getTypeUrl()) + ')'); + this.trace(JSON.stringify(name)); + this.maybeStartAdsStream(); + this.adsCallState?.subscribe(type, name); + } + + unsubscribe(type: XdsResourceType, name: XdsResourceName) { + this.trace('unsubscribe(type=' + type.getTypeUrl() + ', name=' + xdsResourceNameToString(name, type.getTypeUrl()) + ')'); + this.adsCallState?.unsubscribe(type, name); + if (this.adsCallState && !this.adsCallState.hasSubscribedResources()) { + this.adsCallState.end(); + this.adsCallState = null; + } + } + + ref() { + this.refcount += 1; + } + + unref() { + this.refcount -= 1; } - /** - * - * @param lrsServer The target name of the server to send stats to. An empty - * string indicates that the default LRS client should be used. Currently - * only the empty string is supported here. - * @param clusterName - * @param edsServiceName - */ addClusterDropStats( - lrsServer: string, clusterName: string, edsServiceName: string ): XdsClusterDropStats { - trace('addClusterDropStats(lrsServer=' + lrsServer + ', clusterName=' + clusterName + ', edsServiceName=' + edsServiceName + ')'); - if (lrsServer !== '') { - return { - addUncategorizedCallDropped: () => {}, - addCallDropped: (category) => {}, - }; - } + this.trace('addClusterDropStats(clusterName=' + clusterName + ', edsServiceName=' + edsServiceName + ')'); const clusterStats = this.clusterStatsMap.getOrCreate( clusterName, edsServiceName ); + this.maybeStartLrsStream(); return { addUncategorizedCallDropped: () => { clusterStats.uncategorizedCallsDropped += 1; @@ -929,23 +997,22 @@ export class XdsClient { }; } + removeClusterDropStats(clusterName: string, edsServiceName: string) { + this.trace('removeClusterDropStats(clusterName=' + clusterName + ', edsServiceName=' + edsServiceName + ')'); + this.clusterStatsMap.unref(clusterName, edsServiceName); + } + addClusterLocalityStats( - lrsServer: string, clusterName: string, edsServiceName: string, locality: Locality__Output ): XdsClusterLocalityStats { - trace('addClusterLocalityStats(lrsServer=' + lrsServer + ', clusterName=' + clusterName + ', edsServiceName=' + edsServiceName + ', locality=' + JSON.stringify(locality) + ')'); - if (lrsServer !== '') { - return { - addCallStarted: () => {}, - addCallFinished: (fail) => {}, - }; - } + this.trace('addClusterLocalityStats(clusterName=' + clusterName + ', edsServiceName=' + edsServiceName + ', locality=' + JSON.stringify(locality) + ')'); const clusterStats = this.clusterStatsMap.getOrCreate( clusterName, edsServiceName ); + this.maybeStartLrsStream(); let localityStats: ClusterLocalityStats | null = null; for (const statsObj of clusterStats.localityStats) { if (localityEqual(locality, statsObj.locality)) { @@ -960,8 +1027,9 @@ export class XdsClient { callsStarted: 0, callsSucceeded: 0, callsFailed: 0, + refcount: 0, }; - clusterStats.localityStats.push(localityStats); + clusterStats.localityStats.add(localityStats); } /* Help the compiler understand that this object is always non-null in the * closure */ @@ -982,12 +1050,249 @@ export class XdsClient { }; } - private shutdown(): void { - this.adsCall?.cancel(); - this.adsClient?.close(); - this.lrsCall?.cancel(); - this.lrsClient?.close(); - this.hasShutdown = true; + removeClusterLocalityStats( + clusterName: string, + edsServiceName: string, + locality: Locality__Output + ) { + this.trace('removeClusterLocalityStats(clusterName=' + clusterName + ', edsServiceName=' + edsServiceName + ', locality=' + JSON.stringify(locality) + ')'); + const clusterStats = this.clusterStatsMap.get(clusterName, edsServiceName); + if (!clusterStats) { + return; + } + for (const statsObj of clusterStats.localityStats) { + if (localityEqual(locality, statsObj.locality)) { + statsObj.refcount -= 1; + if (statsObj.refcount === 0) { + clusterStats.localityStats.delete(statsObj); + } + break; + } + } + this.clusterStatsMap.unref(clusterName, edsServiceName); + } +} + +interface ClientMapEntry { + serverConfig: XdsServerConfig; + client: XdsSingleServerClient; +} + +type ClientResourceStatus = 'REQUESTED' | 'DOES_NOT_EXIST' | 'ACKED' | 'NACKED'; + +interface ResourceMetadata { + clientStatus: ClientResourceStatus; + rawResource?: Any__Output; + updateTime?: Date; + version?: string; + failedVersion?: string; + failedDetails?: string; + failedUpdateTime?: Date; +} + +interface ResourceState { + watchers: Set; + cachedResource: object | null; + meta: ResourceMetadata; + deletionIgnored: boolean; +} + +interface AuthorityState { + client: XdsSingleServerClient; + /** + * type -> key -> state + */ + resourceMap: Map>; +} + +const userAgentName = 'gRPC Node Pure JS'; + +export class XdsClient { + /** + * authority -> authority state + */ + public authorityStateMap: Map = new Map(); + private clients: ClientMapEntry[] = []; + private typeRegistry: Map = new Map(); + private bootstrapInfo: BootstrapInfo | null = null; + + constructor(bootstrapInfoOverride?: BootstrapInfo) { + if (bootstrapInfoOverride) { + this.bootstrapInfo = bootstrapInfoOverride; + } + registerXdsClientWithCsds(this); + } + + private getBootstrapInfo() { + if (!this.bootstrapInfo) { + this.bootstrapInfo = loadBootstrapInfo(); + } + return this.bootstrapInfo; + } + + get adsNode(): Node | undefined { + if (!this.bootstrapInfo) { + return undefined; + } + return { + ...this.bootstrapInfo.node, + user_agent_name: userAgentName, + user_agent_version: clientVersion, + client_features: ['envoy.lb.does_not_support_overprovisioning'], + } + } + + get lrsNode(): Node | undefined { + if (!this.bootstrapInfo) { + return undefined; + } + return { + ...this.bootstrapInfo.node, + user_agent_name: userAgentName, + user_agent_version: clientVersion, + client_features: ['envoy.lrs.supports_send_all_clusters'], + }; + } + + private getOrCreateClient(authority: string): XdsSingleServerClient { + const bootstrapInfo = this.getBootstrapInfo(); + let serverConfig: XdsServerConfig; + if (authority === 'old:') { + serverConfig = bootstrapInfo.xdsServers[0]; + } else { + if (authority in bootstrapInfo.authorities) { + serverConfig = bootstrapInfo.authorities[authority].xdsServers?.[0] ?? bootstrapInfo.xdsServers[0]; + } else { + throw new Error(`Authority ${authority} not found in bootstrap authorities list`); + } + } + for (const entry of this.clients) { + if (serverConfigEqual(serverConfig, entry.serverConfig)) { + return entry.client; + } + } + const client = new XdsSingleServerClient(this, bootstrapInfo.node, serverConfig); + this.clients.push({client, serverConfig}); + return client; + } + + private getClient(server: XdsServerConfig) { + for (const entry of this.clients) { + if (serverConfigEqual(server, entry.serverConfig)) { + return entry.client; + } + } + return undefined; + } + + getResourceType(typeUrl: string) { + return this.typeRegistry.get(typeUrl); + } + + watchResource(type: XdsResourceType, name: string, watcher: ResourceWatcherInterface) { + trace('watchResource(type=' + type.getTypeUrl() + ', name=' + name + ')'); + if (this.typeRegistry.has(type.getTypeUrl())) { + if (this.typeRegistry.get(type.getTypeUrl()) !== type) { + throw new Error(`Resource type does not match previously used type with the same type URL: ${type.getTypeUrl()}`); + } + } else { + this.typeRegistry.set(type.getTypeUrl(), type); + this.typeRegistry.set(type.getFullTypeUrl(), type); + } + const resourceName = parseXdsResourceName(name, type.getTypeUrl()); + let authorityState = this.authorityStateMap.get(resourceName.authority); + if (!authorityState) { + authorityState = { + client: this.getOrCreateClient(resourceName.authority), + resourceMap: new Map() + }; + authorityState.client.ref(); + this.authorityStateMap.set(resourceName.authority, authorityState); + } + let keyMap = authorityState.resourceMap.get(type); + if (!keyMap) { + keyMap = new Map(); + authorityState.resourceMap.set(type, keyMap); + } + let entry = keyMap.get(resourceName.key); + let isNewSubscription = false; + if (!entry) { + isNewSubscription = true; + entry = { + watchers: new Set(), + cachedResource: null, + deletionIgnored: false, + meta: { + clientStatus: 'REQUESTED' + } + }; + keyMap.set(resourceName.key, entry); + } + entry.watchers.add(watcher); + if (entry.cachedResource) { + process.nextTick(() => { + if (entry?.cachedResource) { + watcher.onGenericResourceChanged(entry.cachedResource); + } + }); + } + if (isNewSubscription) { + authorityState.client.subscribe(type, resourceName); + } + } + + cancelResourceWatch(type: XdsResourceType, name: string, watcher: ResourceWatcherInterface) { + trace('cancelResourceWatch(type=' + type.getTypeUrl() + ', name=' + name + ')'); + const resourceName = parseXdsResourceName(name, type.getTypeUrl()); + const authorityState = this.authorityStateMap.get(resourceName.authority); + if (!authorityState) { + return; + } + const entry = authorityState.resourceMap.get(type)?.get(resourceName.key); + if (entry) { + entry.watchers.delete(watcher); + if (entry.watchers.size === 0) { + authorityState.resourceMap.get(type)!.delete(resourceName.key); + authorityState.client.unsubscribe(type, resourceName); + if (authorityState.resourceMap.get(type)!.size === 0) { + authorityState.resourceMap.delete(type); + if (authorityState.resourceMap.size === 0) { + authorityState.client.unref(); + this.authorityStateMap.delete(resourceName.authority); + } + } + } + } + } + + addClusterDropStats(lrsServer: XdsServerConfig, clusterName: string, edsServiceName: string): XdsClusterDropStats { + const client = this.getClient(lrsServer); + if (!client) { + return { + addUncategorizedCallDropped: () => {}, + addCallDropped: (category) => {}, + }; + } + return client.addClusterDropStats(clusterName, edsServiceName); + } + + removeClusterDropStats(lrsServer: XdsServerConfig, clusterName: string, edsServiceName: string) { + this.getClient(lrsServer)?.removeClusterDropStats(clusterName, edsServiceName); + } + + addClusterLocalityStats(lrsServer: XdsServerConfig, clusterName: string, edsServiceName: string, locality: Locality__Output): XdsClusterLocalityStats { + const client = this.getClient(lrsServer); + if (!client) { + return { + addCallStarted: () => {}, + addCallFinished: (fail) => {}, + }; + } + return client.addClusterLocalityStats(clusterName, edsServiceName, locality); + } + + removeClusterLocalityStats(lrsServer: XdsServerConfig, clusterName: string, edsServiceName: string, locality: Locality__Output) { + this.getClient(lrsServer)?.removeClusterLocalityStats(clusterName, edsServiceName, locality); } } @@ -998,4 +1303,4 @@ export function getSingletonXdsClient(): XdsClient { singletonXdsClient = new XdsClient(); } return singletonXdsClient; -} \ No newline at end of file +} diff --git a/packages/grpc-js-xds/src/xds-resource-type/cluster-resource-type.ts b/packages/grpc-js-xds/src/xds-resource-type/cluster-resource-type.ts new file mode 100644 index 000000000..f431bf238 --- /dev/null +++ b/packages/grpc-js-xds/src/xds-resource-type/cluster-resource-type.ts @@ -0,0 +1,277 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { CDS_TYPE_URL, CLUSTER_CONFIG_TYPE_URL, decodeSingleResource } from "../resources"; +import { XdsDecodeContext, XdsDecodeResult, XdsResourceType } from "./xds-resource-type"; +import { experimental } from "@grpc/grpc-js"; +import { XdsServerConfig } from "../xds-bootstrap"; +import { Duration__Output } from "../generated/google/protobuf/Duration"; +import { OutlierDetection__Output } from "../generated/envoy/config/cluster/v3/OutlierDetection"; +import { EXPERIMENTAL_OUTLIER_DETECTION } from "../environment"; +import { Cluster__Output } from "../generated/envoy/config/cluster/v3/Cluster"; +import { UInt32Value__Output } from "../generated/google/protobuf/UInt32Value"; +import { Any__Output } from "../generated/google/protobuf/Any"; + +import SuccessRateEjectionConfig = experimental.SuccessRateEjectionConfig; +import FailurePercentageEjectionConfig = experimental.FailurePercentageEjectionConfig; +import { Watcher, XdsClient } from "../xds-client"; + +export interface OutlierDetectionUpdate { + intervalMs: number | null; + baseEjectionTimeMs: number | null; + maxEjectionTimeMs: number | null; + maxEjectionPercent: number | null; + successRateConfig: Partial | null; + failurePercentageConfig: Partial | null; +} + +export interface CdsUpdate { + type: 'AGGREGATE' | 'EDS' | 'LOGICAL_DNS'; + name: string; + aggregateChildren: string[]; + lrsLoadReportingServer?: XdsServerConfig; + maxConcurrentRequests?: number; + edsServiceName?: string; + dnsHostname?: string; + outlierDetectionUpdate?: OutlierDetectionUpdate; +} + +function durationToMs(duration: Duration__Output): number { + return (Number(duration.seconds) * 1_000 + duration.nanos / 1_000_000) | 0; +} + +function convertOutlierDetectionUpdate(outlierDetection: OutlierDetection__Output | null): OutlierDetectionUpdate | undefined { + if (!EXPERIMENTAL_OUTLIER_DETECTION) { + return undefined; + } + if (!outlierDetection) { + /* No-op outlier detection config, with all fields unset. */ + return { + intervalMs: null, + baseEjectionTimeMs: null, + maxEjectionTimeMs: null, + maxEjectionPercent: null, + successRateConfig: null, + failurePercentageConfig: null + }; + } + let successRateConfig: Partial | null = null; + /* Success rate ejection is enabled by default, so we only disable it if + * enforcing_success_rate is set and it has the value 0 */ + if (!outlierDetection.enforcing_success_rate || outlierDetection.enforcing_success_rate.value > 0) { + successRateConfig = { + enforcement_percentage: outlierDetection.enforcing_success_rate?.value, + minimum_hosts: outlierDetection.success_rate_minimum_hosts?.value, + request_volume: outlierDetection.success_rate_request_volume?.value, + stdev_factor: outlierDetection.success_rate_stdev_factor?.value + }; + } + let failurePercentageConfig: Partial | null = null; + /* Failure percentage ejection is disabled by default, so we only enable it + * if enforcing_failure_percentage is set and it has a value greater than 0 */ + if (outlierDetection.enforcing_failure_percentage && outlierDetection.enforcing_failure_percentage.value > 0) { + failurePercentageConfig = { + enforcement_percentage: outlierDetection.enforcing_failure_percentage.value, + minimum_hosts: outlierDetection.failure_percentage_minimum_hosts?.value, + request_volume: outlierDetection.failure_percentage_request_volume?.value, + threshold: outlierDetection.failure_percentage_threshold?.value + } + } + return { + intervalMs: outlierDetection.interval ? durationToMs(outlierDetection.interval) : null, + baseEjectionTimeMs: outlierDetection.base_ejection_time ? durationToMs(outlierDetection.base_ejection_time) : null, + maxEjectionTimeMs: outlierDetection.max_ejection_time ? durationToMs(outlierDetection.max_ejection_time) : null, + maxEjectionPercent : outlierDetection.max_ejection_percent?.value ?? null, + successRateConfig: successRateConfig, + failurePercentageConfig: failurePercentageConfig + }; +} + + +export class ClusterResourceType extends XdsResourceType { + private static singleton: ClusterResourceType = new ClusterResourceType(); + + private constructor() { + super(); + } + + static get() { + return ClusterResourceType.singleton; + } + + getTypeUrl(): string { + return 'envoy.config.cluster.v3.Cluster'; + } + + private validateNonnegativeDuration(duration: Duration__Output | null): boolean { + if (!duration) { + return true; + } + /* The maximum values here come from the official Protobuf documentation: + * https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Duration + */ + return Number(duration.seconds) >= 0 && + Number(duration.seconds) <= 315_576_000_000 && + duration.nanos >= 0 && + duration.nanos <= 999_999_999; + } + + private validatePercentage(percentage: UInt32Value__Output | null): boolean { + if (!percentage) { + return true; + } + return percentage.value >=0 && percentage.value <= 100; + } + + private validateResource(context: XdsDecodeContext, message: Cluster__Output): CdsUpdate | null { + if (message.lb_policy !== 'ROUND_ROBIN') { + return null; + } + if (message.lrs_server) { + if (!message.lrs_server.self) { + return null; + } + } + if (EXPERIMENTAL_OUTLIER_DETECTION) { + if (message.outlier_detection) { + if (!this.validateNonnegativeDuration(message.outlier_detection.interval)) { + return null; + } + if (!this.validateNonnegativeDuration(message.outlier_detection.base_ejection_time)) { + return null; + } + if (!this.validateNonnegativeDuration(message.outlier_detection.max_ejection_time)) { + return null; + } + if (!this.validatePercentage(message.outlier_detection.max_ejection_percent)) { + return null; + } + if (!this.validatePercentage(message.outlier_detection.enforcing_success_rate)) { + return null; + } + if (!this.validatePercentage(message.outlier_detection.failure_percentage_threshold)) { + return null; + } + if (!this.validatePercentage(message.outlier_detection.enforcing_failure_percentage)) { + return null; + } + } + } + if (message.cluster_discovery_type === 'cluster_type') { + if (!(message.cluster_type?.typed_config && message.cluster_type.typed_config.type_url === CLUSTER_CONFIG_TYPE_URL)) { + return null; + } + const clusterConfig = decodeSingleResource(CLUSTER_CONFIG_TYPE_URL, message.cluster_type.typed_config.value); + if (clusterConfig.clusters.length === 0) { + return null; + } + return { + type: 'AGGREGATE', + name: message.name, + aggregateChildren: clusterConfig.clusters, + outlierDetectionUpdate: convertOutlierDetectionUpdate(null) + }; + } else { + let maxConcurrentRequests: number | undefined = undefined; + for (const threshold of message.circuit_breakers?.thresholds ?? []) { + if (threshold.priority === 'DEFAULT') { + maxConcurrentRequests = threshold.max_requests?.value; + } + } + if (message.type === 'EDS') { + if (!message.eds_cluster_config?.eds_config?.ads && !message.eds_cluster_config?.eds_config?.self) { + return null; + } + if (message.name.startsWith('xdstp:') && message.eds_cluster_config.service_name === '') { + return null; + } + return { + type: 'EDS', + name: message.name, + aggregateChildren: [], + maxConcurrentRequests: maxConcurrentRequests, + edsServiceName: message.eds_cluster_config.service_name === '' ? undefined : message.eds_cluster_config.service_name, + lrsLoadReportingServer: message.lrs_server ? context.server : undefined, + outlierDetectionUpdate: convertOutlierDetectionUpdate(message.outlier_detection) + } + } else if (message.type === 'LOGICAL_DNS') { + if (!message.load_assignment) { + return null; + } + if (message.load_assignment.endpoints.length !== 1) { + return null; + } + if (message.load_assignment.endpoints[0].lb_endpoints.length !== 1) { + return null; + } + const socketAddress = message.load_assignment.endpoints[0].lb_endpoints[0].endpoint?.address?.socket_address; + if (!socketAddress) { + return null; + } + if (socketAddress.address === '') { + return null; + } + if (socketAddress.port_specifier !== 'port_value') { + return null; + } + return { + type: 'LOGICAL_DNS', + name: message.name, + aggregateChildren: [], + maxConcurrentRequests: maxConcurrentRequests, + dnsHostname: `${socketAddress.address}:${socketAddress.port_value}`, + lrsLoadReportingServer: message.lrs_server ? context.server : undefined, + outlierDetectionUpdate: convertOutlierDetectionUpdate(message.outlier_detection) + }; + } + } + return null; + } + + decode(context:XdsDecodeContext, resource: Any__Output): XdsDecodeResult { + if (resource.type_url !== CDS_TYPE_URL) { + throw new Error( + `ADS Error: Invalid resource type ${resource.type_url}, expected ${CDS_TYPE_URL}` + ); + } + const message = decodeSingleResource(CDS_TYPE_URL, resource.value); + const validatedMessage = this.validateResource(context, message); + if (validatedMessage) { + return { + name: validatedMessage.name, + value: validatedMessage + }; + } else { + return { + name: message.name, + error: 'Cluster message validation failed' + }; + } + } + + allResourcesRequiredInSotW(): boolean { + return true; + } + + static startWatch(client: XdsClient, name: string, watcher: Watcher) { + client.watchResource(ClusterResourceType.get(), name, watcher); + } + + static cancelWatch(client: XdsClient, name: string, watcher: Watcher) { + client.cancelResourceWatch(ClusterResourceType.get(), name, watcher); + } +} diff --git a/packages/grpc-js-xds/src/xds-stream-state/eds-state.ts b/packages/grpc-js-xds/src/xds-resource-type/endpoint-resource-type.ts similarity index 60% rename from packages/grpc-js-xds/src/xds-stream-state/eds-state.ts rename to packages/grpc-js-xds/src/xds-resource-type/endpoint-resource-type.ts index b043ebbc0..6ffd7788d 100644 --- a/packages/grpc-js-xds/src/xds-stream-state/eds-state.ts +++ b/packages/grpc-js-xds/src/xds-resource-type/endpoint-resource-type.ts @@ -1,27 +1,12 @@ -/* - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import { experimental, logVerbosity, StatusObject } from "@grpc/grpc-js"; -import { isIPv4, isIPv6 } from "net"; +import { experimental, logVerbosity } from "@grpc/grpc-js"; +import { ClusterLoadAssignment__Output } from "../generated/envoy/config/endpoint/v3/ClusterLoadAssignment"; +import { XdsDecodeContext, XdsDecodeResult, XdsResourceType } from "./xds-resource-type"; import { Locality__Output } from "../generated/envoy/config/core/v3/Locality"; import { SocketAddress__Output } from "../generated/envoy/config/core/v3/SocketAddress"; -import { ClusterLoadAssignment__Output } from "../generated/envoy/config/endpoint/v3/ClusterLoadAssignment"; +import { isIPv4, isIPv6 } from "net"; import { Any__Output } from "../generated/google/protobuf/Any"; -import { BaseXdsStreamState, HandleResponseResult, RejectedResourceEntry, ResourcePair, Watcher, XdsStreamState } from "./xds-stream-state"; +import { EDS_TYPE_URL, decodeSingleResource } from "../resources"; +import { Watcher, XdsClient } from "../xds-client"; const TRACER_NAME = 'xds_client'; @@ -39,35 +24,34 @@ function addressesEqual(a: SocketAddress__Output, b: SocketAddress__Output) { return a.address === b.address && a.port_value === b.port_value; } -export class EdsState extends BaseXdsStreamState implements XdsStreamState { - protected getResourceName(resource: ClusterLoadAssignment__Output): string { - return resource.cluster_name; +export class EndpointResourceType extends XdsResourceType { + private static singleton: EndpointResourceType = new EndpointResourceType(); + + private constructor() { + super(); } - protected getProtocolName(): string { - return 'EDS'; + + static get() { + return EndpointResourceType.singleton; } - protected isStateOfTheWorld(): boolean { - return false; + + getTypeUrl(): string { + return 'envoy.config.endpoint.v3.ClusterLoadAssignment'; } - /** - * Validate the ClusterLoadAssignment object by these rules: - * https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md#clusterloadassignment-proto - * @param message - */ - public validateResponse(message: ClusterLoadAssignment__Output) { + private validateResource(message: ClusterLoadAssignment__Output): ClusterLoadAssignment__Output | null { const seenLocalities: {locality: Locality__Output, priority: number}[] = []; const seenAddresses: SocketAddress__Output[] = []; const priorityTotalWeights: Map = new Map(); for (const endpoint of message.endpoints) { if (!endpoint.locality) { trace('EDS validation: endpoint locality unset'); - return false; + return null; } for (const {locality, priority} of seenLocalities) { if (localitiesEqual(endpoint.locality, locality) && endpoint.priority === priority) { trace('EDS validation: endpoint locality duplicated: ' + JSON.stringify(locality) + ', priority=' + priority); - return false; + return null; } } seenLocalities.push({locality: endpoint.locality, priority: endpoint.priority}); @@ -75,20 +59,20 @@ export class EdsState extends BaseXdsStreamState const socketAddress = lb.endpoint?.address?.socket_address; if (!socketAddress) { trace('EDS validation: endpoint socket_address not set'); - return false; + return null; } if (socketAddress.port_specifier !== 'port_value') { trace('EDS validation: socket_address.port_specifier !== "port_value"'); - return false; + return null; } if (!(isIPv4(socketAddress.address) || isIPv6(socketAddress.address))) { trace('EDS validation: address not a valid IPv4 or IPv6 address: ' + socketAddress.address); - return false; + return null; } for (const address of seenAddresses) { if (addressesEqual(socketAddress, address)) { trace('EDS validation: duplicate address seen: ' + address); - return false; + return null; } } seenAddresses.push(socketAddress); @@ -98,15 +82,48 @@ export class EdsState extends BaseXdsStreamState for (const totalWeight of priorityTotalWeights.values()) { if (totalWeight > UINT32_MAX) { trace('EDS validation: total weight > UINT32_MAX') - return false; + return null; } } for (const priority of priorityTotalWeights.keys()) { if (priority > 0 && !priorityTotalWeights.has(priority - 1)) { trace('EDS validation: priorities not contiguous'); - return false; + return null; } } - return true; + return message; } -} \ No newline at end of file + + decode(context: XdsDecodeContext, resource: Any__Output): XdsDecodeResult { + if (resource.type_url !== EDS_TYPE_URL) { + throw new Error( + `ADS Error: Invalid resource type ${resource.type_url}, expected ${EDS_TYPE_URL}` + ); + } + const message = decodeSingleResource(EDS_TYPE_URL, resource.value); + const validatedMessage = this.validateResource(message); + if (validatedMessage) { + return { + name: validatedMessage.cluster_name, + value: validatedMessage + }; + } else { + return { + name: message.cluster_name, + error: 'Endpoint message validation failed' + }; + } + } + + allResourcesRequiredInSotW(): boolean { + return false; + } + + static startWatch(client: XdsClient, name: string, watcher: Watcher) { + client.watchResource(EndpointResourceType.get(), name, watcher); + } + + static cancelWatch(client: XdsClient, name: string, watcher: Watcher) { + client.cancelResourceWatch(EndpointResourceType.get(), name, watcher); + } +} diff --git a/packages/grpc-js-xds/src/xds-resource-type/listener-resource-type.ts b/packages/grpc-js-xds/src/xds-resource-type/listener-resource-type.ts new file mode 100644 index 000000000..20e243b86 --- /dev/null +++ b/packages/grpc-js-xds/src/xds-resource-type/listener-resource-type.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { logVerbosity, experimental } from "@grpc/grpc-js"; +import { EXPERIMENTAL_FAULT_INJECTION } from "../environment"; +import { Listener__Output } from "../generated/envoy/config/listener/v3/Listener"; +import { Any__Output } from "../generated/google/protobuf/Any"; +import { HTTP_CONNECTION_MANGER_TYPE_URL, LDS_TYPE_URL, decodeSingleResource } from "../resources"; +import { XdsDecodeContext, XdsDecodeResult, XdsResourceType } from "./xds-resource-type"; +import { getTopLevelFilterUrl, validateTopLevelFilter } from "../http-filter"; +import { RouteConfigurationResourceType } from "./route-config-resource-type"; +import { Watcher, XdsClient } from "../xds-client"; + +const TRACER_NAME = 'xds_client'; + +function trace(text: string): void { + experimental.trace(logVerbosity.DEBUG, TRACER_NAME, text); +} + +const ROUTER_FILTER_URL = 'type.googleapis.com/envoy.extensions.filters.http.router.v3.Router'; + +export class ListenerResourceType extends XdsResourceType { + private static singleton: ListenerResourceType = new ListenerResourceType(); + private constructor() { + super(); + } + + static get() { + return ListenerResourceType.singleton; + } + getTypeUrl(): string { + return 'envoy.config.listener.v3.Listener'; + } + + private validateResource(message: Listener__Output): Listener__Output | null { + if ( + !( + message.api_listener?.api_listener && + message.api_listener.api_listener.type_url === HTTP_CONNECTION_MANGER_TYPE_URL + ) + ) { + return null; + } + const httpConnectionManager = decodeSingleResource(HTTP_CONNECTION_MANGER_TYPE_URL, message.api_listener!.api_listener.value); + if (EXPERIMENTAL_FAULT_INJECTION) { + const filterNames = new Set(); + for (const [index, httpFilter] of httpConnectionManager.http_filters.entries()) { + if (filterNames.has(httpFilter.name)) { + trace('LDS response validation failed: duplicate HTTP filter name ' + httpFilter.name); + return null; + } + filterNames.add(httpFilter.name); + if (!validateTopLevelFilter(httpFilter)) { + trace('LDS response validation failed: ' + httpFilter.name + ' filter validation failed'); + return null; + } + /* Validate that the last filter, and only the last filter, is the + * router filter. */ + const filterUrl = getTopLevelFilterUrl(httpFilter.typed_config!) + if (index < httpConnectionManager.http_filters.length - 1) { + if (filterUrl === ROUTER_FILTER_URL) { + trace('LDS response validation failed: router filter is before end of list'); + return null; + } + } else { + if (filterUrl !== ROUTER_FILTER_URL) { + trace('LDS response validation failed: final filter is ' + filterUrl); + return null; + } + } + } + } + switch (httpConnectionManager.route_specifier) { + case 'rds': + if (!httpConnectionManager.rds?.config_source?.ads && !httpConnectionManager.rds?.config_source?.self) { + return null; + } + return message; + case 'route_config': + if (!RouteConfigurationResourceType.get().validateResource(httpConnectionManager.route_config!)) { + return null; + } + return message; + } + return null; + } + + decode(context: XdsDecodeContext, resource: Any__Output): XdsDecodeResult { + if (resource.type_url !== LDS_TYPE_URL) { + throw new Error( + `ADS Error: Invalid resource type ${resource.type_url}, expected ${LDS_TYPE_URL}` + ); + } + const message = decodeSingleResource(LDS_TYPE_URL, resource.value); + const validatedMessage = this.validateResource(message); + if (validatedMessage) { + return { + name: validatedMessage.name, + value: validatedMessage + }; + } else { + return { + name: message.name, + error: 'Listener message validation failed' + }; + } + } + + allResourcesRequiredInSotW(): boolean { + return true; + } + + static startWatch(client: XdsClient, name: string, watcher: Watcher) { + client.watchResource(ListenerResourceType.get(), name, watcher); + } + + static cancelWatch(client: XdsClient, name: string, watcher: Watcher) { + client.cancelResourceWatch(ListenerResourceType.get(), name, watcher); + } +} diff --git a/packages/grpc-js-xds/src/xds-stream-state/rds-state.ts b/packages/grpc-js-xds/src/xds-resource-type/route-config-resource-type.ts similarity index 69% rename from packages/grpc-js-xds/src/xds-stream-state/rds-state.ts rename to packages/grpc-js-xds/src/xds-resource-type/route-config-resource-type.ts index 57e25edd6..cdc2e3196 100644 --- a/packages/grpc-js-xds/src/xds-stream-state/rds-state.ts +++ b/packages/grpc-js-xds/src/xds-resource-type/route-config-resource-type.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 gRPC authors. + * Copyright 2023 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,9 +18,12 @@ import { EXPERIMENTAL_FAULT_INJECTION, EXPERIMENTAL_RETRY } from "../environment"; import { RetryPolicy__Output } from "../generated/envoy/config/route/v3/RetryPolicy"; import { RouteConfiguration__Output } from "../generated/envoy/config/route/v3/RouteConfiguration"; +import { Any__Output } from "../generated/google/protobuf/Any"; import { Duration__Output } from "../generated/google/protobuf/Duration"; import { validateOverrideFilter } from "../http-filter"; -import { BaseXdsStreamState, XdsStreamState } from "./xds-stream-state"; +import { RDS_TYPE_URL, decodeSingleResource } from "../resources"; +import { Watcher, XdsClient } from "../xds-client"; +import { XdsDecodeContext, XdsDecodeResult, XdsResourceType } from "./xds-resource-type"; const SUPPORTED_PATH_SPECIFIERS = ['prefix', 'path', 'safe_regex']; const SUPPPORTED_HEADER_MATCH_SPECIFIERS = [ @@ -42,15 +45,19 @@ function durationToMs(duration: Duration__Output | null): number | null { return (Number.parseInt(duration.seconds) * 1000 + duration.nanos / 1_000_000) | 0; } -export class RdsState extends BaseXdsStreamState implements XdsStreamState { - protected isStateOfTheWorld(): boolean { - return false; +export class RouteConfigurationResourceType extends XdsResourceType { + private static singleton: RouteConfigurationResourceType = new RouteConfigurationResourceType(); + + private constructor() { + super(); } - protected getResourceName(resource: RouteConfiguration__Output): string { - return resource.name; + + static get() { + return RouteConfigurationResourceType.singleton; } - protected getProtocolName(): string { - return 'RDS'; + + getTypeUrl(): string { + return 'envoy.config.route.v3.RouteConfiguration'; } private validateRetryPolicy(policy: RetryPolicy__Output | null): boolean { @@ -74,7 +81,7 @@ export class RdsState extends BaseXdsStreamState imp return true; } - validateResponse(message: RouteConfiguration__Output): boolean { + public validateResource(message: RouteConfiguration__Output): RouteConfiguration__Output | null { // https://github.com/grpc/proposal/blob/master/A28-xds-traffic-splitting-and-routing.md#response-validation for (const virtualHost of message.virtual_hosts) { for (const domainPattern of virtualHost.domains) { @@ -82,54 +89,54 @@ export class RdsState extends BaseXdsStreamState imp const lastStarIndex = domainPattern.lastIndexOf('*'); // A domain pattern can have at most one wildcard * if (starIndex !== lastStarIndex) { - return false; + return null; } // A wildcard * can either be absent or at the beginning or end of the pattern if (!(starIndex === -1 || starIndex === 0 || starIndex === domainPattern.length - 1)) { - return false; + return null; } } if (EXPERIMENTAL_FAULT_INJECTION) { for (const filterConfig of Object.values(virtualHost.typed_per_filter_config ?? {})) { if (!validateOverrideFilter(filterConfig)) { - return false; + return null; } } } if (EXPERIMENTAL_RETRY) { if (!this.validateRetryPolicy(virtualHost.retry_policy)) { - return false; + return null; } } for (const route of virtualHost.routes) { const match = route.match; if (!match) { - return false; + return null; } if (SUPPORTED_PATH_SPECIFIERS.indexOf(match.path_specifier) < 0) { - return false; + return null; } for (const headers of match.headers) { if (SUPPPORTED_HEADER_MATCH_SPECIFIERS.indexOf(headers.header_match_specifier) < 0) { - return false; + return null; } } if (route.action !== 'route') { - return false; + return null; } if ((route.route === undefined) || (route.route === null) || SUPPORTED_CLUSTER_SPECIFIERS.indexOf(route.route.cluster_specifier) < 0) { - return false; + return null; } if (EXPERIMENTAL_FAULT_INJECTION) { for (const [name, filterConfig] of Object.entries(route.typed_per_filter_config ?? {})) { if (!validateOverrideFilter(filterConfig)) { - return false; + return null; } } } if (EXPERIMENTAL_RETRY) { if (!this.validateRetryPolicy(route.route.retry_policy)) { - return false; + return null; } } if (route.route!.cluster_specifier === 'weighted_clusters') { @@ -138,13 +145,13 @@ export class RdsState extends BaseXdsStreamState imp weightSum += clusterWeight.weight?.value ?? 0; } if (weightSum === 0 || weightSum > UINT32_MAX) { - return false; + return null; } if (EXPERIMENTAL_FAULT_INJECTION) { for (const weightedCluster of route.route!.weighted_clusters!.clusters) { for (const filterConfig of Object.values(weightedCluster.typed_per_filter_config ?? {})) { if (!validateOverrideFilter(filterConfig)) { - return false; + return null; } } } @@ -152,6 +159,39 @@ export class RdsState extends BaseXdsStreamState imp } } } - return true; + return message; + } + + decode(context: XdsDecodeContext, resource: Any__Output): XdsDecodeResult { + if (resource.type_url !== RDS_TYPE_URL) { + throw new Error( + `ADS Error: Invalid resource type ${resource.type_url}, expected ${RDS_TYPE_URL}` + ); + } + const message = decodeSingleResource(RDS_TYPE_URL, resource.value); + const validatedMessage = this.validateResource(message); + if (validatedMessage) { + return { + name: validatedMessage.name, + value: validatedMessage + }; + } else { + return { + name: message.name, + error: 'Route configuration message validation failed' + }; + } } -} \ No newline at end of file + + allResourcesRequiredInSotW(): boolean { + return false; + } + + static startWatch(client: XdsClient, name: string, watcher: Watcher) { + client.watchResource(RouteConfigurationResourceType.get(), name, watcher); + } + + static cancelWatch(client: XdsClient, name: string, watcher: Watcher) { + client.cancelResourceWatch(RouteConfigurationResourceType.get(), name, watcher); + } +} diff --git a/packages/grpc-js-xds/src/xds-resource-type/xds-resource-type.ts b/packages/grpc-js-xds/src/xds-resource-type/xds-resource-type.ts new file mode 100644 index 000000000..8c2dc5e4a --- /dev/null +++ b/packages/grpc-js-xds/src/xds-resource-type/xds-resource-type.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Any__Output } from "../generated/google/protobuf/Any"; +import { XdsServerConfig } from "../xds-bootstrap"; + +export interface XdsDecodeContext { + server: XdsServerConfig; +} + +export interface XdsDecodeResult { + name: string; + /** + * Mutually exclusive with error. + */ + value?: object; + /** + * Mutually exclusive with value. + */ + error?: string; +} + +type ValueType = string | number | bigint | boolean | undefined | null | symbol | {[key: string]: ValueType} | ValueType[]; + +function deepEqual(value1: ValueType, value2: ValueType): boolean { + if (value1 === value2) { + return true; + } + // Extra null check to narrow type result of typeof value === 'object' + if (value1 === null || value2 === null) { + // They are not equal per previous check + return false; + } + if (Array.isArray(value1) && Array.isArray(value2)) { + if (value1.length !== value2.length) { + return false; + } + for (const [index, entry] of value1.entries()) { + if (!deepEqual(entry, value2[index])) { + return false; + } + } + return true; + } else if (Array.isArray(value1) || Array.isArray(value2)) { + return false; + } else if (typeof value1 === 'object' && typeof value2 === 'object') { + for (const [key, entry] of Object.entries(value1)) { + if (!deepEqual(entry, value2[key])) { + return false; + } + } + return true; + } + return false; +} + +export abstract class XdsResourceType { + /** + * The type URL as used in xdstp: names + */ + abstract getTypeUrl(): string; + + /** + * The type URL as used in the `DiscoveryResponse.type_url` field and the `Any.type_url` field + */ + getFullTypeUrl(): string { + return `type.googleapis.com/${this.getTypeUrl()}`; + } + + abstract decode(context: XdsDecodeContext, resource: Any__Output): XdsDecodeResult; + + abstract allResourcesRequiredInSotW(): boolean; + + resourcesEqual(value1: object | null, value2: object | null): boolean { + return deepEqual(value1 as ValueType, value2 as ValueType); + } +} diff --git a/packages/grpc-js-xds/src/xds-stream-state/cds-state.ts b/packages/grpc-js-xds/src/xds-stream-state/cds-state.ts deleted file mode 100644 index 656a25418..000000000 --- a/packages/grpc-js-xds/src/xds-stream-state/cds-state.ts +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import { EXPERIMENTAL_OUTLIER_DETECTION } from "../environment"; -import { Cluster__Output } from "../generated/envoy/config/cluster/v3/Cluster"; -import { Duration__Output } from "../generated/google/protobuf/Duration"; -import { UInt32Value__Output } from "../generated/google/protobuf/UInt32Value"; -import { CLUSTER_CONFIG_TYPE_URL, decodeSingleResource } from "../resources"; -import { BaseXdsStreamState, XdsStreamState } from "./xds-stream-state"; - -export class CdsState extends BaseXdsStreamState implements XdsStreamState { - protected isStateOfTheWorld(): boolean { - return true; - } - protected getResourceName(resource: Cluster__Output): string { - return resource.name; - } - protected getProtocolName(): string { - return 'CDS'; - } - - private validateNonnegativeDuration(duration: Duration__Output | null): boolean { - if (!duration) { - return true; - } - /* The maximum values here come from the official Protobuf documentation: - * https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Duration - */ - return Number(duration.seconds) >= 0 && - Number(duration.seconds) <= 315_576_000_000 && - duration.nanos >= 0 && - duration.nanos <= 999_999_999; - } - - private validatePercentage(percentage: UInt32Value__Output | null): boolean { - if (!percentage) { - return true; - } - return percentage.value >=0 && percentage.value <= 100; - } - - public validateResponse(message: Cluster__Output): boolean { - if (message.cluster_discovery_type === 'cluster_type') { - if (!(message.cluster_type?.typed_config && message.cluster_type.typed_config.type_url === CLUSTER_CONFIG_TYPE_URL)) { - return false; - } - const clusterConfig = decodeSingleResource(CLUSTER_CONFIG_TYPE_URL, message.cluster_type.typed_config.value); - if (clusterConfig.clusters.length === 0) { - return false; - } - } else { - if (message.type === 'EDS') { - if (!message.eds_cluster_config?.eds_config?.ads) { - return false; - } - } else if (message.type === 'LOGICAL_DNS') { - if (!message.load_assignment) { - return false; - } - if (message.load_assignment.endpoints.length !== 1) { - return false; - } - if (message.load_assignment.endpoints[0].lb_endpoints.length !== 1) { - return false; - } - const socketAddress = message.load_assignment.endpoints[0].lb_endpoints[0].endpoint?.address?.socket_address; - if (!socketAddress) { - return false; - } - if (socketAddress.address === '') { - return false; - } - if (socketAddress.port_specifier !== 'port_value') { - return false; - } - } - } - if (message.lb_policy !== 'ROUND_ROBIN') { - return false; - } - if (message.lrs_server) { - if (!message.lrs_server.self) { - return false; - } - } - if (EXPERIMENTAL_OUTLIER_DETECTION) { - if (message.outlier_detection) { - if (!this.validateNonnegativeDuration(message.outlier_detection.interval)) { - return false; - } - if (!this.validateNonnegativeDuration(message.outlier_detection.base_ejection_time)) { - return false; - } - if (!this.validateNonnegativeDuration(message.outlier_detection.max_ejection_time)) { - return false; - } - if (!this.validatePercentage(message.outlier_detection.max_ejection_percent)) { - return false; - } - if (!this.validatePercentage(message.outlier_detection.enforcing_success_rate)) { - return false; - } - if (!this.validatePercentage(message.outlier_detection.failure_percentage_threshold)) { - return false; - } - if (!this.validatePercentage(message.outlier_detection.enforcing_failure_percentage)) { - return false; - } - } - } - return true; - } -} \ No newline at end of file diff --git a/packages/grpc-js-xds/src/xds-stream-state/lds-state.ts b/packages/grpc-js-xds/src/xds-stream-state/lds-state.ts deleted file mode 100644 index c215076db..000000000 --- a/packages/grpc-js-xds/src/xds-stream-state/lds-state.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import { experimental, logVerbosity } from "@grpc/grpc-js"; -import { Listener__Output } from '../generated/envoy/config/listener/v3/Listener'; -import { RdsState } from "./rds-state"; -import { BaseXdsStreamState, XdsStreamState } from "./xds-stream-state"; -import { decodeSingleResource, HTTP_CONNECTION_MANGER_TYPE_URL } from '../resources'; -import { getTopLevelFilterUrl, validateTopLevelFilter } from '../http-filter'; -import { EXPERIMENTAL_FAULT_INJECTION } from '../environment'; - -const TRACER_NAME = 'xds_client'; - -function trace(text: string): void { - experimental.trace(logVerbosity.DEBUG, TRACER_NAME, text); -} - -const ROUTER_FILTER_URL = 'type.googleapis.com/envoy.extensions.filters.http.router.v3.Router'; - -export class LdsState extends BaseXdsStreamState implements XdsStreamState { - protected getResourceName(resource: Listener__Output): string { - return resource.name; - } - protected getProtocolName(): string { - return 'LDS'; - } - protected isStateOfTheWorld(): boolean { - return true; - } - - constructor(private rdsState: RdsState, updateResourceNames: () => void) { - super(updateResourceNames); - } - - public validateResponse(message: Listener__Output): boolean { - if ( - !( - message.api_listener?.api_listener && - message.api_listener.api_listener.type_url === HTTP_CONNECTION_MANGER_TYPE_URL - ) - ) { - return false; - } - const httpConnectionManager = decodeSingleResource(HTTP_CONNECTION_MANGER_TYPE_URL, message.api_listener!.api_listener.value); - if (EXPERIMENTAL_FAULT_INJECTION) { - const filterNames = new Set(); - for (const [index, httpFilter] of httpConnectionManager.http_filters.entries()) { - if (filterNames.has(httpFilter.name)) { - trace('LDS response validation failed: duplicate HTTP filter name ' + httpFilter.name); - return false; - } - filterNames.add(httpFilter.name); - if (!validateTopLevelFilter(httpFilter)) { - trace('LDS response validation failed: ' + httpFilter.name + ' filter validation failed'); - return false; - } - /* Validate that the last filter, and only the last filter, is the - * router filter. */ - const filterUrl = getTopLevelFilterUrl(httpFilter.typed_config!) - if (index < httpConnectionManager.http_filters.length - 1) { - if (filterUrl === ROUTER_FILTER_URL) { - trace('LDS response validation failed: router filter is before end of list'); - return false; - } - } else { - if (filterUrl !== ROUTER_FILTER_URL) { - trace('LDS response validation failed: final filter is ' + filterUrl); - return false; - } - } - } - } - switch (httpConnectionManager.route_specifier) { - case 'rds': - return !!httpConnectionManager.rds?.config_source?.ads; - case 'route_config': - return this.rdsState.validateResponse(httpConnectionManager.route_config!); - } - return false; - } -} \ No newline at end of file diff --git a/packages/grpc-js-xds/src/xds-stream-state/xds-stream-state.ts b/packages/grpc-js-xds/src/xds-stream-state/xds-stream-state.ts deleted file mode 100644 index 86b81f01c..000000000 --- a/packages/grpc-js-xds/src/xds-stream-state/xds-stream-state.ts +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import { experimental, logVerbosity, Metadata, status, StatusObject } from "@grpc/grpc-js"; -import { Any__Output } from "../generated/google/protobuf/Any"; - -const TRACER_NAME = 'xds_client'; - -export interface Watcher { - /* Including the isV2 flag here is a bit of a kludge. It would probably be - * better for XdsStreamState#handleResponses to transform the protobuf - * message type into a library-specific configuration object type, to - * remove a lot of duplicate logic, including logic for handling that - * flag. */ - onValidUpdate(update: UpdateType): void; - onTransientError(error: StatusObject): void; - onResourceDoesNotExist(): void; -} - -export interface ResourcePair { - resource: ResourceType; - raw: Any__Output; -} - -export interface AcceptedResourceEntry { - name: string; - raw: Any__Output; -} - -export interface RejectedResourceEntry { - name: string; - raw: Any__Output; - error: string; -} - -export interface HandleResponseResult { - accepted: AcceptedResourceEntry[]; - rejected: RejectedResourceEntry[]; - missing: string[]; -} - -export interface XdsStreamState { - versionInfo: string; - nonce: string; - getResourceNames(): string[]; - /** - * Returns a string containing the error details if the message should be nacked, - * or null if it should be acked. - * @param responses - */ - handleResponses(responses: ResourcePair[], isV2: boolean): HandleResponseResult; - - reportStreamError(status: StatusObject): void; - reportAdsStreamStart(): void; - - addWatcher(name: string, watcher: Watcher): void; - removeWatcher(resourceName: string, watcher: Watcher): void; -} - -interface SubscriptionEntry { - watchers: Watcher[]; - cachedResponse: ResponseType | null; - resourceTimer: NodeJS.Timer; - deletionIgnored: boolean; -} - -const RESOURCE_TIMEOUT_MS = 15_000; - -export abstract class BaseXdsStreamState implements XdsStreamState { - versionInfo = ''; - nonce = ''; - - private subscriptions: Map> = new Map>(); - private isAdsStreamRunning = false; - private ignoreResourceDeletion = false; - - constructor(private updateResourceNames: () => void) {} - - protected trace(text: string) { - experimental.trace(logVerbosity.DEBUG, TRACER_NAME, this.getProtocolName() + ' | ' + text); - } - - private startResourceTimer(subscriptionEntry: SubscriptionEntry) { - clearTimeout(subscriptionEntry.resourceTimer); - subscriptionEntry.resourceTimer = setTimeout(() => { - for (const watcher of subscriptionEntry.watchers) { - watcher.onResourceDoesNotExist(); - } - }, RESOURCE_TIMEOUT_MS); - } - - addWatcher(name: string, watcher: Watcher): void { - this.trace('Adding watcher for name ' + name); - let subscriptionEntry = this.subscriptions.get(name); - let addedName = false; - if (subscriptionEntry === undefined) { - addedName = true; - subscriptionEntry = { - watchers: [], - cachedResponse: null, - resourceTimer: setTimeout(() => {}, 0), - deletionIgnored: false - }; - if (this.isAdsStreamRunning) { - this.startResourceTimer(subscriptionEntry); - } - this.subscriptions.set(name, subscriptionEntry); - } - subscriptionEntry.watchers.push(watcher); - if (subscriptionEntry.cachedResponse !== null) { - const cachedResponse = subscriptionEntry.cachedResponse; - /* These updates normally occur asynchronously, so we ensure that - * the same happens here */ - process.nextTick(() => { - this.trace('Reporting existing update for new watcher for name ' + name); - watcher.onValidUpdate(cachedResponse); - }); - } - if (addedName) { - this.updateResourceNames(); - } - } - removeWatcher(resourceName: string, watcher: Watcher): void { - this.trace('Removing watcher for name ' + resourceName); - const subscriptionEntry = this.subscriptions.get(resourceName); - if (subscriptionEntry !== undefined) { - const entryIndex = subscriptionEntry.watchers.indexOf(watcher); - if (entryIndex >= 0) { - subscriptionEntry.watchers.splice(entryIndex, 1); - } - if (subscriptionEntry.watchers.length === 0) { - clearTimeout(subscriptionEntry.resourceTimer); - if (subscriptionEntry.deletionIgnored) { - experimental.log(logVerbosity.INFO, 'Unsubscribing from resource with previously ignored deletion: ' + resourceName); - } - this.subscriptions.delete(resourceName); - this.updateResourceNames(); - } - } - } - - getResourceNames(): string[] { - return Array.from(this.subscriptions.keys()); - } - handleResponses(responses: ResourcePair[]): HandleResponseResult { - let result: HandleResponseResult = { - accepted: [], - rejected: [], - missing: [] - } - const allResourceNames = new Set(); - for (const {resource, raw} of responses) { - const resourceName = this.getResourceName(resource); - allResourceNames.add(resourceName); - const subscriptionEntry = this.subscriptions.get(resourceName); - if (this.validateResponse(resource)) { - result.accepted.push({ - name: resourceName, - raw: raw}); - if (subscriptionEntry) { - for (const watcher of subscriptionEntry.watchers) { - /* Use process.nextTick to prevent errors from the watcher from - * bubbling up through here. */ - process.nextTick(() => { - watcher.onValidUpdate(resource); - }); - } - clearTimeout(subscriptionEntry.resourceTimer); - subscriptionEntry.cachedResponse = resource; - if (subscriptionEntry.deletionIgnored) { - experimental.log(logVerbosity.INFO, `Received resource with previously ignored deletion: ${resourceName}`); - subscriptionEntry.deletionIgnored = false; - } - } - } else { - this.trace('Validation failed for message ' + JSON.stringify(resource)); - result.rejected.push({ - name: resourceName, - raw: raw, - error: `Validation failed for resource ${resourceName}` - }); - if (subscriptionEntry) { - for (const watcher of subscriptionEntry.watchers) { - /* Use process.nextTick to prevent errors from the watcher from - * bubbling up through here. */ - process.nextTick(() => { - watcher.onTransientError({ - code: status.UNAVAILABLE, - details: `Validation failed for resource ${resourceName}`, - metadata: new Metadata() - }); - }); - } - clearTimeout(subscriptionEntry.resourceTimer); - } - } - } - result.missing = this.handleMissingNames(allResourceNames); - this.trace('Received response with resource names [' + Array.from(allResourceNames) + ']'); - return result; - } - reportStreamError(status: StatusObject): void { - for (const subscriptionEntry of this.subscriptions.values()) { - for (const watcher of subscriptionEntry.watchers) { - watcher.onTransientError(status); - } - clearTimeout(subscriptionEntry.resourceTimer); - } - this.isAdsStreamRunning = false; - this.nonce = ''; - } - - reportAdsStreamStart() { - if (this.isAdsStreamRunning) { - return; - } - this.isAdsStreamRunning = true; - for (const subscriptionEntry of this.subscriptions.values()) { - if (subscriptionEntry.cachedResponse === null) { - this.startResourceTimer(subscriptionEntry); - } - } - } - - private handleMissingNames(allResponseNames: Set): string[] { - if (this.isStateOfTheWorld()) { - const missingNames: string[] = []; - for (const [resourceName, subscriptionEntry] of this.subscriptions.entries()) { - if (!allResponseNames.has(resourceName) && subscriptionEntry.cachedResponse !== null) { - if (this.ignoreResourceDeletion) { - if (!subscriptionEntry.deletionIgnored) { - experimental.log(logVerbosity.ERROR, 'Ignoring nonexistent resource ' + resourceName); - subscriptionEntry.deletionIgnored = true; - } - } else { - this.trace('Reporting resource does not exist named ' + resourceName); - missingNames.push(resourceName); - for (const watcher of subscriptionEntry.watchers) { - /* Use process.nextTick to prevent errors from the watcher from - * bubbling up through here. */ - process.nextTick(() => { - watcher.onResourceDoesNotExist(); - }); - } - subscriptionEntry.cachedResponse = null; - } - } - } - return missingNames; - } else { - return []; - } - } - - enableIgnoreResourceDeletion() { - this.ignoreResourceDeletion = true; - } - - /** - * Apply the validation rules for this resource type to this resource - * instance. - * This function is public so that the LDS validateResponse can call into - * the RDS validateResponse. - * @param resource The resource object sent by the xDS server - */ - public abstract validateResponse(resource: ResponseType): boolean; - /** - * Get the name of a resource object. The name is some field of the object, so - * getting it depends on the specific type. - * @param resource - */ - protected abstract getResourceName(resource: ResponseType): string; - protected abstract getProtocolName(): string; - /** - * Indicates whether responses are "state of the world", i.e. that they - * contain all resources and that omitted previously-seen resources should - * be treated as removed. - */ - protected abstract isStateOfTheWorld(): boolean; -} \ No newline at end of file diff --git a/packages/grpc-js-xds/test/client.ts b/packages/grpc-js-xds/test/client.ts index bcc6f3cd8..6d346f918 100644 --- a/packages/grpc-js-xds/test/client.ts +++ b/packages/grpc-js-xds/test/client.ts @@ -44,12 +44,16 @@ export class XdsTestClient { private client: EchoTestServiceClient; private callInterval: NodeJS.Timer; - constructor(targetName: string, xdsServer: XdsServer) { - this.client = new loadedProtos.grpc.testing.EchoTestService(`xds:///${targetName}`, credentials.createInsecure(), {[BOOTSTRAP_CONFIG_KEY]: xdsServer.getBootstrapInfoString()}); + constructor(target: string, bootstrapInfo: string) { + this.client = new loadedProtos.grpc.testing.EchoTestService(target, credentials.createInsecure(), {[BOOTSTRAP_CONFIG_KEY]: bootstrapInfo}); this.callInterval = setInterval(() => {}, 0); clearInterval(this.callInterval); } + static createFromServer(targetName: string, xdsServer: XdsServer) { + return new XdsTestClient(`xds:///${targetName}`, xdsServer.getBootstrapInfoString()); + } + startCalls(interval: number) { clearInterval(this.callInterval); this.callInterval = setInterval(() => { @@ -94,4 +98,4 @@ export class XdsTestClient { } sendInner(count, callback); } -} \ No newline at end of file +} diff --git a/packages/grpc-js-xds/test/framework.ts b/packages/grpc-js-xds/test/framework.ts index bcc3cd1a5..4bb9fa142 100644 --- a/packages/grpc-js-xds/test/framework.ts +++ b/packages/grpc-js-xds/test/framework.ts @@ -64,24 +64,25 @@ export interface FakeCluster { getAllClusterConfigs(): Cluster[]; getName(): string; startAllBackends(): Promise; + haveAllBackendsReceivedTraffic(): boolean; waitForAllBackendsToReceiveTraffic(): Promise; } export class FakeEdsCluster implements FakeCluster { - constructor(private name: string, private endpoints: Endpoint[]) {} + constructor(private clusterName: string, private endpointName: string, private endpoints: Endpoint[]) {} getEndpointConfig(): ClusterLoadAssignment { return { - cluster_name: this.name, + cluster_name: this.endpointName, endpoints: this.endpoints.map(getLocalityLbEndpoints) }; } getClusterConfig(): Cluster { return { - name: this.name, + name: this.clusterName, type: 'EDS', - eds_cluster_config: {eds_config: {ads: {}}}, + eds_cluster_config: {eds_config: {ads: {}}, service_name: this.endpointName}, lb_policy: 'ROUND_ROBIN' } } @@ -91,14 +92,14 @@ export class FakeEdsCluster implements FakeCluster { } getName() { - return this.name; + return this.clusterName; } startAllBackends(): Promise { return Promise.all(this.endpoints.map(endpoint => Promise.all(endpoint.backends.map(backend => backend.startAsync())))); } - private haveAllBackendsReceivedTraffic(): boolean { + haveAllBackendsReceivedTraffic(): boolean { for (const endpoint of this.endpoints) { for (const backend of endpoint.backends) { if (backend.getCallCount() < 1) { @@ -167,6 +168,9 @@ export class FakeDnsCluster implements FakeCluster { startAllBackends(): Promise { return this.backend.startAsync(); } + haveAllBackendsReceivedTraffic(): boolean { + return this.backend.getCallCount() > 0; + } waitForAllBackendsToReceiveTraffic(): Promise { return new Promise((resolve, reject) => { this.backend.onCall(resolve); @@ -203,6 +207,14 @@ export class FakeAggregateCluster implements FakeCluster { startAllBackends(): Promise { return Promise.all(this.children.map(child => child.startAllBackends())); } + haveAllBackendsReceivedTraffic(): boolean { + for (const child of this.children) { + if (!child.haveAllBackendsReceivedTraffic()) { + return false; + } + } + return true; + } waitForAllBackendsToReceiveTraffic(): Promise { return Promise.all(this.children.map(child => child.waitForAllBackendsToReceiveTraffic())).then(() => {}); } @@ -241,11 +253,11 @@ function createRouteConfig(route: FakeRoute): Route { } export class FakeRouteGroup { - constructor(private name: string, private routes: FakeRoute[]) {} + constructor(private listenerName: string, private routeName: string, private routes: FakeRoute[]) {} getRouteConfiguration(): RouteConfiguration { return { - name: this.name, + name: this.routeName, virtual_hosts: [{ domains: ['*'], routes: this.routes.map(createRouteConfig) @@ -257,12 +269,12 @@ export class FakeRouteGroup { const httpConnectionManager: HttpConnectionManager & AnyExtension = { '@type': HTTP_CONNECTION_MANGER_TYPE_URL, rds: { - route_config_name: this.name, + route_config_name: this.routeName, config_source: {ads: {}} } } return { - name: this.name, + name: this.listenerName, api_listener: { api_listener: httpConnectionManager } @@ -281,6 +293,21 @@ export class FakeRouteGroup { })); } + haveAllBackendsReceivedTraffic(): boolean { + for (const route of this.routes) { + if (route.cluster) { + return route.cluster.haveAllBackendsReceivedTraffic(); + } else if (route.weightedClusters) { + for (const weightedCluster of route.weightedClusters) { + if (!weightedCluster.cluster.haveAllBackendsReceivedTraffic()) { + return false; + } + } + } + } + return true; + } + waitForAllBackendsToReceiveTraffic(): Promise { return Promise.all(this.routes.map(route => { if (route.cluster) { @@ -292,4 +319,4 @@ export class FakeRouteGroup { } })); } -} \ No newline at end of file +} diff --git a/packages/grpc-js-xds/test/test-cluster-type.ts b/packages/grpc-js-xds/test/test-cluster-type.ts index 483180d9c..416f17727 100644 --- a/packages/grpc-js-xds/test/test-cluster-type.ts +++ b/packages/grpc-js-xds/test/test-cluster-type.ts @@ -40,7 +40,7 @@ describe('Cluster types', () => { describe('Logical DNS Clusters', () => { it('Should successfully make RPCs', done => { const cluster = new FakeDnsCluster('dnsCluster', new Backend()); - const routeGroup = new FakeRouteGroup('route1', [{cluster: cluster}]); + const routeGroup = new FakeRouteGroup('listener1', 'route1', [{cluster: cluster}]); routeGroup.startAllBackends().then(() => { xdsServer.addResponseListener((typeUrl, responseState) => { if (responseState.state === 'NACKED') { @@ -50,7 +50,7 @@ describe('Cluster types', () => { xdsServer.setCdsResource(cluster.getClusterConfig()); xdsServer.setRdsResource(routeGroup.getRouteConfiguration()); xdsServer.setLdsResource(routeGroup.getListener()); - client = new XdsTestClient('route1', xdsServer); + client = XdsTestClient.createFromServer('listener1', xdsServer); client.sendOneCall(error => { done(error); }); @@ -63,10 +63,10 @@ describe('Cluster types', () => { it('Should result in prioritized clusters', () => { const backend1 = new Backend(); const backend2 = new Backend(); - const cluster1 = new FakeEdsCluster('cluster1', [{backends: [backend1], locality:{region: 'region1'}}]); - const cluster2 = new FakeEdsCluster('cluster2', [{backends: [backend2], locality:{region: 'region2'}}]); + const cluster1 = new FakeEdsCluster('cluster1', 'endpoint1', [{backends: [backend1], locality:{region: 'region1'}}]); + const cluster2 = new FakeEdsCluster('cluster2', 'endpoint2', [{backends: [backend2], locality:{region: 'region2'}}]); const aggregateCluster = new FakeAggregateCluster('aggregateCluster', [cluster1, cluster2]); - const routeGroup = new FakeRouteGroup('route1', [{cluster: aggregateCluster}]); + const routeGroup = new FakeRouteGroup('listener1', 'route1', [{cluster: aggregateCluster}]); return routeGroup.startAllBackends().then(() => { xdsServer.setEdsResource(cluster1.getEndpointConfig()); xdsServer.setCdsResource(cluster1.getClusterConfig()); @@ -81,7 +81,7 @@ describe('Cluster types', () => { assert.fail(`Client NACKED ${typeUrl} resource with message ${responseState.errorMessage}`); } }); - client = new XdsTestClient('route1', xdsServer); + client = XdsTestClient.createFromServer('listener1', xdsServer); client.startCalls(100); return cluster1.waitForAllBackendsToReceiveTraffic(); }).then(() => backend1.shutdownAsync() @@ -92,11 +92,11 @@ describe('Cluster types', () => { it('Should handle a diamond dependency', () => { const backend1 = new Backend(); const backend2 = new Backend(); - const cluster1 = new FakeEdsCluster('cluster1', [{backends: [backend1], locality:{region: 'region1'}}]); - const cluster2 = new FakeEdsCluster('cluster2', [{backends: [backend2], locality:{region: 'region2'}}]); + const cluster1 = new FakeEdsCluster('cluster1', 'endpoint1', [{backends: [backend1], locality:{region: 'region1'}}]); + const cluster2 = new FakeEdsCluster('cluster2', 'endpoint2', [{backends: [backend2], locality:{region: 'region2'}}]); const aggregateCluster1 = new FakeAggregateCluster('aggregateCluster1', [cluster1, cluster2]); const aggregateCluster2 = new FakeAggregateCluster('aggregateCluster2', [cluster1, aggregateCluster1]); - const routeGroup = new FakeRouteGroup('route1', [{cluster: aggregateCluster2}]); + const routeGroup = new FakeRouteGroup('listener1', 'route1', [{cluster: aggregateCluster2}]); return Promise.all([backend1.startAsync(), backend2.startAsync()]).then(() => { xdsServer.setEdsResource(cluster1.getEndpointConfig()); xdsServer.setCdsResource(cluster1.getClusterConfig()); @@ -112,7 +112,7 @@ describe('Cluster types', () => { assert.fail(`Client NACKED ${typeUrl} resource with message ${responseState.errorMessage}`); } }); - client = new XdsTestClient('route1', xdsServer); + client = XdsTestClient.createFromServer('listener1', xdsServer); client.startCalls(100); return cluster1.waitForAllBackendsToReceiveTraffic(); }).then(() => backend1.shutdownAsync() @@ -123,10 +123,10 @@ describe('Cluster types', () => { it('Should handle EDS then DNS cluster order', () => { const backend1 = new Backend(); const backend2 = new Backend(); - const cluster1 = new FakeEdsCluster('cluster1', [{backends: [backend1], locality:{region: 'region1'}}]); + const cluster1 = new FakeEdsCluster('cluster1', 'endpoint1', [{backends: [backend1], locality:{region: 'region1'}}]); const cluster2 = new FakeDnsCluster('cluster2', backend2); const aggregateCluster = new FakeAggregateCluster('aggregateCluster', [cluster1, cluster2]); - const routeGroup = new FakeRouteGroup('route1', [{cluster: aggregateCluster}]); + const routeGroup = new FakeRouteGroup('listener1', 'route1', [{cluster: aggregateCluster}]); return routeGroup.startAllBackends().then(() => { xdsServer.setEdsResource(cluster1.getEndpointConfig()); xdsServer.setCdsResource(cluster1.getClusterConfig()); @@ -140,7 +140,7 @@ describe('Cluster types', () => { assert.fail(`Client NACKED ${typeUrl} resource with message ${responseState.errorMessage}`); } }); - client = new XdsTestClient('route1', xdsServer); + client = XdsTestClient.createFromServer('listener1', xdsServer); client.startCalls(100); return cluster1.waitForAllBackendsToReceiveTraffic(); }).then(() => backend1.shutdownAsync() @@ -152,9 +152,9 @@ describe('Cluster types', () => { const backend1 = new Backend(); const backend2 = new Backend(); const cluster1 = new FakeDnsCluster('cluster1', backend1); - const cluster2 = new FakeEdsCluster('cluster2', [{backends: [backend2], locality:{region: 'region2'}}]); + const cluster2 = new FakeEdsCluster('cluster2', 'endpoint2', [{backends: [backend2], locality:{region: 'region2'}}]); const aggregateCluster = new FakeAggregateCluster('aggregateCluster', [cluster1, cluster2]); - const routeGroup = new FakeRouteGroup('route1', [{cluster: aggregateCluster}]); + const routeGroup = new FakeRouteGroup('listener1', 'route1', [{cluster: aggregateCluster}]); return routeGroup.startAllBackends().then(() => { xdsServer.setCdsResource(cluster1.getClusterConfig()); xdsServer.setEdsResource(cluster2.getEndpointConfig()); @@ -168,7 +168,7 @@ describe('Cluster types', () => { assert.fail(`Client NACKED ${typeUrl} resource with message ${responseState.errorMessage}`); } }); - client = new XdsTestClient('route1', xdsServer); + client = XdsTestClient.createFromServer('listener1', xdsServer); client.startCalls(100); return cluster1.waitForAllBackendsToReceiveTraffic(); }).then(() => backend1.shutdownAsync() diff --git a/packages/grpc-js-xds/test/test-core.ts b/packages/grpc-js-xds/test/test-core.ts index 7c28f0920..cb145eb81 100644 --- a/packages/grpc-js-xds/test/test-core.ts +++ b/packages/grpc-js-xds/test/test-core.ts @@ -39,8 +39,8 @@ describe('core xDS functionality', () => { xdsServer?.shutdownServer(); }) it('should route requests to the single backend', done => { - const cluster = new FakeEdsCluster('cluster1', [{backends: [new Backend()], locality:{region: 'region1'}}]); - const routeGroup = new FakeRouteGroup('route1', [{cluster: cluster}]); + const cluster = new FakeEdsCluster('cluster1', 'endpoint1', [{backends: [new Backend()], locality:{region: 'region1'}}]); + const routeGroup = new FakeRouteGroup('listener1', 'route1', [{cluster: cluster}]); routeGroup.startAllBackends().then(() => { xdsServer.setEdsResource(cluster.getEndpointConfig()); xdsServer.setCdsResource(cluster.getClusterConfig()); @@ -52,7 +52,7 @@ describe('core xDS functionality', () => { assert.fail(`Client NACKED ${typeUrl} resource with message ${responseState.errorMessage}`); } }) - client = new XdsTestClient('route1', xdsServer); + client = XdsTestClient.createFromServer('listener1', xdsServer); client.startCalls(100); routeGroup.waitForAllBackendsToReceiveTraffic().then(() => { client.stopCalls(); @@ -60,4 +60,4 @@ describe('core xDS functionality', () => { }, reason => done(reason)); }, reason => done(reason)); }); -}); \ No newline at end of file +}); diff --git a/packages/grpc-js-xds/test/test-federation.ts b/packages/grpc-js-xds/test/test-federation.ts new file mode 100644 index 000000000..5d4099bbf --- /dev/null +++ b/packages/grpc-js-xds/test/test-federation.ts @@ -0,0 +1,209 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Backend } from "./backend"; +import { XdsTestClient } from "./client"; +import { FakeEdsCluster, FakeRouteGroup } from "./framework"; +import { XdsServer } from "./xds-server"; +import assert = require("assert"); + +/* Test cases in this file are derived from examples in the xDS federation proposal + * https://github.com/grpc/proposal/blob/master/A47-xds-federation.md */ +describe('Federation', () => { + let xdsServers: XdsServer[] = []; + let xdsClient: XdsTestClient; + afterEach(() => { + xdsClient?.close(); + for (const server of xdsServers) { + server.shutdownServer(); + } + xdsServers = []; + }); + describe('Bootstrap Config Contains No New Fields', () => { + let bootstrap: string; + beforeEach((done) => { + const xdsServer = new XdsServer(); + xdsServers.push(xdsServer); + xdsServer.startServer(error => { + if (error) { + done(error); + return; + } + const cluster = new FakeEdsCluster('cluster1', 'endpoint1', [{backends: [new Backend()], locality:{region: 'region1'}}]); + const routeGroup = new FakeRouteGroup('server.example.com', 'route1', [{cluster: cluster}]); + routeGroup.startAllBackends().then(() => { + xdsServer.setEdsResource(cluster.getEndpointConfig()); + xdsServer.setCdsResource(cluster.getClusterConfig()); + xdsServer.setRdsResource(routeGroup.getRouteConfiguration()); + xdsServer.setLdsResource(routeGroup.getListener()); + const bootstrapInfo = { + xds_servers: [xdsServer.getBootstrapServerConfig()], + node: { + id: 'test', + locality: {} + } + }; + bootstrap = JSON.stringify(bootstrapInfo); + done(); + }); + }); + }); + it('Should accept an old-style name', (done) => { + xdsClient = new XdsTestClient('xds:server.example.com', bootstrap); + // There is only one server, so a successful request must go to that server + xdsClient.sendOneCall(done); + }); + it('Should reject a new-style name', (done) => { + xdsClient = new XdsTestClient('xds://xds.authority.com/server.example.com', bootstrap); + xdsClient.sendOneCall(error => { + assert(error); + done(); + }); + }); + }); + describe('New-Style Names on gRPC Client', () => { + let bootstrap: string; + beforeEach((done) => { + const xdsServer = new XdsServer(); + xdsServers.push(xdsServer); + xdsServer.startServer(error => { + if (error) { + done(error); + return; + } + const cluster = new FakeEdsCluster('xdstp://xds.authority.com/envoy.config.cluster.v3.Cluster/cluster1', 'xdstp://xds.authority.com/envoy.config.endpoint.v3.ClusterLoadAssignment/endpoint1', [{backends: [new Backend()], locality:{region: 'region1'}}]); + const routeGroup = new FakeRouteGroup('xdstp://xds.authority.com/envoy.config.listener.v3.Listener/server.example.com', 'xdstp://xds.authority.com/envoy.config.route.v3.RouteConfiguration/route1', [{cluster: cluster}]); + routeGroup.startAllBackends().then(() => { + xdsServer.setEdsResource(cluster.getEndpointConfig()); + xdsServer.setCdsResource(cluster.getClusterConfig()); + xdsServer.setRdsResource(routeGroup.getRouteConfiguration()); + xdsServer.setLdsResource(routeGroup.getListener()); + const bootstrapInfo = { + xds_servers: [xdsServer.getBootstrapServerConfig()], + node: { + id: 'test', + locality: {} + }, + "client_default_listener_resource_name_template": "xdstp://xds.authority.com/envoy.config.listener.v3.Listener/%s", + "authorities": { + "xds.authority.com": { + } + } + }; + bootstrap = JSON.stringify(bootstrapInfo); + done(); + }); + }); + }); + it('Should accept a target with no authority', (done) => { + xdsClient = new XdsTestClient('xds:server.example.com', bootstrap); + // There is only one server, so a successful request must go to that server + xdsClient.sendOneCall(done); + }); + it('Should accept a target with a listed authority', (done) => { + xdsClient = new XdsTestClient('xds://xds.authority.com/server.example.com', bootstrap); + // There is only one server, so a successful request must go to that server + xdsClient.sendOneCall(done); + }); + }); + describe('Multiple authorities', () => { + let bootstrap: string; + let defaultRouteGroup: FakeRouteGroup; + let otherRouteGroup: FakeRouteGroup; + beforeEach((done) => { + const defaultServer = new XdsServer(); + xdsServers.push(defaultServer); + const otherServer = new XdsServer(); + xdsServers.push(otherServer); + defaultServer.startServer(error => { + if (error) { + done(error); + return; + } + otherServer.startServer(error => { + if (error) { + done(error); + return; + } + const defaultCluster = new FakeEdsCluster('xdstp://xds.authority.com/envoy.config.cluster.v3.Cluster/cluster1', 'xdstp://xds.authority.com/envoy.config.endpoint.v3.ClusterLoadAssignment/endpoint1', [{backends: [new Backend()], locality:{region: 'region1'}}]); + defaultRouteGroup = new FakeRouteGroup('xdstp://xds.authority.com/envoy.config.listener.v3.Listener/grpc/client/server.example.com?project_id=1234', 'xdstp://xds.authority.com/envoy.config.route.v3.RouteConfiguration/route1', [{cluster: defaultCluster}]); + const otherCluster = new FakeEdsCluster('xdstp://xds.other.com/envoy.config.cluster.v3.Cluster/cluster2', 'xdstp://xds.other.com/envoy.config.endpoint.v3.ClusterLoadAssignment/endpoint2', [{backends: [new Backend()], locality:{region: 'region2'}}]); + otherRouteGroup = new FakeRouteGroup('xdstp://xds.other.com/envoy.config.listener.v3.Listener/server.other.com', 'xdstp://xds.other.com/envoy.config.route.v3.RouteConfiguration/route2', [{cluster: otherCluster}]); + Promise.all([defaultRouteGroup.startAllBackends(), otherRouteGroup.startAllBackends()]).then(() => { + defaultServer.setEdsResource(defaultCluster.getEndpointConfig()); + defaultServer.setCdsResource(defaultCluster.getClusterConfig()); + defaultServer.setRdsResource(defaultRouteGroup.getRouteConfiguration()); + defaultServer.setLdsResource(defaultRouteGroup.getListener()); + otherServer.setEdsResource(otherCluster.getEndpointConfig()); + otherServer.setCdsResource(otherCluster.getClusterConfig()); + otherServer.setRdsResource(otherRouteGroup.getRouteConfiguration()); + otherServer.setLdsResource(otherRouteGroup.getListener()); + const bootstrapInfo = { + xds_servers: [defaultServer.getBootstrapServerConfig()], + node: { + id: 'test', + locality: {} + }, + + // Resource name template for xds: target URIs with no authority. + "client_default_listener_resource_name_template": "xdstp://xds.authority.com/envoy.config.listener.v3.Listener/grpc/client/%s?project_id=1234", + + // Resource name template for xDS-enabled gRPC servers. + "server_listener_resource_name_template": "xdstp://xds.authority.com/envoy.config.listener.v3.Listener/grpc/server/%s?project_id=1234", + + // Authorities map. + "authorities": { + "xds.authority.com": { + "client_listener_resource_name_template": "xdstp://xds.authority.com/envoy.config.listener.v3.Listener/grpc/client/%s?project_id=1234" + }, + "xds.other.com": { + "xds_servers": [otherServer.getBootstrapServerConfig()] + } + } + }; + bootstrap = JSON.stringify(bootstrapInfo); + done(); + }); + }); + }); + }); + it('Should accept a name with no authority', (done) => { + xdsClient = new XdsTestClient('xds:server.example.com', bootstrap); + xdsClient.sendOneCall(error => { + assert.ifError(error); + assert(defaultRouteGroup.haveAllBackendsReceivedTraffic()); + done(); + }); + }); + it('Should accept a with an authority that has no server configured', (done) => { + xdsClient = new XdsTestClient('xds://xds.authority.com/server.example.com', bootstrap); + xdsClient.sendOneCall(error => { + assert.ifError(error); + assert(defaultRouteGroup.haveAllBackendsReceivedTraffic()); + done(); + }); + }); + it('Should accept a name with an authority that has no template configured', (done) => { + xdsClient = new XdsTestClient('xds://xds.other.com/server.other.com', bootstrap); + xdsClient.sendOneCall(error => { + assert.ifError(error); + assert(otherRouteGroup.haveAllBackendsReceivedTraffic()); + done(); + }); + }); + }); +}); diff --git a/packages/grpc-js-xds/test/test-listener-resource-name.ts b/packages/grpc-js-xds/test/test-listener-resource-name.ts new file mode 100644 index 000000000..2aeb2d3d6 --- /dev/null +++ b/packages/grpc-js-xds/test/test-listener-resource-name.ts @@ -0,0 +1,125 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { BootstrapInfo, Node, validateBootstrapConfig } from "../src/xds-bootstrap"; +import { experimental } from "@grpc/grpc-js"; +import * as assert from 'assert'; +import GrpcUri = experimental.GrpcUri; +import { getListenerResourceName } from "../src/resolver-xds"; + +const testNode: Node = { + id: 'test', + locality: {} +}; + +/* Test cases in this file are derived from examples in the xDS federation proposal + * https://github.com/grpc/proposal/blob/master/A47-xds-federation.md */ +describe('Listener resource name evaluation', () => { + describe('No new bootstrap fields', () => { + const bootstrap = validateBootstrapConfig({ + node: testNode, + xds_servers: [] + }); + it('xds:server.example.com', () => { + const target: GrpcUri = { + scheme: 'xds', + path: 'server.example.com' + }; + assert.strictEqual(getListenerResourceName(bootstrap, target), 'server.example.com'); + }); + it('xds://xds.authority.com/server.example.com', () => { + const target: GrpcUri = { + scheme: 'xds', + authority: 'xds.authority.com', + path: 'server.example.com' + }; + assert.throws(() => getListenerResourceName(bootstrap, target), /xds.authority.com/); + }); + }); + describe('New-style names', () => { + const bootstrap = validateBootstrapConfig({ + node: testNode, + xds_servers: [], + client_default_listener_resource_name_template: 'xdstp://xds.authority.com/envoy.config.listener.v3.Listener/%s', + authorities: { + 'xds.authority.com': {} + } + }); + it('xds:server.example.com', () => { + const target: GrpcUri = { + scheme: 'xds', + path: 'server.example.com' + }; + assert.strictEqual(getListenerResourceName(bootstrap, target), 'xdstp://xds.authority.com/envoy.config.listener.v3.Listener/server.example.com'); + }); + it('xds://xds.authority.com/server.example.com', () => { + const target: GrpcUri = { + scheme: 'xds', + authority: 'xds.authority.com', + path: 'server.example.com' + }; + assert.strictEqual(getListenerResourceName(bootstrap, target), 'xdstp://xds.authority.com/envoy.config.listener.v3.Listener/server.example.com'); + }); + }); + describe('Multiple authorities', () => { + const bootstrap = validateBootstrapConfig({ + node: testNode, + xds_servers: [{ + "server_uri": "xds-server.authority.com", + "channel_creds": [ { "type": "google_default" } ] + }], + client_default_listener_resource_name_template: 'xdstp://xds.authority.com/envoy.config.listener.v3.Listener/grpc/client/%s?project_id=1234', + authorities: { + "xds.authority.com": { + "client_listener_resource_name_template": "xdstp://xds.authority.com/envoy.config.listener.v3.Listener/grpc/client/%s?project_id=1234" + }, + + "xds.other.com": { + "xds_servers": [ + { + "server_uri": "xds-server.other.com", + "channel_creds": [ { "type": "google_default" } ] + } + ] + } + } + }); + it('xds:server.example.com', () => { + const target: GrpcUri = { + scheme: 'xds', + path: 'server.example.com' + }; + assert.strictEqual(getListenerResourceName(bootstrap, target), 'xdstp://xds.authority.com/envoy.config.listener.v3.Listener/grpc/client/server.example.com?project_id=1234'); + }); + it('xds://xds.authority.com/server.example.com', () => { + const target: GrpcUri = { + scheme: 'xds', + authority: 'xds.authority.com', + path: 'server.example.com' + }; + assert.strictEqual(getListenerResourceName(bootstrap, target), 'xdstp://xds.authority.com/envoy.config.listener.v3.Listener/grpc/client/server.example.com?project_id=1234'); + }); + it('xds://xds.other.com/server.other.com', () => { + const target: GrpcUri = { + scheme: 'xds', + authority: 'xds.other.com', + path: 'server.other.com' + }; + assert.strictEqual(getListenerResourceName(bootstrap, target), 'xdstp://xds.other.com/envoy.config.listener.v3.Listener/server.other.com'); + }); + }); +}); diff --git a/packages/grpc-js-xds/test/test-nack.ts b/packages/grpc-js-xds/test/test-nack.ts index ae2a9b3e4..ce6b6f45b 100644 --- a/packages/grpc-js-xds/test/test-nack.ts +++ b/packages/grpc-js-xds/test/test-nack.ts @@ -39,14 +39,14 @@ describe('Validation errors', () => { xdsServer?.shutdownServer(); }); it('Should continue to use a valid resource after receiving an invalid EDS update', done => { - const cluster = new FakeEdsCluster('cluster1', [{backends: [new Backend()], locality: {region: 'region1'}}]); - const routeGroup = new FakeRouteGroup('route1', [{cluster: cluster}]); + const cluster = new FakeEdsCluster('cluster1', 'endpoint1', [{backends: [new Backend()], locality: {region: 'region1'}}]); + const routeGroup = new FakeRouteGroup('listener1', 'route1', [{cluster: cluster}]); routeGroup.startAllBackends().then(() => { xdsServer.setEdsResource(cluster.getEndpointConfig()); xdsServer.setCdsResource(cluster.getClusterConfig()); xdsServer.setRdsResource(routeGroup.getRouteConfiguration()); xdsServer.setLdsResource(routeGroup.getListener()); - client = new XdsTestClient('route1', xdsServer); + client = XdsTestClient.createFromServer('listener1', xdsServer); client.startCalls(100); routeGroup.waitForAllBackendsToReceiveTraffic().then(() => { // After backends receive calls, set invalid EDS resource @@ -69,14 +69,14 @@ describe('Validation errors', () => { }, reason => done(reason)); }); it('Should continue to use a valid resource after receiving an invalid CDS update', done => { - const cluster = new FakeEdsCluster('cluster1', [{backends: [new Backend()], locality: {region: 'region1'}}]); - const routeGroup = new FakeRouteGroup('route1', [{cluster: cluster}]); + const cluster = new FakeEdsCluster('cluster1', 'endpoint1', [{backends: [new Backend()], locality: {region: 'region1'}}]); + const routeGroup = new FakeRouteGroup('listener1', 'route1', [{cluster: cluster}]); routeGroup.startAllBackends().then(() => { xdsServer.setEdsResource(cluster.getEndpointConfig()); xdsServer.setCdsResource(cluster.getClusterConfig()); xdsServer.setRdsResource(routeGroup.getRouteConfiguration()); xdsServer.setLdsResource(routeGroup.getListener()); - client = new XdsTestClient('route1', xdsServer); + client = XdsTestClient.createFromServer('listener1', xdsServer); client.startCalls(100); routeGroup.waitForAllBackendsToReceiveTraffic().then(() => { // After backends receive calls, set invalid CDS resource @@ -99,14 +99,14 @@ describe('Validation errors', () => { }, reason => done(reason)); }); it('Should continue to use a valid resource after receiving an invalid RDS update', done => { - const cluster = new FakeEdsCluster('cluster1', [{backends: [new Backend()], locality: {region: 'region1'}}]); - const routeGroup = new FakeRouteGroup('route1', [{cluster: cluster}]); + const cluster = new FakeEdsCluster('cluster1', 'endpoint1', [{backends: [new Backend()], locality: {region: 'region1'}}]); + const routeGroup = new FakeRouteGroup('listener1', 'route1', [{cluster: cluster}]); routeGroup.startAllBackends().then(() => { xdsServer.setEdsResource(cluster.getEndpointConfig()); xdsServer.setCdsResource(cluster.getClusterConfig()); xdsServer.setRdsResource(routeGroup.getRouteConfiguration()); xdsServer.setLdsResource(routeGroup.getListener()); - client = new XdsTestClient('route1', xdsServer); + client = XdsTestClient.createFromServer('listener1', xdsServer); client.startCalls(100); routeGroup.waitForAllBackendsToReceiveTraffic().then(() => { // After backends receive calls, set invalid RDS resource @@ -129,14 +129,14 @@ describe('Validation errors', () => { }, reason => done(reason)); }); it('Should continue to use a valid resource after receiving an invalid LDS update', done => { - const cluster = new FakeEdsCluster('cluster1', [{backends: [new Backend()], locality: {region: 'region1'}}]); - const routeGroup = new FakeRouteGroup('route1', [{cluster: cluster}]); + const cluster = new FakeEdsCluster('cluster1', 'endpoint1', [{backends: [new Backend()], locality: {region: 'region1'}}]); + const routeGroup = new FakeRouteGroup('listener1', 'route1', [{cluster: cluster}]); routeGroup.startAllBackends().then(() => { xdsServer.setEdsResource(cluster.getEndpointConfig()); xdsServer.setCdsResource(cluster.getClusterConfig()); xdsServer.setRdsResource(routeGroup.getRouteConfiguration()); xdsServer.setLdsResource(routeGroup.getListener()); - client = new XdsTestClient('route1', xdsServer); + client = XdsTestClient.createFromServer('listener1', xdsServer); client.startCalls(100); routeGroup.waitForAllBackendsToReceiveTraffic().then(() => { // After backends receive calls, set invalid LDS resource diff --git a/packages/grpc-js-xds/test/xds-server.ts b/packages/grpc-js-xds/test/xds-server.ts index 36fd27b85..c9b829642 100644 --- a/packages/grpc-js-xds/test/xds-server.ts +++ b/packages/grpc-js-xds/test/xds-server.ts @@ -324,15 +324,22 @@ export class XdsServer { this.server?.forceShutdown(); } + getBootstrapServerConfig() { + if (this.port === null) { + throw new Error('Bootstrap info unavailable; server not started'); + } + return { + server_uri: `localhost:${this.port}`, + channel_creds: [{type: 'insecure'}] + }; + } + getBootstrapInfoString(): string { if (this.port === null) { throw new Error('Bootstrap info unavailable; server not started'); } const bootstrapInfo = { - xds_servers: [{ - server_uri: `localhost:${this.port}`, - channel_creds: [{type: 'insecure'}] - }], + xds_servers: [this.getBootstrapServerConfig()], node: { id: 'test', locality: {} @@ -340,4 +347,4 @@ export class XdsServer { } return JSON.stringify(bootstrapInfo); } -} \ No newline at end of file +}