diff --git a/packages/grpc-js-xds/interop/xds-interop-client.ts b/packages/grpc-js-xds/interop/xds-interop-client.ts index dda8bfe92..6db5aff9a 100644 --- a/packages/grpc-js-xds/interop/xds-interop-client.ts +++ b/packages/grpc-js-xds/interop/xds-interop-client.ts @@ -91,11 +91,11 @@ class RpcBehaviorLoadBalancer implements LoadBalancer { private latestConfig: RpcBehaviorLoadBalancingConfig | null = null; constructor(channelControlHelper: ChannelControlHelper) { const childChannelControlHelper = createChildChannelControlHelper(channelControlHelper, { - updateState: (connectivityState, picker) => { + updateState: (connectivityState, picker, errorMessage) => { if (connectivityState === grpc.connectivityState.READY && this.latestConfig) { picker = new RpcBehaviorPicker(picker, this.latestConfig.getRpcBehavior()); } - channelControlHelper.updateState(connectivityState, picker); + channelControlHelper.updateState(connectivityState, picker, errorMessage); } }); this.child = new ChildLoadBalancerHandler(childChannelControlHelper); diff --git a/packages/grpc-js-xds/src/load-balancer-cds.ts b/packages/grpc-js-xds/src/load-balancer-cds.ts index 9bcc18014..23e8e7d15 100644 --- a/packages/grpc-js-xds/src/load-balancer-cds.ts +++ b/packages/grpc-js-xds/src/load-balancer-cds.ts @@ -254,7 +254,7 @@ export class CdsLoadBalancer implements LoadBalancer { } if (!maybeClusterConfig.success) { this.childBalancer.destroy(); - this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error)); + this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error), maybeClusterConfig.error.details); return; } const clusterConfig = maybeClusterConfig.value; @@ -265,7 +265,8 @@ export class CdsLoadBalancer implements LoadBalancer { leafClusters = getLeafClusters(xdsConfig, clusterName); } catch (e) { trace('xDS config parsing failed with error ' + (e as Error).message); - this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `xDS config parsing failed with error ${(e as Error).message}`})); + const errorMessage = `xDS config parsing failed with error ${(e as Error).message}`; + this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage); return; } const priorityChildren: {[name: string]: PriorityChildRaw} = {}; @@ -290,14 +291,16 @@ export class CdsLoadBalancer implements LoadBalancer { typedChildConfig = parseLoadBalancingConfig(childConfig); } catch (e) { trace('LB policy config parsing failed with error ' + (e as Error).message); - this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `LB policy config parsing failed with error ${(e as Error).message}`})); + const errorMessage = `LB policy config parsing failed with error ${(e as Error).message}`; + this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage); return; } this.childBalancer.updateAddressList(endpointList, typedChildConfig, {...options, [ROOT_CLUSTER_KEY]: clusterName}); } else { if (!clusterConfig.children.endpoints) { trace('Received update with no resolved endpoints for cluster ' + clusterName); - this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} resolution failed: ${clusterConfig.children.resolutionNote}`})); + const errorMessage = `Cluster ${clusterName} resolution failed: ${clusterConfig.children.resolutionNote}`; + this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage); return; } const newPriorityNames: string[] = []; @@ -402,7 +405,8 @@ export class CdsLoadBalancer implements LoadBalancer { typedChildConfig = parseLoadBalancingConfig(childConfig); } catch (e) { trace('LB policy config parsing failed with error ' + (e as Error).message); - this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `LB policy config parsing failed with error ${(e as Error).message}`})); + const errorMessage = `LB policy config parsing failed with error ${(e as Error).message}`; + this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage); return; } const childOptions: ChannelOptions = {...options}; @@ -411,13 +415,15 @@ export class CdsLoadBalancer implements LoadBalancer { const xdsClient = options[XDS_CLIENT_KEY] as XdsClient; const caCertProvider = xdsClient.getCertificateProvider(securityUpdate.caCertificateProviderInstance); if (!caCertProvider) { - this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} configured with CA certificate provider ${securityUpdate.caCertificateProviderInstance} not in bootstrap`})); + const errorMessage = `Cluster ${clusterName} configured with CA certificate provider ${securityUpdate.caCertificateProviderInstance} not in bootstrap`; + this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage); return; } if (securityUpdate.identityCertificateProviderInstance) { const identityCertProvider = xdsClient.getCertificateProvider(securityUpdate.identityCertificateProviderInstance); if (!identityCertProvider) { - this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} configured with identity certificate provider ${securityUpdate.identityCertificateProviderInstance} not in bootstrap`})); + const errorMessage = `Cluster ${clusterName} configured with identity certificate provider ${securityUpdate.identityCertificateProviderInstance} not in bootstrap`; + this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage); return; } childOptions[IDENTITY_CERT_PROVIDER_KEY] = identityCertProvider; diff --git a/packages/grpc-js-xds/src/load-balancer-priority.ts b/packages/grpc-js-xds/src/load-balancer-priority.ts index d9df0191a..9d6ed5c64 100644 --- a/packages/grpc-js-xds/src/load-balancer-priority.ts +++ b/packages/grpc-js-xds/src/load-balancer-priority.ts @@ -166,6 +166,7 @@ interface PriorityChildBalancer { isFailoverTimerPending(): boolean; getConnectivityState(): ConnectivityState; getPicker(): Picker; + getErrorMessage(): string | null; getName(): string; destroy(): void; } @@ -183,14 +184,15 @@ export class PriorityLoadBalancer implements LoadBalancer { private PriorityChildImpl = class implements PriorityChildBalancer { private connectivityState: ConnectivityState = ConnectivityState.IDLE; private picker: Picker; + private errorMessage: string | null = null; private childBalancer: ChildLoadBalancerHandler; private failoverTimer: NodeJS.Timeout | null = null; private deactivationTimer: NodeJS.Timeout | null = null; private seenReadyOrIdleSinceTransientFailure = false; constructor(private parent: PriorityLoadBalancer, private name: string, ignoreReresolutionRequests: boolean) { this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, { - updateState: (connectivityState: ConnectivityState, picker: Picker) => { - this.updateState(connectivityState, picker); + updateState: (connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) => { + this.updateState(connectivityState, picker, errorMessage); }, requestReresolution: () => { if (!ignoreReresolutionRequests) { @@ -202,10 +204,11 @@ export class PriorityLoadBalancer implements LoadBalancer { this.startFailoverTimer(); } - private updateState(connectivityState: ConnectivityState, picker: Picker) { + private updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) { trace('Child ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]); this.connectivityState = connectivityState; this.picker = picker; + this.errorMessage = errorMessage; if (connectivityState === ConnectivityState.CONNECTING) { if (this.seenReadyOrIdleSinceTransientFailure && this.failoverTimer === null) { this.startFailoverTimer(); @@ -226,9 +229,11 @@ export class PriorityLoadBalancer implements LoadBalancer { this.failoverTimer = setTimeout(() => { trace('Failover timer triggered for child ' + this.name); this.failoverTimer = null; + const errorMessage = `No connection established. Last error: ${this.errorMessage}`; this.updateState( ConnectivityState.TRANSIENT_FAILURE, - new UnavailablePicker() + new UnavailablePicker({code: Status.UNAVAILABLE, details: errorMessage}), + errorMessage ); }, DEFAULT_FAILOVER_TIME_MS); } @@ -285,6 +290,10 @@ export class PriorityLoadBalancer implements LoadBalancer { return this.picker; } + getErrorMessage() { + return this.errorMessage; + } + getName() { return this.name; } @@ -325,7 +334,7 @@ export class PriorityLoadBalancer implements LoadBalancer { constructor(private channelControlHelper: ChannelControlHelper) {} - private updateState(state: ConnectivityState, picker: Picker) { + private updateState(state: ConnectivityState, picker: Picker, errorMessage: string | null) { trace( 'Transitioning to ' + ConnectivityState[state] @@ -336,7 +345,7 @@ export class PriorityLoadBalancer implements LoadBalancer { if (state === ConnectivityState.IDLE) { picker = new QueuePicker(this, picker); } - this.channelControlHelper.updateState(state, picker); + this.channelControlHelper.updateState(state, picker, errorMessage); } private onChildStateChange(child: PriorityChildBalancer) { @@ -363,7 +372,8 @@ export class PriorityLoadBalancer implements LoadBalancer { const chosenChild = this.children.get(this.priorities[priority])!; this.updateState( chosenChild.getConnectivityState(), - chosenChild.getPicker() + chosenChild.getPicker(), + chosenChild.getErrorMessage() ); if (deactivateLowerPriorities) { for (let i = priority + 1; i < this.priorities.length; i++) { @@ -374,7 +384,8 @@ export class PriorityLoadBalancer implements LoadBalancer { private choosePriority() { if (this.priorities.length === 0) { - this.updateState(ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: Status.UNAVAILABLE, details: 'priority policy has empty priority list', metadata: new Metadata()})); + const errorMessage = 'priority policy has empty priority list'; + this.updateState(ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: Status.UNAVAILABLE, details: errorMessage}), errorMessage); return; } diff --git a/packages/grpc-js-xds/src/load-balancer-ring-hash.ts b/packages/grpc-js-xds/src/load-balancer-ring-hash.ts index 77f0ab317..b10452f36 100644 --- a/packages/grpc-js-xds/src/load-balancer-ring-hash.ts +++ b/packages/grpc-js-xds/src/load-balancer-ring-hash.ts @@ -225,11 +225,15 @@ class RingHashLoadBalancer implements LoadBalancer { private updatesPaused = false; private currentState: connectivityState = connectivityState.IDLE; private ring: RingEntry[] = []; + private latestErrorMessage: string | null = null; constructor(private channelControlHelper: ChannelControlHelper) { this.childChannelControlHelper = createChildChannelControlHelper( channelControlHelper, { - updateState: (state, picker) => { + updateState: (state, picker, errorMessage) => { + if (errorMessage) { + this.latestErrorMessage = errorMessage; + } this.calculateAndUpdateState(); /* If this LB policy is in the TRANSIENT_FAILURE state, requests will * not trigger new connections, so we need to explicitly try connecting @@ -270,17 +274,20 @@ class RingHashLoadBalancer implements LoadBalancer { stateCounts[leaf.getConnectivityState()] += 1; } if (stateCounts[connectivityState.READY] > 0) { - this.updateState(connectivityState.READY, new RingHashPicker(this.ring)); + this.updateState(connectivityState.READY, new RingHashPicker(this.ring), null); // REPORT READY } else if (stateCounts[connectivityState.TRANSIENT_FAILURE] > 1) { + const errorMessage = `ring hash: no connection established. Latest error: ${this.latestErrorMessage}`; this.updateState( connectivityState.TRANSIENT_FAILURE, - new UnavailablePicker() + new UnavailablePicker({details: errorMessage}), + errorMessage ); } else if (stateCounts[connectivityState.CONNECTING] > 0) { this.updateState( connectivityState.CONNECTING, - new RingHashPicker(this.ring) + new RingHashPicker(this.ring), + null ); } else if ( stateCounts[connectivityState.TRANSIENT_FAILURE] > 0 && @@ -288,26 +295,29 @@ class RingHashLoadBalancer implements LoadBalancer { ) { this.updateState( connectivityState.CONNECTING, - new RingHashPicker(this.ring) + new RingHashPicker(this.ring), + null ); } else if (stateCounts[connectivityState.IDLE] > 0) { - this.updateState(connectivityState.IDLE, new RingHashPicker(this.ring)); + this.updateState(connectivityState.IDLE, new RingHashPicker(this.ring), null); } else { + const errorMessage = `ring hash: no connection established. Latest error: ${this.latestErrorMessage}`; this.updateState( connectivityState.TRANSIENT_FAILURE, - new UnavailablePicker() + new UnavailablePicker({details: errorMessage}), + errorMessage ); } } - private updateState(newState: connectivityState, picker: Picker) { + private updateState(newState: connectivityState, picker: Picker, errorMessage: string | null) { trace( connectivityState[this.currentState] + ' -> ' + connectivityState[newState] ); this.currentState = newState; - this.channelControlHelper.updateState(newState, picker); + this.channelControlHelper.updateState(newState, picker, errorMessage); } private constructRing( diff --git a/packages/grpc-js-xds/src/load-balancer-weighted-target.ts b/packages/grpc-js-xds/src/load-balancer-weighted-target.ts index 218b1ab44..2e73a7d97 100644 --- a/packages/grpc-js-xds/src/load-balancer-weighted-target.ts +++ b/packages/grpc-js-xds/src/load-balancer-weighted-target.ts @@ -175,18 +175,21 @@ export class WeightedTargetLoadBalancer implements LoadBalancer { constructor(private parent: WeightedTargetLoadBalancer, private name: string) { this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, { - updateState: (connectivityState: ConnectivityState, picker: Picker) => { - this.updateState(connectivityState, picker); + updateState: (connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) => { + this.updateState(connectivityState, picker, errorMessage); }, })); this.picker = new QueuePicker(this.childBalancer); } - private updateState(connectivityState: ConnectivityState, picker: Picker) { + private updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) { trace('Target ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]); this.connectivityState = connectivityState; this.picker = picker; + if (errorMessage) { + this.parent.latestChildErrorMessage = errorMessage; + } this.parent.maybeUpdateState(); } @@ -242,6 +245,7 @@ export class WeightedTargetLoadBalancer implements LoadBalancer { */ private targetList: string[] = []; private updatesPaused = false; + private latestChildErrorMessage: string | null = null; constructor(private channelControlHelper: ChannelControlHelper) {} @@ -297,6 +301,7 @@ export class WeightedTargetLoadBalancer implements LoadBalancer { } let picker: Picker; + let errorMessage: string | null = null; switch (connectivityState) { case ConnectivityState.READY: picker = new WeightedTargetPicker(pickerList); @@ -306,9 +311,10 @@ export class WeightedTargetLoadBalancer implements LoadBalancer { picker = new QueuePicker(this); break; default: + const errorMessage = `weighted_target: all children report state TRANSIENT_FAILURE. Latest error: ${this.latestChildErrorMessage}`; picker = new UnavailablePicker({ code: Status.UNAVAILABLE, - details: 'weighted_target: all children report state TRANSIENT_FAILURE', + details: errorMessage, metadata: new Metadata() }); } @@ -316,7 +322,7 @@ export class WeightedTargetLoadBalancer implements LoadBalancer { 'Transitioning to ' + ConnectivityState[connectivityState] ); - this.channelControlHelper.updateState(connectivityState, picker); + this.channelControlHelper.updateState(connectivityState, picker, errorMessage); } updateAddressList(addressList: Endpoint[], lbConfig: TypedLoadBalancingConfig, options: ChannelOptions): void { diff --git a/packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts b/packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts index c2558652d..a7a981090 100644 --- a/packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts +++ b/packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts @@ -241,12 +241,12 @@ class XdsClusterImplBalancer implements LoadBalancer { } return new LocalitySubchannelWrapper(wrapperChild, statsObj); }, - updateState: (connectivityState, originalPicker) => { + updateState: (connectivityState, originalPicker, errorMessage) => { if (this.latestConfig === null || this.latestClusterConfig === null || this.latestClusterConfig.children.type === 'aggregate' || !this.latestClusterConfig.children.endpoints) { - channelControlHelper.updateState(connectivityState, originalPicker); + channelControlHelper.updateState(connectivityState, originalPicker, errorMessage); } else { const picker = new XdsClusterImplPicker(originalPicker, getCallCounterMapKey(this.latestConfig.getCluster(), this.latestClusterConfig.cluster.edsServiceName), this.latestClusterConfig.cluster.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS, this.latestClusterConfig.children.endpoints.dropCategories, this.clusterDropStats); - channelControlHelper.updateState(connectivityState, picker); + channelControlHelper.updateState(connectivityState, picker, errorMessage); } } })); @@ -266,7 +266,7 @@ class XdsClusterImplBalancer implements LoadBalancer { if (!maybeClusterConfig.success) { this.latestClusterConfig = null; this.childBalancer.destroy(); - this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error)); + this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error), maybeClusterConfig.error.details); return; } const clusterConfig = maybeClusterConfig.value; @@ -276,7 +276,7 @@ class XdsClusterImplBalancer implements LoadBalancer { } if (!clusterConfig.children.endpoints) { this.childBalancer.destroy(); - this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({details: clusterConfig.children.resolutionNote})); + this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({details: clusterConfig.children.resolutionNote}), clusterConfig.children.resolutionNote ?? null); } this.lastestEndpointList = endpointList; diff --git a/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts b/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts index 0843c4505..9dc4ee948 100644 --- a/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts +++ b/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts @@ -128,18 +128,21 @@ class XdsClusterManager implements LoadBalancer { constructor(private parent: XdsClusterManager, private name: string) { this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, { - updateState: (connectivityState: ConnectivityState, picker: Picker) => { - this.updateState(connectivityState, picker); + updateState: (connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) => { + this.updateState(connectivityState, picker, errorMessage); }, })); this.picker = new QueuePicker(this.childBalancer); } - private updateState(connectivityState: ConnectivityState, picker: Picker) { + private updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) { trace('Child ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]); this.connectivityState = connectivityState; this.picker = picker; + if (errorMessage) { + this.parent.latestChildErrorMessage = errorMessage; + } this.parent.maybeUpdateState(); } updateAddressList(endpointList: Endpoint[], childConfig: TypedLoadBalancingConfig, options: ChannelOptions): void { @@ -167,6 +170,7 @@ class XdsClusterManager implements LoadBalancer { // Shutdown is a placeholder value that will never appear in normal operation. private currentState: ConnectivityState = ConnectivityState.SHUTDOWN; private updatesPaused = false; + private latestChildErrorMessage: string | null = null; constructor(private channelControlHelper: ChannelControlHelper) {} private maybeUpdateState() { @@ -195,6 +199,7 @@ class XdsClusterManager implements LoadBalancer { } } let connectivityState: ConnectivityState; + let errorMessage: string | null = null; if (anyReady) { connectivityState = ConnectivityState.READY; } else if (anyConnecting) { @@ -203,8 +208,9 @@ class XdsClusterManager implements LoadBalancer { connectivityState = ConnectivityState.IDLE; } else { connectivityState = ConnectivityState.TRANSIENT_FAILURE; + errorMessage = `xds_cluster_manager: No connection established. Latest error: ${this.latestChildErrorMessage}`; } - this.channelControlHelper.updateState(connectivityState, new XdsClusterManagerPicker(pickerMap)); + this.channelControlHelper.updateState(connectivityState, new XdsClusterManagerPicker(pickerMap), errorMessage); } updateAddressList(endpointList: Endpoint[], lbConfig: TypedLoadBalancingConfig, options: ChannelOptions): void { diff --git a/packages/grpc-js-xds/test/test-custom-lb-policies.ts b/packages/grpc-js-xds/test/test-custom-lb-policies.ts index 8d91c2b5d..bec7986b2 100644 --- a/packages/grpc-js-xds/test/test-custom-lb-policies.ts +++ b/packages/grpc-js-xds/test/test-custom-lb-policies.ts @@ -86,11 +86,11 @@ class RpcBehaviorLoadBalancer implements LoadBalancer { private latestConfig: RpcBehaviorLoadBalancingConfig | null = null; constructor(channelControlHelper: ChannelControlHelper) { const childChannelControlHelper = createChildChannelControlHelper(channelControlHelper, { - updateState: (state, picker) => { + updateState: (state, picker, errorMessage) => { if (state === connectivityState.READY && this.latestConfig) { picker = new RpcBehaviorPicker(picker, this.latestConfig.getRpcBehavior()); } - channelControlHelper.updateState(state, picker); + channelControlHelper.updateState(state, picker, errorMessage); } }); this.child = new ChildLoadBalancerHandler(childChannelControlHelper); diff --git a/packages/grpc-js/src/load-balancer-child-handler.ts b/packages/grpc-js/src/load-balancer-child-handler.ts index 3d1baf1ba..1fcb30b31 100644 --- a/packages/grpc-js/src/load-balancer-child-handler.ts +++ b/packages/grpc-js/src/load-balancer-child-handler.ts @@ -47,7 +47,7 @@ export class ChildLoadBalancerHandler { subchannelArgs ); } - updateState(connectivityState: ConnectivityState, picker: Picker): void { + updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null): void { if (this.calledByPendingChild()) { if (connectivityState === ConnectivityState.CONNECTING) { return; @@ -58,7 +58,7 @@ export class ChildLoadBalancerHandler { } else if (!this.calledByCurrentChild()) { return; } - this.parent.channelControlHelper.updateState(connectivityState, picker); + this.parent.channelControlHelper.updateState(connectivityState, picker, errorMessage); } requestReresolution(): void { const latestChild = this.parent.pendingChild ?? this.parent.currentChild; diff --git a/packages/grpc-js/src/load-balancer-outlier-detection.ts b/packages/grpc-js/src/load-balancer-outlier-detection.ts index c5f7e179b..a83e40bcf 100644 --- a/packages/grpc-js/src/load-balancer-outlier-detection.ts +++ b/packages/grpc-js/src/load-balancer-outlier-detection.ts @@ -493,14 +493,15 @@ export class OutlierDetectionLoadBalancer implements LoadBalancer { mapEntry?.subchannelWrappers.push(subchannelWrapper); return subchannelWrapper; }, - updateState: (connectivityState: ConnectivityState, picker: Picker) => { + updateState: (connectivityState: ConnectivityState, picker: Picker, errorMessage: string) => { if (connectivityState === ConnectivityState.READY) { channelControlHelper.updateState( connectivityState, - new OutlierDetectionPicker(picker, this.isCountingEnabled()) + new OutlierDetectionPicker(picker, this.isCountingEnabled()), + errorMessage ); } else { - channelControlHelper.updateState(connectivityState, picker); + channelControlHelper.updateState(connectivityState, picker, errorMessage); } }, }) diff --git a/packages/grpc-js/src/load-balancer-pick-first.ts b/packages/grpc-js/src/load-balancer-pick-first.ts index bf3798e4b..28410ed77 100644 --- a/packages/grpc-js/src/load-balancer-pick-first.ts +++ b/packages/grpc-js/src/load-balancer-pick-first.ts @@ -261,37 +261,44 @@ export class PickFirstLoadBalancer implements LoadBalancer { private calculateAndReportNewState() { if (this.currentPick) { if (this.reportHealthStatus && !this.currentPick.isHealthy()) { + const errorMessage = `Picked subchannel ${this.currentPick.getAddress()} is unhealthy`; this.updateState( ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({ - details: `Picked subchannel ${this.currentPick.getAddress()} is unhealthy`, - }) + details: errorMessage, + }), + errorMessage ); } else { this.updateState( ConnectivityState.READY, - new PickFirstPicker(this.currentPick) + new PickFirstPicker(this.currentPick), + null ); } } else if (this.latestAddressList?.length === 0) { + const errorMessage = `No connection established. Last error: ${this.lastError}`; this.updateState( ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({ - details: `No connection established. Last error: ${this.lastError}`, - }) + details: errorMessage, + }), + errorMessage ); } else if (this.children.length === 0) { - this.updateState(ConnectivityState.IDLE, new QueuePicker(this)); + this.updateState(ConnectivityState.IDLE, new QueuePicker(this), null); } else { if (this.stickyTransientFailureMode) { + const errorMessage = `No connection established. Last error: ${this.lastError}`; this.updateState( ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({ - details: `No connection established. Last error: ${this.lastError}`, - }) + details: errorMessage, + }), + errorMessage ); } else { - this.updateState(ConnectivityState.CONNECTING, new QueuePicker(this)); + this.updateState(ConnectivityState.CONNECTING, new QueuePicker(this), null); } } } @@ -431,14 +438,14 @@ export class PickFirstLoadBalancer implements LoadBalancer { this.calculateAndReportNewState(); } - private updateState(newState: ConnectivityState, picker: Picker) { + private updateState(newState: ConnectivityState, picker: Picker, errorMessage: string | null) { trace( ConnectivityState[this.currentState] + ' -> ' + ConnectivityState[newState] ); this.currentState = newState; - this.channelControlHelper.updateState(newState, picker); + this.channelControlHelper.updateState(newState, picker, errorMessage); } private resetSubchannelList() { @@ -568,10 +575,10 @@ export class LeafLoadBalancer { const childChannelControlHelper = createChildChannelControlHelper( channelControlHelper, { - updateState: (connectivityState, picker) => { + updateState: (connectivityState, picker, errorMessage) => { this.latestState = connectivityState; this.latestPicker = picker; - channelControlHelper.updateState(connectivityState, picker); + channelControlHelper.updateState(connectivityState, picker, errorMessage); }, } ); diff --git a/packages/grpc-js/src/load-balancer-round-robin.ts b/packages/grpc-js/src/load-balancer-round-robin.ts index 4482b86b8..1b1d97fc8 100644 --- a/packages/grpc-js/src/load-balancer-round-robin.ts +++ b/packages/grpc-js/src/load-balancer-round-robin.ts @@ -108,7 +108,7 @@ export class RoundRobinLoadBalancer implements LoadBalancer { this.childChannelControlHelper = createChildChannelControlHelper( channelControlHelper, { - updateState: (connectivityState, picker) => { + updateState: (connectivityState, picker, errorMessage) => { /* Ensure that name resolution is requested again after active * connections are dropped. This is more aggressive than necessary to * accomplish that, so we are counting on resolvers to have @@ -116,6 +116,9 @@ export class RoundRobinLoadBalancer implements LoadBalancer { if (this.currentState === ConnectivityState.READY && connectivityState !== ConnectivityState.READY) { this.channelControlHelper.requestReresolution(); } + if (errorMessage) { + this.lastError = errorMessage; + } this.calculateAndUpdateState(); }, } @@ -153,21 +156,24 @@ export class RoundRobinLoadBalancer implements LoadBalancer { picker: child.getPicker(), })), index - ) + ), + null ); } else if (this.countChildrenWithState(ConnectivityState.CONNECTING) > 0) { - this.updateState(ConnectivityState.CONNECTING, new QueuePicker(this)); + this.updateState(ConnectivityState.CONNECTING, new QueuePicker(this), null); } else if ( this.countChildrenWithState(ConnectivityState.TRANSIENT_FAILURE) > 0 ) { + const errorMessage = `round_robin: No connection established. Last error: ${this.lastError}`; this.updateState( ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({ - details: `No connection established. Last error: ${this.lastError}`, - }) + details: errorMessage, + }), + errorMessage ); } else { - this.updateState(ConnectivityState.IDLE, new QueuePicker(this)); + this.updateState(ConnectivityState.IDLE, new QueuePicker(this), null); } /* round_robin should keep all children connected, this is how we do that. * We can't do this more efficiently in the individual child's updateState @@ -180,7 +186,7 @@ export class RoundRobinLoadBalancer implements LoadBalancer { } } - private updateState(newState: ConnectivityState, picker: Picker) { + private updateState(newState: ConnectivityState, picker: Picker, errorMessage: string | null) { trace( ConnectivityState[this.currentState] + ' -> ' + @@ -192,7 +198,7 @@ export class RoundRobinLoadBalancer implements LoadBalancer { this.currentReadyPicker = null; } this.currentState = newState; - this.channelControlHelper.updateState(newState, picker); + this.channelControlHelper.updateState(newState, picker, errorMessage); } private resetSubchannelList() { diff --git a/packages/grpc-js/src/load-balancer.ts b/packages/grpc-js/src/load-balancer.ts index 1ea9c60c3..22f0c4f57 100644 --- a/packages/grpc-js/src/load-balancer.ts +++ b/packages/grpc-js/src/load-balancer.ts @@ -46,7 +46,11 @@ export interface ChannelControlHelper { * @param connectivityState New connectivity state * @param picker New picker */ - updateState(connectivityState: ConnectivityState, picker: Picker): void; + updateState( + connectivityState: ConnectivityState, + picker: Picker, + errorMessage: string | null + ): void; /** * Request new data from the resolver. */ diff --git a/packages/grpc-js/src/resolving-load-balancer.ts b/packages/grpc-js/src/resolving-load-balancer.ts index d18cf700e..d3eeb2369 100644 --- a/packages/grpc-js/src/resolving-load-balancer.ts +++ b/packages/grpc-js/src/resolving-load-balancer.ts @@ -160,6 +160,7 @@ export class ResolvingLoadBalancer implements LoadBalancer { private readonly childLoadBalancer: ChildLoadBalancerHandler; private latestChildState: ConnectivityState = ConnectivityState.IDLE; private latestChildPicker: Picker = new QueuePicker(this); + private latestChildErrorMessage: string | null = null; /** * This resolving load balancer's current connectivity state. */ @@ -213,7 +214,7 @@ export class ResolvingLoadBalancer implements LoadBalancer { }; } - this.updateState(ConnectivityState.IDLE, new QueuePicker(this)); + this.updateState(ConnectivityState.IDLE, new QueuePicker(this), null); this.childLoadBalancer = new ChildLoadBalancerHandler( { createSubchannel: @@ -233,10 +234,11 @@ export class ResolvingLoadBalancer implements LoadBalancer { this.updateResolution(); } }, - updateState: (newState: ConnectivityState, picker: Picker) => { + updateState: (newState: ConnectivityState, picker: Picker, errorMessage: string | null) => { this.latestChildState = newState; this.latestChildPicker = picker; - this.updateState(newState, picker); + this.latestChildErrorMessage = errorMessage; + this.updateState(newState, picker, errorMessage); }, addChannelzChild: channelControlHelper.addChannelzChild.bind(channelControlHelper), @@ -325,7 +327,7 @@ export class ResolvingLoadBalancer implements LoadBalancer { this.updateResolution(); this.continueResolving = false; } else { - this.updateState(this.latestChildState, this.latestChildPicker); + this.updateState(this.latestChildState, this.latestChildPicker, this.latestChildErrorMessage); } }, backoffOptions); this.backoffTimeout.unref(); @@ -338,12 +340,12 @@ export class ResolvingLoadBalancer implements LoadBalancer { * is an appropriate value here if the child LB policy is unset. * Otherwise, we want to delegate to the child here, in case that * triggers something. */ - this.updateState(ConnectivityState.CONNECTING, this.latestChildPicker); + this.updateState(ConnectivityState.CONNECTING, this.latestChildPicker, this.latestChildErrorMessage); } this.backoffTimeout.runOnce(); } - private updateState(connectivityState: ConnectivityState, picker: Picker) { + private updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) { trace( uriToString(this.target) + ' ' + @@ -356,14 +358,15 @@ export class ResolvingLoadBalancer implements LoadBalancer { picker = new QueuePicker(this, picker); } this.currentState = connectivityState; - this.channelControlHelper.updateState(connectivityState, picker); + this.channelControlHelper.updateState(connectivityState, picker, errorMessage); } private handleResolutionFailure(error: StatusObject) { if (this.latestChildState === ConnectivityState.IDLE) { this.updateState( ConnectivityState.TRANSIENT_FAILURE, - new UnavailablePicker(error) + new UnavailablePicker(error), + error.details ); this.onFailedResolution(error); }