Skip to content

Commit

Permalink
Merge pull request #2868 from murgatroid99/grpc-js_error_propagation
Browse files Browse the repository at this point in the history 
grpc-js: Propagate error messages through LB policy tree
  • Loading branch information
@murgatroid99
murgatroid99 authored Dec 10, 2024
2 parents 614e5f9 + fad797f commit 5cd30ae
Show file tree
Hide file tree
Showing 14 changed files with 137 additions and 77 deletions.
4 changes: 2 additions & 2 deletions packages/grpc-js-xds/interop/xds-interop-client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,11 @@ class RpcBehaviorLoadBalancer implements LoadBalancer {
private latestConfig: RpcBehaviorLoadBalancingConfig | null = null;
constructor(channelControlHelper: ChannelControlHelper) {
const childChannelControlHelper = createChildChannelControlHelper(channelControlHelper, {
updateState: (connectivityState, picker) => {
updateState: (connectivityState, picker, errorMessage) => {
if (connectivityState === grpc.connectivityState.READY && this.latestConfig) {
picker = new RpcBehaviorPicker(picker, this.latestConfig.getRpcBehavior());
}
channelControlHelper.updateState(connectivityState, picker);
channelControlHelper.updateState(connectivityState, picker, errorMessage);
}
});
this.child = new ChildLoadBalancerHandler(childChannelControlHelper);
Expand Down
20 changes: 13 additions & 7 deletions packages/grpc-js-xds/src/load-balancer-cds.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ export class CdsLoadBalancer implements LoadBalancer {
}
if (!maybeClusterConfig.success) {
this.childBalancer.destroy();
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error));
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error), maybeClusterConfig.error.details);
return;
}
const clusterConfig = maybeClusterConfig.value;
Expand All @@ -265,7 +265,8 @@ export class CdsLoadBalancer implements LoadBalancer {
leafClusters = getLeafClusters(xdsConfig, clusterName);
} catch (e) {
trace('xDS config parsing failed with error ' + (e as Error).message);
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `xDS config parsing failed with error ${(e as Error).message}`}));
const errorMessage = `xDS config parsing failed with error ${(e as Error).message}`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
const priorityChildren: {[name: string]: PriorityChildRaw} = {};
Expand All @@ -290,14 +291,16 @@ export class CdsLoadBalancer implements LoadBalancer {
typedChildConfig = parseLoadBalancingConfig(childConfig);
} catch (e) {
trace('LB policy config parsing failed with error ' + (e as Error).message);
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `LB policy config parsing failed with error ${(e as Error).message}`}));
const errorMessage = `LB policy config parsing failed with error ${(e as Error).message}`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
this.childBalancer.updateAddressList(endpointList, typedChildConfig, {...options, [ROOT_CLUSTER_KEY]: clusterName});
} else {
if (!clusterConfig.children.endpoints) {
trace('Received update with no resolved endpoints for cluster ' + clusterName);
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} resolution failed: ${clusterConfig.children.resolutionNote}`}));
const errorMessage = `Cluster ${clusterName} resolution failed: ${clusterConfig.children.resolutionNote}`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
const newPriorityNames: string[] = [];
Expand Down Expand Up @@ -402,7 +405,8 @@ export class CdsLoadBalancer implements LoadBalancer {
typedChildConfig = parseLoadBalancingConfig(childConfig);
} catch (e) {
trace('LB policy config parsing failed with error ' + (e as Error).message);
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `LB policy config parsing failed with error ${(e as Error).message}`}));
const errorMessage = `LB policy config parsing failed with error ${(e as Error).message}`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
const childOptions: ChannelOptions = {...options};
Expand All @@ -411,13 +415,15 @@ export class CdsLoadBalancer implements LoadBalancer {
const xdsClient = options[XDS_CLIENT_KEY] as XdsClient;
const caCertProvider = xdsClient.getCertificateProvider(securityUpdate.caCertificateProviderInstance);
if (!caCertProvider) {
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} configured with CA certificate provider ${securityUpdate.caCertificateProviderInstance} not in bootstrap`}));
const errorMessage = `Cluster ${clusterName} configured with CA certificate provider ${securityUpdate.caCertificateProviderInstance} not in bootstrap`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
if (securityUpdate.identityCertificateProviderInstance) {
const identityCertProvider = xdsClient.getCertificateProvider(securityUpdate.identityCertificateProviderInstance);
if (!identityCertProvider) {
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: `Cluster ${clusterName} configured with identity certificate provider ${securityUpdate.identityCertificateProviderInstance} not in bootstrap`}));
const errorMessage = `Cluster ${clusterName} configured with identity certificate provider ${securityUpdate.identityCertificateProviderInstance} not in bootstrap`;
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}
childOptions[IDENTITY_CERT_PROVIDER_KEY] = identityCertProvider;
Expand Down
27 changes: 19 additions & 8 deletions packages/grpc-js-xds/src/load-balancer-priority.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ interface PriorityChildBalancer {
isFailoverTimerPending(): boolean;
getConnectivityState(): ConnectivityState;
getPicker(): Picker;
getErrorMessage(): string | null;
getName(): string;
destroy(): void;
}
Expand All @@ -183,14 +184,15 @@ export class PriorityLoadBalancer implements LoadBalancer {
private PriorityChildImpl = class implements PriorityChildBalancer {
private connectivityState: ConnectivityState = ConnectivityState.IDLE;
private picker: Picker;
private errorMessage: string | null = null;
private childBalancer: ChildLoadBalancerHandler;
private failoverTimer: NodeJS.Timeout | null = null;
private deactivationTimer: NodeJS.Timeout | null = null;
private seenReadyOrIdleSinceTransientFailure = false;
constructor(private parent: PriorityLoadBalancer, private name: string, ignoreReresolutionRequests: boolean) {
this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, {
updateState: (connectivityState: ConnectivityState, picker: Picker) => {
this.updateState(connectivityState, picker);
updateState: (connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) => {
this.updateState(connectivityState, picker, errorMessage);
},
requestReresolution: () => {
if (!ignoreReresolutionRequests) {
Expand All @@ -202,10 +204,11 @@ export class PriorityLoadBalancer implements LoadBalancer {
this.startFailoverTimer();
}

private updateState(connectivityState: ConnectivityState, picker: Picker) {
private updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) {
trace('Child ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]);
this.connectivityState = connectivityState;
this.picker = picker;
this.errorMessage = errorMessage;
if (connectivityState === ConnectivityState.CONNECTING) {
if (this.seenReadyOrIdleSinceTransientFailure && this.failoverTimer === null) {
this.startFailoverTimer();
Expand All @@ -226,9 +229,11 @@ export class PriorityLoadBalancer implements LoadBalancer {
this.failoverTimer = setTimeout(() => {
trace('Failover timer triggered for child ' + this.name);
this.failoverTimer = null;
const errorMessage = `No connection established. Last error: ${this.errorMessage}`;
this.updateState(
ConnectivityState.TRANSIENT_FAILURE,
new UnavailablePicker()
new UnavailablePicker({code: Status.UNAVAILABLE, details: errorMessage}),
errorMessage
);
}, DEFAULT_FAILOVER_TIME_MS);
}
Expand Down Expand Up @@ -285,6 +290,10 @@ export class PriorityLoadBalancer implements LoadBalancer {
return this.picker;
}

getErrorMessage() {
return this.errorMessage;
}

getName() {
return this.name;
}
Expand Down Expand Up @@ -325,7 +334,7 @@ export class PriorityLoadBalancer implements LoadBalancer {

constructor(private channelControlHelper: ChannelControlHelper) {}

private updateState(state: ConnectivityState, picker: Picker) {
private updateState(state: ConnectivityState, picker: Picker, errorMessage: string | null) {
trace(
'Transitioning to ' +
ConnectivityState[state]
Expand All @@ -336,7 +345,7 @@ export class PriorityLoadBalancer implements LoadBalancer {
if (state === ConnectivityState.IDLE) {
picker = new QueuePicker(this, picker);
}
this.channelControlHelper.updateState(state, picker);
this.channelControlHelper.updateState(state, picker, errorMessage);
}

private onChildStateChange(child: PriorityChildBalancer) {
Expand All @@ -363,7 +372,8 @@ export class PriorityLoadBalancer implements LoadBalancer {
const chosenChild = this.children.get(this.priorities[priority])!;
this.updateState(
chosenChild.getConnectivityState(),
chosenChild.getPicker()
chosenChild.getPicker(),
chosenChild.getErrorMessage()
);
if (deactivateLowerPriorities) {
for (let i = priority + 1; i < this.priorities.length; i++) {
Expand All @@ -374,7 +384,8 @@ export class PriorityLoadBalancer implements LoadBalancer {

private choosePriority() {
if (this.priorities.length === 0) {
this.updateState(ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: Status.UNAVAILABLE, details: 'priority policy has empty priority list', metadata: new Metadata()}));
const errorMessage = 'priority policy has empty priority list';
this.updateState(ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: Status.UNAVAILABLE, details: errorMessage}), errorMessage);
return;
}

Expand Down
28 changes: 19 additions & 9 deletions packages/grpc-js-xds/src/load-balancer-ring-hash.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,11 +225,15 @@ class RingHashLoadBalancer implements LoadBalancer {
private updatesPaused = false;
private currentState: connectivityState = connectivityState.IDLE;
private ring: RingEntry[] = [];
private latestErrorMessage: string | null = null;
constructor(private channelControlHelper: ChannelControlHelper) {
this.childChannelControlHelper = createChildChannelControlHelper(
channelControlHelper,
{
updateState: (state, picker) => {
updateState: (state, picker, errorMessage) => {
if (errorMessage) {
this.latestErrorMessage = errorMessage;
}
this.calculateAndUpdateState();
/* If this LB policy is in the TRANSIENT_FAILURE state, requests will
* not trigger new connections, so we need to explicitly try connecting
Expand Down Expand Up @@ -270,44 +274,50 @@ class RingHashLoadBalancer implements LoadBalancer {
stateCounts[leaf.getConnectivityState()] += 1;
}
if (stateCounts[connectivityState.READY] > 0) {
this.updateState(connectivityState.READY, new RingHashPicker(this.ring));
this.updateState(connectivityState.READY, new RingHashPicker(this.ring), null);
// REPORT READY
} else if (stateCounts[connectivityState.TRANSIENT_FAILURE] > 1) {
const errorMessage = `ring hash: no connection established. Latest error: ${this.latestErrorMessage}`;
this.updateState(
connectivityState.TRANSIENT_FAILURE,
new UnavailablePicker()
new UnavailablePicker({details: errorMessage}),
errorMessage
);
} else if (stateCounts[connectivityState.CONNECTING] > 0) {
this.updateState(
connectivityState.CONNECTING,
new RingHashPicker(this.ring)
new RingHashPicker(this.ring),
null
);
} else if (
stateCounts[connectivityState.TRANSIENT_FAILURE] > 0 &&
this.leafMap.size > 1
) {
this.updateState(
connectivityState.CONNECTING,
new RingHashPicker(this.ring)
new RingHashPicker(this.ring),
null
);
} else if (stateCounts[connectivityState.IDLE] > 0) {
this.updateState(connectivityState.IDLE, new RingHashPicker(this.ring));
this.updateState(connectivityState.IDLE, new RingHashPicker(this.ring), null);
} else {
const errorMessage = `ring hash: no connection established. Latest error: ${this.latestErrorMessage}`;
this.updateState(
connectivityState.TRANSIENT_FAILURE,
new UnavailablePicker()
new UnavailablePicker({details: errorMessage}),
errorMessage
);
}
}

private updateState(newState: connectivityState, picker: Picker) {
private updateState(newState: connectivityState, picker: Picker, errorMessage: string | null) {
trace(
connectivityState[this.currentState] +
' -> ' +
connectivityState[newState]
);
this.currentState = newState;
this.channelControlHelper.updateState(newState, picker);
this.channelControlHelper.updateState(newState, picker, errorMessage);
}

private constructRing(
Expand Down
16 changes: 11 additions & 5 deletions packages/grpc-js-xds/src/load-balancer-weighted-target.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,18 +175,21 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {

constructor(private parent: WeightedTargetLoadBalancer, private name: string) {
this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, {
updateState: (connectivityState: ConnectivityState, picker: Picker) => {
this.updateState(connectivityState, picker);
updateState: (connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) => {
this.updateState(connectivityState, picker, errorMessage);
},
}));

this.picker = new QueuePicker(this.childBalancer);
}

private updateState(connectivityState: ConnectivityState, picker: Picker) {
private updateState(connectivityState: ConnectivityState, picker: Picker, errorMessage: string | null) {
trace('Target ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]);
this.connectivityState = connectivityState;
this.picker = picker;
if (errorMessage) {
this.parent.latestChildErrorMessage = errorMessage;
}
this.parent.maybeUpdateState();
}

Expand Down Expand Up @@ -242,6 +245,7 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {
*/
private targetList: string[] = [];
private updatesPaused = false;
private latestChildErrorMessage: string | null = null;

constructor(private channelControlHelper: ChannelControlHelper) {}

Expand Down Expand Up @@ -297,6 +301,7 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {
}

let picker: Picker;
let errorMessage: string | null = null;
switch (connectivityState) {
case ConnectivityState.READY:
picker = new WeightedTargetPicker(pickerList);
Expand All @@ -306,17 +311,18 @@ export class WeightedTargetLoadBalancer implements LoadBalancer {
picker = new QueuePicker(this);
break;
default:
const errorMessage = `weighted_target: all children report state TRANSIENT_FAILURE. Latest error: ${this.latestChildErrorMessage}`;
picker = new UnavailablePicker({
code: Status.UNAVAILABLE,
details: 'weighted_target: all children report state TRANSIENT_FAILURE',
details: errorMessage,
metadata: new Metadata()
});
}
trace(
'Transitioning to ' +
ConnectivityState[connectivityState]
);
this.channelControlHelper.updateState(connectivityState, picker);
this.channelControlHelper.updateState(connectivityState, picker, errorMessage);
}

updateAddressList(addressList: Endpoint[], lbConfig: TypedLoadBalancingConfig, options: ChannelOptions): void {
Expand Down
10 changes: 5 additions & 5 deletions packages/grpc-js-xds/src/load-balancer-xds-cluster-impl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,12 +241,12 @@ class XdsClusterImplBalancer implements LoadBalancer {
}
return new LocalitySubchannelWrapper(wrapperChild, statsObj);
},
updateState: (connectivityState, originalPicker) => {
updateState: (connectivityState, originalPicker, errorMessage) => {
if (this.latestConfig === null || this.latestClusterConfig === null || this.latestClusterConfig.children.type === 'aggregate' || !this.latestClusterConfig.children.endpoints) {
channelControlHelper.updateState(connectivityState, originalPicker);
channelControlHelper.updateState(connectivityState, originalPicker, errorMessage);
} else {
const picker = new XdsClusterImplPicker(originalPicker, getCallCounterMapKey(this.latestConfig.getCluster(), this.latestClusterConfig.cluster.edsServiceName), this.latestClusterConfig.cluster.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS, this.latestClusterConfig.children.endpoints.dropCategories, this.clusterDropStats);
channelControlHelper.updateState(connectivityState, picker);
channelControlHelper.updateState(connectivityState, picker, errorMessage);
}
}
}));
Expand All @@ -266,7 +266,7 @@ class XdsClusterImplBalancer implements LoadBalancer {
if (!maybeClusterConfig.success) {
this.latestClusterConfig = null;
this.childBalancer.destroy();
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error));
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker(maybeClusterConfig.error), maybeClusterConfig.error.details);
return;
}
const clusterConfig = maybeClusterConfig.value;
Expand All @@ -276,7 +276,7 @@ class XdsClusterImplBalancer implements LoadBalancer {
}
if (!clusterConfig.children.endpoints) {
this.childBalancer.destroy();
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({details: clusterConfig.children.resolutionNote}));
this.channelControlHelper.updateState(connectivityState.TRANSIENT_FAILURE, new UnavailablePicker({details: clusterConfig.children.resolutionNote}), clusterConfig.children.resolutionNote ?? null);

}
this.lastestEndpointList = endpointList;
Expand Down
Loading

0 comments on commit 5cd30ae

Please sign in to comment.