From 5a90b253ea6bd7e4882733bbb33e857394dbc8cd Mon Sep 17 00:00:00 2001 From: Ran Mishael Date: Thu, 7 Nov 2024 11:41:36 +0100 Subject: [PATCH 1/3] fix: PRT - provider node error handing fixed --- protocol/chainlib/jsonRPC.go | 8 ++++---- protocol/chainlib/tendermintRPC.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/protocol/chainlib/jsonRPC.go b/protocol/chainlib/jsonRPC.go index 99dbe2a116..8120f17f6a 100644 --- a/protocol/chainlib/jsonRPC.go +++ b/protocol/chainlib/jsonRPC.go @@ -671,14 +671,14 @@ func (cp *JrpcChainProxy) SendNodeMsg(ctx context.Context, ch chan interface{}, cp.NodeUrl.SetIpForwardingIfNecessary(ctx, rpc.SetHeader) rpcMessage, nodeErr = rpc.CallContext(connectCtx, nodeMessage.ID, nodeMessage.Method, nodeMessage.Params, true, nodeMessage.GetDisableErrorHandling()) - if err != nil { + if nodeErr != nil { // here we are getting an error for every code that is not 200-300 - if common.StatusCodeError504.Is(err) || common.StatusCodeError429.Is(err) || common.StatusCodeErrorStrict.Is(err) { - return nil, "", nil, utils.LavaFormatWarning("Received invalid status code", err, utils.Attribute{Key: "chainID", Value: cp.BaseChainProxy.ChainID}, utils.Attribute{Key: "apiName", Value: chainMessage.GetApi().Name}) + if common.StatusCodeError504.Is(nodeErr) || common.StatusCodeError429.Is(nodeErr) || common.StatusCodeErrorStrict.Is(nodeErr) { + return nil, "", nil, utils.LavaFormatWarning("Received invalid status code", nodeErr, utils.Attribute{Key: "chainID", Value: cp.BaseChainProxy.ChainID}, utils.Attribute{Key: "apiName", Value: chainMessage.GetApi().Name}) } // Validate if the error is related to the provider connection to the node or it is a valid error // in case the error is valid (e.g. bad input parameters) the error will return in the form of a valid error reply - if parsedError := cp.HandleNodeError(ctx, err); parsedError != nil { + if parsedError := cp.HandleNodeError(ctx, nodeErr); parsedError != nil { return nil, "", nil, parsedError } } diff --git a/protocol/chainlib/tendermintRPC.go b/protocol/chainlib/tendermintRPC.go index be0dae3648..0375d2099d 100644 --- a/protocol/chainlib/tendermintRPC.go +++ b/protocol/chainlib/tendermintRPC.go @@ -768,13 +768,13 @@ func (cp *tendermintRpcChainProxy) SendRPC(ctx context.Context, nodeMessage *rpc cp.NodeUrl.SetIpForwardingIfNecessary(ctx, rpc.SetHeader) // perform the rpc call rpcMessage, nodeErr = rpc.CallContext(connectCtx, nodeMessage.ID, nodeMessage.Method, nodeMessage.Params, false, nodeMessage.GetDisableErrorHandling()) - if err != nil { - if common.StatusCodeError504.Is(err) || common.StatusCodeError429.Is(err) || common.StatusCodeErrorStrict.Is(err) { - return nil, "", nil, utils.LavaFormatWarning("Received invalid status code", err, utils.Attribute{Key: "chainID", Value: cp.BaseChainProxy.ChainID}, utils.Attribute{Key: "apiName", Value: chainMessage.GetApi().Name}) + if nodeErr != nil { + if common.StatusCodeError504.Is(nodeErr) || common.StatusCodeError429.Is(nodeErr) || common.StatusCodeErrorStrict.Is(nodeErr) { + return nil, "", nil, utils.LavaFormatWarning("Received invalid status code", nodeErr, utils.Attribute{Key: "chainID", Value: cp.BaseChainProxy.ChainID}, utils.Attribute{Key: "apiName", Value: chainMessage.GetApi().Name}) } // Validate if the error is related to the provider connection to the node or it is a valid error // in case the error is valid (e.g. bad input parameters) the error will return in the form of a valid error reply - if parsedError := cp.HandleNodeError(ctx, err); parsedError != nil { + if parsedError := cp.HandleNodeError(ctx, nodeErr); parsedError != nil { return nil, "", nil, parsedError } } From eaeee67cc17d351e860a7cefaa82628dbc36ebd4 Mon Sep 17 00:00:00 2001 From: Ran Mishael Date: Thu, 7 Nov 2024 12:44:04 +0100 Subject: [PATCH 2/3] fix: attempt recovering json message from error body --- protocol/chainlib/jsonRPC.go | 21 +++++++++++---------- protocol/chainlib/node_error_handler.go | 13 +++++++++++++ protocol/chainlib/tendermintRPC.go | 17 ++++++++--------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/protocol/chainlib/jsonRPC.go b/protocol/chainlib/jsonRPC.go index 8120f17f6a..1fdf0f0524 100644 --- a/protocol/chainlib/jsonRPC.go +++ b/protocol/chainlib/jsonRPC.go @@ -650,7 +650,6 @@ func (cp *JrpcChainProxy) SendNodeMsg(ctx context.Context, ch chan interface{}, // Call our node var rpcMessage *rpcclient.JsonrpcMessage - var replyMessage *rpcInterfaceMessages.JsonrpcMessage var sub *rpcclient.ClientSubscription // support setting headers if len(nodeMessage.GetHeaders()) > 0 { @@ -684,28 +683,30 @@ func (cp *JrpcChainProxy) SendNodeMsg(ctx context.Context, ch chan interface{}, } } - var replyMsg rpcInterfaceMessages.JsonrpcMessage + var replyMsg *rpcInterfaceMessages.JsonrpcMessage // the error check here would only wrap errors not from the rpc - if nodeErr != nil { - utils.LavaFormatDebug("got error from node", utils.LogAttr("GUID", ctx), utils.LogAttr("nodeErr", nodeErr)) - return nil, "", nil, nodeErr + // try to parse node error as json message + rpcMessage = TryRecoverNodeErrorFromClientError(nodeErr) + if rpcMessage == nil { + utils.LavaFormatDebug("got error from node", utils.LogAttr("GUID", ctx), utils.LogAttr("nodeErr", nodeErr)) + return nil, "", nil, nodeErr + } } - replyMessage, err = rpcInterfaceMessages.ConvertJsonRPCMsg(rpcMessage) + replyMsg, err = rpcInterfaceMessages.ConvertJsonRPCMsg(rpcMessage) if err != nil { return nil, "", nil, utils.LavaFormatError("jsonRPC error", err, utils.Attribute{Key: "GUID", Value: ctx}) } // validate result is valid - if replyMessage.Error == nil { - responseIsNilValidationError := ValidateNilResponse(string(replyMessage.Result)) + if replyMsg.Error == nil { + responseIsNilValidationError := ValidateNilResponse(string(replyMsg.Result)) if responseIsNilValidationError != nil { return nil, "", nil, responseIsNilValidationError } } - replyMsg = *replyMessage - err = cp.ValidateRequestAndResponseIds(nodeMessage.ID, replyMessage.ID) + err = cp.ValidateRequestAndResponseIds(nodeMessage.ID, replyMsg.ID) if err != nil { return nil, "", nil, utils.LavaFormatError("jsonRPC ID mismatch error", err, utils.Attribute{Key: "GUID", Value: ctx}, diff --git a/protocol/chainlib/node_error_handler.go b/protocol/chainlib/node_error_handler.go index 9376d54644..0097753cda 100644 --- a/protocol/chainlib/node_error_handler.go +++ b/protocol/chainlib/node_error_handler.go @@ -139,6 +139,19 @@ func (geh *genericErrorHandler) ValidateRequestAndResponseIds(nodeMessageID json return nil } +func TryRecoverNodeErrorFromClientError(nodeErr error) *rpcclient.JsonrpcMessage { + // try to parse node error as json message + httpError, ok := nodeErr.(rpcclient.HTTPError) + if ok { + jsonMessage := &rpcclient.JsonrpcMessage{} + err := json.Unmarshal(httpError.Body, jsonMessage) + if err == nil { + return jsonMessage + } + } + return nil +} + type RestErrorHandler struct{ genericErrorHandler } // Validating if the error is related to the provider connection or not diff --git a/protocol/chainlib/tendermintRPC.go b/protocol/chainlib/tendermintRPC.go index 0375d2099d..431f323930 100644 --- a/protocol/chainlib/tendermintRPC.go +++ b/protocol/chainlib/tendermintRPC.go @@ -739,7 +739,6 @@ func (cp *tendermintRpcChainProxy) SendRPC(ctx context.Context, nodeMessage *rpc // create variables for the rpc message and reply message var rpcMessage *rpcclient.JsonrpcMessage - var replyMessage *rpcInterfaceMessages.RPCResponse var sub *rpcclient.ClientSubscription if len(nodeMessage.GetHeaders()) > 0 { for _, metadata := range nodeMessage.GetHeaders() { @@ -782,27 +781,27 @@ func (cp *tendermintRpcChainProxy) SendRPC(ctx context.Context, nodeMessage *rpc var replyMsg *rpcInterfaceMessages.RPCResponse // the error check here would only wrap errors not from the rpc - if nodeErr != nil { - utils.LavaFormatDebug("got error from node", utils.LogAttr("GUID", ctx), utils.LogAttr("nodeErr", nodeErr)) - return nil, "", nil, nodeErr + rpcMessage = TryRecoverNodeErrorFromClientError(nodeErr) + if rpcMessage == nil { + utils.LavaFormatDebug("got error from node", utils.LogAttr("GUID", ctx), utils.LogAttr("nodeErr", nodeErr)) + return nil, "", nil, nodeErr + } } - replyMessage, err = rpcInterfaceMessages.ConvertTendermintMsg(rpcMessage) + replyMsg, err = rpcInterfaceMessages.ConvertTendermintMsg(rpcMessage) if err != nil { return nil, "", nil, utils.LavaFormatError("tendermintRPC error", err) } // if we didn't get a node error. - if replyMessage.Error == nil { + if replyMsg.Error == nil { // validate result is valid - responseIsNilValidationError := ValidateNilResponse(string(replyMessage.Result)) + responseIsNilValidationError := ValidateNilResponse(string(replyMsg.Result)) if responseIsNilValidationError != nil { return nil, "", nil, responseIsNilValidationError } } - replyMsg = replyMessage - err = cp.ValidateRequestAndResponseIds(nodeMessage.ID, rpcMessage.ID) if err != nil { return nil, "", nil, utils.LavaFormatError("tendermintRPC ID mismatch error", err, From 75b819a578b752f7d3e8a4fae8cba8f781f243dc Mon Sep 17 00:00:00 2001 From: Ran Mishael Date: Thu, 7 Nov 2024 12:45:34 +0100 Subject: [PATCH 3/3] add logs --- protocol/chainlib/node_error_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/protocol/chainlib/node_error_handler.go b/protocol/chainlib/node_error_handler.go index 0097753cda..5d5d52bf4c 100644 --- a/protocol/chainlib/node_error_handler.go +++ b/protocol/chainlib/node_error_handler.go @@ -146,6 +146,7 @@ func TryRecoverNodeErrorFromClientError(nodeErr error) *rpcclient.JsonrpcMessage jsonMessage := &rpcclient.JsonrpcMessage{} err := json.Unmarshal(httpError.Body, jsonMessage) if err == nil { + utils.LavaFormatDebug("Successfully recovered HTTPError to node message", utils.LogAttr("jsonMessage", jsonMessage)) return jsonMessage } }