Identify recoverable errors and add retry mechanisms for failures

[m-Peter]

Related discord thread: https://discord.com/channels/613813861610684416/1167476806333513800/1317243233431982120

[bjartek]

grpc retry could be used to just do a general retry.

this is some example code we use at .find

func CreateFlowClientFromConfig(cfg Config, logger *zap.Logger, opt ...overflow.OverflowOption) (overflow.OverflowClient, error) {
	interceptors := []grpc.UnaryClientInterceptor{}

	// if we have grpc logging enable we add the grpc logging interceptor
	if cfg.FlowGrpcLogging {

		opts := []logging.Option{
			logging.WithLogOnEvents(logging.StartCall, logging.FinishCall),
			logging.WithFieldsFromContext(logging.ExtractFields),
		}

		unaryLoggingInterceptor := logging.UnaryClientInterceptor(InterceptorLogger(logger), opts...)
		interceptors = append(interceptors, unaryLoggingInterceptor)
	}

	// if we have retry set to anything greater then 0 then we retry grpc calls
	if cfg.FlowGrpcRetryMax > 0 {

		retryCallback := func(_ context.Context, attempt uint, err error) {
			logger.Debug("retry", zap.Uint("attempt", attempt), zap.Error(err))
		}

		opts := []retry.CallOption{
			retry.WithMax(cfg.FlowGrpcRetryMax),
			retry.WithPerRetryTimeout(cfg.FlowGrpcRetryTimeout),
			retry.WithBackoff(retry.BackoffExponential(cfg.FlowGrpcRetryExponentialBackoff)),
			retry.WithOnRetryCallback(retryCallback),
		}

		retryInterceptor := retry.UnaryClientInterceptor(opts...)
		interceptors = append(interceptors, retryInterceptor)
	}

	if cfg.FlowGrpcGlobalTimeout > 0 {
		timeoutInterceptor := timeout.UnaryClientInterceptor(cfg.FlowGrpcGlobalTimeout)
		interceptors = append(interceptors, timeoutInterceptor)
	}

	overflowOptions := []overflow.OverflowOption{
		overflow.WithNetwork(cfg.FlowNetwork),
		overflow.WithNetworkHost(cfg.FlowAccessNode),
		overflow.WithLogNone(),
		overflow.WithReturnErrors(),
		overflow.WithEmbedFS(efs),
		overflow.WithBasePath(""),
	}

	if len(interceptors) > 0 {
		unaryOpt := grpc.WithChainUnaryInterceptor(interceptors...)
		overflowOptions = append(overflowOptions, overflow.WithGrpcDialOption(unaryOpt))
	}

	overflowOptions = append(overflowOptions, opt...)

	o := overflow.Overflow(overflowOptions...)
	if o.Error != nil {
		return nil, o.Error
	}
	return o, nil
}