Add exponential backoff retries to send transactions

chainio/txmgr/simple.go

@@ -8,6 +8,8 @@ import (
 
 	"github.com/Layr-Labs/eigensdk-go/chainio/clients/wallet"
 	"github.com/Layr-Labs/eigensdk-go/logging"
+	"github.com/Layr-Labs/eigensdk-go/utils"
+	"github.com/cenkalti/backoff/v4"
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
@@ -91,6 +93,36 @@ func (m *SimpleTxManager) Send(
 	return receipt, nil
 }
 
+// SendWithRetry is used to send a transaction to the Ethereum node, same as Send but adding retry logic.
+// If the transaction fails, it will retry sending the transaction until it gets a receipt, using
+// **exponential backoff** with factor `multiplier`, starting with `initialInterval`.
+func (m *SimpleTxManager) SendWithRetry(
+	ctx context.Context,
+	tx *types.Transaction,
+	initialInterval time.Duration,
+	maxElapsedTime time.Duration,
+	multiplier float64,
+) (*types.Receipt, error) {
+	backoffConfig := backoff.NewExponentialBackOff(
+		backoff.WithInitialInterval(initialInterval),
+		backoff.WithMultiplier(multiplier),
+		backoff.WithMaxElapsedTime(maxElapsedTime),
+	)
+
+	sendAndWait := func() (*types.Receipt, error) {
+		r, err := m.send(ctx, tx)
+		if err != nil {
+			return nil, err
+		}
+		m.logger.Error("failed to send transaction", err)
+		m.logger.Debugf("waiting %f seconds for backoff", initialInterval.Seconds())
+
+		return m.waitForReceipt(ctx, r.TxHash.Hex())
+	}
+
+	return backoff.RetryWithData(sendAndWait, backoffConfig)
+}
+
 func (m *SimpleTxManager) send(ctx context.Context, tx *types.Transaction) (*types.Receipt, error) {
 	// Estimate gas and nonce
 	// can't print tx hash in logs because the tx changes below when we complete and sign it
@@ -139,7 +171,7 @@ func (m *SimpleTxManager) waitForReceipt(ctx context.Context, txID wallet.TxID)
 	for {
 		select {
 		case <-ctx.Done():
-			return nil, errors.Join(errors.New("Context done before tx was mined"), ctx.Err())
+			return nil, utils.WrapError(ctx.Err(), "context done before tx was mined")
 		case <-queryTicker.C:
 			if receipt := m.queryReceipt(ctx, txID); receipt != nil {
 				return receipt, nil

chainio/txmgr/simple_test.go

@@ -0,0 +1,130 @@
+package txmgr_test
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/Layr-Labs/eigensdk-go/chainio/clients/wallet"
+	"github.com/Layr-Labs/eigensdk-go/chainio/txmgr"
+	"github.com/Layr-Labs/eigensdk-go/signerv2"
+	"github.com/Layr-Labs/eigensdk-go/testutils"
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newTx() *types.Transaction {
+	return types.NewTx(&types.DynamicFeeTx{
+		ChainID:   big.NewInt(31337),
+		Nonce:     0,
+		GasTipCap: big.NewInt(1),
+		GasFeeCap: big.NewInt(1_000_000_000),
+		Gas:       21000,
+		To:        testutils.ZeroAddress(),
+		Value:     big.NewInt(1),
+	})
+}
+
+func TestSendWithRetryWithNoError(t *testing.T) {
+	// Test SendWithRetry with a non-failing transaction to verify normal behavior
+	ecdsaPrivateKey, err := crypto.HexToECDSA("ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80")
+	require.NoError(t, err)
+	anvilC, err := testutils.StartAnvilContainer("")
+	require.NoError(t, err)
+	anvilHttpEndpoint, err := anvilC.Endpoint(context.Background(), "http")
+	require.NoError(t, err)
+	logger := testutils.NewTestLogger()
+
+	ethHttpClient, err := ethclient.Dial(anvilHttpEndpoint)
+	require.NoError(t, err)
+
+	chainid, err := ethHttpClient.ChainID(context.Background())
+	require.NoError(t, err)
+
+	signerV2, addr, err := signerv2.SignerFromConfig(signerv2.Config{PrivateKey: ecdsaPrivateKey}, chainid)
+	require.NoError(t, err)
+
+	pkWallet, err := wallet.NewPrivateKeyWallet(ethHttpClient, signerV2, addr, logger)
+	require.NoError(t, err)
+
+	txMgr := txmgr.NewSimpleTxManager(pkWallet, ethHttpClient, logger, addr)
+
+	tx := newTx()
+	retryTimeout := 200 * time.Millisecond
+	maxElapsedTime := 2 * time.Second
+	multiplier := 1.5
+
+	_, err = txMgr.SendWithRetry(context.Background(), tx, retryTimeout, maxElapsedTime, multiplier)
+	require.NoError(t, err)
+}
+
+func TestSendWithRetryDoesBackoff(t *testing.T) {
+	// Test SendWithRetry using a FailingEthBackend to simulate errors when sending transactions
+	logger := testutils.NewTestLogger()
+	ethBackend := NewFailingEthBackend(3)
+
+	chainid := big.NewInt(31337)
+	ecdsaSk, _, err := testutils.NewEcdsaSkAndAddress()
+	require.NoError(t, err)
+
+	signerV2, addr, err := signerv2.SignerFromConfig(signerv2.Config{PrivateKey: ecdsaSk}, chainid)
+	require.NoError(t, err)
+
+	pkWallet, err := wallet.NewPrivateKeyWallet(ethBackend, signerV2, addr, logger)
+	require.NoError(t, err)
+
+	txMgr := txmgr.NewSimpleTxManager(pkWallet, ethBackend, logger, addr)
+
+	tx := newTx()
+	retryTimeout := 200 * time.Millisecond
+	maxElapsedTime := 3 * time.Second
+	multiplier := 1.5
+
+	_, err = txMgr.SendWithRetry(context.Background(), tx, retryTimeout, maxElapsedTime, multiplier)
+	require.NoError(t, err)
+	assert.Equal(t, ethBackend.pendingFailures, uint32(0))
+}
+
+// Mock of the EthBackend that returns an error when sending transactions.
+// Once pendingFailures reaches zero, SendTransaction will no longer fail
+type FailingEthBackend struct {
+	pendingFailures uint32
+}
+
+func NewFailingEthBackend(pendingFailures uint32) *FailingEthBackend {
+	backend := &FailingEthBackend{pendingFailures: pendingFailures}
+	return backend
+}
+
+func (s *FailingEthBackend) SendTransaction(ctx context.Context, tx *types.Transaction) error {
+	if s.pendingFailures == 0 {
+		return nil
+	}
+	s.pendingFailures--
+	return fmt.Errorf("did not send tx")
+}
+
+func (s *FailingEthBackend) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) {
+	return &types.Receipt{}, nil
+}
+
+func (s *FailingEthBackend) EstimateGas(ctx context.Context, msg ethereum.CallMsg) (uint64, error) {
+	return 0, nil
+}
+
+func (s *FailingEthBackend) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) {
+	return &types.Header{
+		BaseFee: big.NewInt(0),
+	}, nil
+}
+
+func (s *FailingEthBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) {
+	return big.NewInt(0), nil
+}

go.mod

@@ -53,7 +53,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
-	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/containerd/containerd v1.7.12 // indirect

go.sum

@@ -54,6 +54,8 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOF
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=