Implement a Keystore for handling signing of multiple transactions co…

…ncurrently

bootstrap/bootstrap.go

@@ -11,7 +11,6 @@ import (
 	"github.com/onflow/flow-evm-gateway/metrics"
 	"github.com/onflow/flow-go-sdk/access"
 	"github.com/onflow/flow-go-sdk/access/grpc"
-	"github.com/onflow/flow-go-sdk/crypto"
 	"github.com/onflow/flow-go/fvm/environment"
 	"github.com/onflow/flow-go/fvm/evm"
 	flowGo "github.com/onflow/flow-go/model/flow"
@@ -61,6 +60,7 @@ type Bootstrap struct {
 	events     *ingestion.Engine
 	profiler   *api.ProfileServer
 	db         *pebbleDB.DB
+	keystore   *requester.Keystore
 }
 
 func New(config config.Config) (*Bootstrap, error) {
@@ -131,6 +131,7 @@ func (b *Bootstrap) StartEventIngestion(ctx context.Context) error {
 		b.logger,
 		b.client,
 		chainID,
+		b.keystore,
 		latestCadenceHeight,
 	)
 
@@ -184,28 +185,6 @@ func (b *Bootstrap) StartAPIServer(ctx context.Context) error {
 
 	b.server = api.NewServer(b.logger, b.collector, b.config)
 
-	// create the signer based on either a single coa key being provided and using a simple in-memory
-	// signer, or multiple keys being provided and using signer with key-rotation mechanism.
-	var signer crypto.Signer
-	var err error
-	switch {
-	case b.config.COAKey != nil:
-		signer, err = crypto.NewInMemorySigner(b.config.COAKey, crypto.SHA3_256)
-	case b.config.COAKeys != nil:
-		signer, err = requester.NewKeyRotationSigner(b.config.COAKeys, crypto.SHA3_256)
-	case len(b.config.COACloudKMSKeys) > 0:
-		signer, err = requester.NewKMSKeyRotationSigner(
-			ctx,
-			b.config.COACloudKMSKeys,
-			b.logger,
-		)
-	default:
-		return fmt.Errorf("must provide either single COA / keylist of COA keys / COA cloud KMS keys")
-	}
-	if err != nil {
-		return fmt.Errorf("failed to create a COA signer: %w", err)
-	}
-
 	// create transaction pool
 	txPool := requester.NewTxPool(
 		b.client,
@@ -220,16 +199,39 @@ func (b *Bootstrap) StartAPIServer(ctx context.Context) error {
 		nil,
 	)
 
+	accountKeys := make([]*requester.AccountKey, 0)
+	account, err := b.client.GetAccount(ctx, b.config.COAAddress)
+	if err != nil {
+		return fmt.Errorf(
+			"failed to get signer info account for address: %s, with: %w",
+			b.config.COAAddress,
+			err,
+		)
+	}
+	signer, err := createSigner(ctx, b.config, b.logger)
+	if err != nil {
+		return err
+	}
+	for _, key := range account.Keys {
+		accountKeys = append(accountKeys, &requester.AccountKey{
+			AccountKey: *key,
+			Address:    b.config.COAAddress,
+			Signer:     signer,
+		})
+	}
+
+	b.keystore = requester.NewKeystore(accountKeys)
+
 	evm, err := requester.NewEVM(
 		b.storages.Registers,
 		blocksProvider,
 		b.client,
 		b.config,
-		signer,
 		b.logger,
 		b.storages.Blocks,
 		txPool,
 		b.collector,
+		b.keystore,
 	)
 	if err != nil {
 		return fmt.Errorf("failed to create EVM requester: %w", err)

bootstrap/create-multi-key-account.go

@@ -48,16 +48,13 @@ func RunCreateMultiKeyAccount() {
 		panic(err)
 	}
 
-	address, keys, err := CreateMultiKeyAccount(client, keyCount, payer, ftFlag, flowFlag, key)
+	address, privateKey, err := CreateMultiKeyAccount(client, keyCount, payer, ftFlag, flowFlag, key)
 	if err != nil {
 		panic(err)
 	}
 
 	fmt.Println("Address: ", address.Hex())
-	fmt.Println("Keys:")
-	for _, pk := range keys {
-		fmt.Println(pk.String())
-	}
+	fmt.Println("Key: ", privateKey.String())
 }
 
 /*
@@ -71,35 +68,29 @@ func CreateMultiKeyAccount(
 	ftAddress string,
 	flowAddress string,
 	key crypto.PrivateKey,
-) (*flow.Address, []crypto.PrivateKey, error) {
-
-	privKeys := make([]*flow.AccountKey, keyCount)
-	pks := make([]crypto.PrivateKey, keyCount)
+) (*flow.Address, crypto.PrivateKey, error) {
+	accountKeys := make([]*flow.AccountKey, keyCount)
+	seed := make([]byte, crypto.MinSeedLength)
+	_, err := rand.Read(seed)
+	if err != nil {
+		return nil, nil, err
+	}
+	privateKey, err := crypto.GeneratePrivateKey(crypto.ECDSA_P256, seed)
+	if err != nil {
+		return nil, nil, err
+	}
 	for i := 0; i < keyCount; i++ {
-		seed := make([]byte, crypto.MinSeedLength)
-		_, err := rand.Read(seed)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		pk, err := crypto.GeneratePrivateKey(crypto.ECDSA_P256, seed)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		pks[i] = pk
-		privKeys[i] = &flow.AccountKey{
+		accountKeys[i] = &flow.AccountKey{
 			Index:     uint32(i),
-			PublicKey: pk.PublicKey(),
+			PublicKey: privateKey.PublicKey(),
 			SigAlgo:   crypto.ECDSA_P256,
 			HashAlgo:  crypto.SHA3_256,
 			Weight:    1000,
 		}
 	}
 
-	var err error
 	keyList := make([]cadence.Value, keyCount)
-	for i, key := range privKeys {
+	for i, key := range accountKeys {
 		keyList[i], err = templates.AccountKeyToCadenceCryptoKey(key)
 		if err != nil {
 			return nil, nil, err
@@ -197,7 +188,7 @@ func CreateMultiKeyAccount(
 	events := eventsFromTx(res)
 	createdAddrs := events.GetCreatedAddresses()
 
-	return createdAddrs[0], pks, nil
+	return createdAddrs[0], privateKey, nil
 }
 
 func CreateMultiCloudKMSKeysAccount(

bootstrap/utils.go

@@ -0,0 +1,40 @@
+package bootstrap
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/onflow/flow-evm-gateway/config"
+	"github.com/onflow/flow-evm-gateway/services/requester"
+	"github.com/onflow/flow-go-sdk/crypto"
+	"github.com/rs/zerolog"
+)
+
+// createSigner creates the signer based on either a single coa key being
+// provided and using a simple in-memory signer, or a Cloud KMS key being
+// provided and using a Cloud KMS signer.
+func createSigner(
+	ctx context.Context,
+	config config.Config,
+	logger zerolog.Logger,
+) (crypto.Signer, error) {
+	var signer crypto.Signer
+	var err error
+	switch {
+	case config.COAKey != nil:
+		signer, err = crypto.NewInMemorySigner(config.COAKey, crypto.SHA3_256)
+	case config.COACloudKMSKey != nil:
+		signer, err = requester.NewKMSKeySigner(
+			ctx,
+			*config.COACloudKMSKey,
+			logger,
+		)
+	default:
+		return nil, fmt.Errorf("must provide either single COA / Cloud KMS key")
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to create a COA signer: %w", err)
+	}
+
+	return signer, nil
+}

cmd/run/cmd.go

@@ -2,7 +2,6 @@ package run
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"math/big"
@@ -120,50 +119,24 @@ func parseConfigFromFlags() error {
 			return fmt.Errorf("invalid COA private key: %w", err)
 		}
 		cfg.COAKey = pkey
-	} else if keysPath != "" {
-		raw, err := os.ReadFile(keysPath)
-		if err != nil {
-			return fmt.Errorf("could not read the file containing list of keys for key-rotation mechanism, check if coa-key-file specifies valid path: %w", err)
-		}
-		var keysJSON []string
-		if err := json.Unmarshal(raw, &keysJSON); err != nil {
-			return fmt.Errorf("could not parse file containing the list of keys for key-rotation, make sure keys are in JSON array format: %w", err)
-		}
-
-		cfg.COAKeys = make([]crypto.PrivateKey, len(keysJSON))
-		sigAlgo := crypto.StringToSignatureAlgorithm(keyAlg)
-		if sigAlgo == crypto.UnknownSignatureAlgorithm {
-			return fmt.Errorf("invalid signature algorithm: %s", keyAlg)
-		}
-		for i, k := range keysJSON {
-			pk, err := crypto.DecodePrivateKeyHex(sigAlgo, k)
-			if err != nil {
-				return fmt.Errorf("a key from the COA key list file is not valid, key %s, error: %w", k, err)
-			}
-			cfg.COAKeys[i] = pk
-		}
-	} else if cloudKMSKeys != "" {
+	} else if cloudKMSKey != "" {
 		if cloudKMSProjectID == "" || cloudKMSLocationID == "" || cloudKMSKeyRingID == "" {
 			return fmt.Errorf(
 				"using coa-cloud-kms-keys requires also coa-cloud-kms-project-id & coa-cloud-kms-location-id & coa-cloud-kms-key-ring-id",
 			)
 		}
 
-		kmsKeys := strings.Split(cloudKMSKeys, ",")
-		cfg.COACloudKMSKeys = make([]flowGoKMS.Key, len(kmsKeys))
-		for i, key := range kmsKeys {
-			// key has the form "{keyID}@{keyVersion}"
-			keyParts := strings.Split(key, "@")
-			if len(keyParts) != 2 {
-				return fmt.Errorf("wrong format for Cloud KMS key: %s", key)
-			}
-			cfg.COACloudKMSKeys[i] = flowGoKMS.Key{
-				ProjectID:  cloudKMSProjectID,
-				LocationID: cloudKMSLocationID,
-				KeyRingID:  cloudKMSKeyRingID,
-				KeyID:      keyParts[0],
-				KeyVersion: keyParts[1],
-			}
+		// key has the form "{keyID}@{keyVersion}"
+		keyParts := strings.Split(cloudKMSKey, "@")
+		if len(keyParts) != 2 {
+			return fmt.Errorf("wrong format for Cloud KMS key: %s", key)
+		}
+		cfg.COACloudKMSKey = &flowGoKMS.Key{
+			ProjectID:  cloudKMSProjectID,
+			LocationID: cloudKMSLocationID,
+			KeyRingID:  cloudKMSKeyRingID,
+			KeyID:      keyParts[0],
+			KeyVersion: keyParts[1],
 		}
 	} else {
 		return fmt.Errorf(
@@ -259,13 +232,12 @@ var (
 	coa,
 	key,
 	keyAlg,
-	keysPath,
 	flowNetwork,
 	logLevel,
 	logWriter,
 	filterExpiry,
 	accessSporkHosts,
-	cloudKMSKeys,
+	cloudKMSKey,
 	cloudKMSProjectID,
 	cloudKMSLocationID,
 	cloudKMSKeyRingID,
@@ -293,7 +265,6 @@ func init() {
 	Cmd.Flags().StringVar(&coa, "coa-address", "", "Flow address that holds COA account used for submitting transactions")
 	Cmd.Flags().StringVar(&key, "coa-key", "", "Private key value for the COA address used for submitting transactions")
 	Cmd.Flags().StringVar(&keyAlg, "coa-key-alg", "ECDSA_P256", "Private key algorithm for the COA private key, only effective if coa-key/coa-key-file is present. Available values (ECDSA_P256 / ECDSA_secp256k1 / BLS_BLS12_381), defaults to ECDSA_P256.")
-	Cmd.Flags().StringVar(&keysPath, "coa-key-file", "", "File path that contains JSON array of COA keys used in key-rotation mechanism, this is exclusive with coa-key flag.")
 	Cmd.Flags().StringVar(&logLevel, "log-level", "debug", "Define verbosity of the log output ('debug', 'info', 'warn', 'error', 'fatal', 'panic')")
 	Cmd.Flags().StringVar(&logWriter, "log-writer", "stderr", "Log writer used for output ('stderr', 'console')")
 	Cmd.Flags().Float64Var(&cfg.StreamLimit, "stream-limit", 10, "Rate-limits the events sent to the client within one second")
@@ -305,7 +276,7 @@ func init() {
 	Cmd.Flags().StringVar(&cloudKMSProjectID, "coa-cloud-kms-project-id", "", "The project ID containing the KMS keys, e.g. 'flow-evm-gateway'")
 	Cmd.Flags().StringVar(&cloudKMSLocationID, "coa-cloud-kms-location-id", "", "The location ID where the key ring is grouped into, e.g. 'global'")
 	Cmd.Flags().StringVar(&cloudKMSKeyRingID, "coa-cloud-kms-key-ring-id", "", "The key ring ID where the KMS keys exist, e.g. 'tx-signing'")
-	Cmd.Flags().StringVar(&cloudKMSKeys, "coa-cloud-kms-keys", "", `Names of the KMS keys and their versions as a comma separated list, e.g. "gw-key-6@1,gw-key-7@1,gw-key-8@1"`)
+	Cmd.Flags().StringVar(&cloudKMSKey, "coa-cloud-kms-key", "", `Name of the KMS key and its version, e.g. "gw-key-6@1"`)
 	Cmd.Flags().StringVar(&walletKey, "wallet-api-key", "", "ECDSA private key used for wallet APIs. WARNING: This should only be used locally or for testing, never in production.")
 	Cmd.Flags().IntVar(&cfg.MetricsPort, "metrics-port", 9091, "Port for the metrics server")
 	Cmd.Flags().BoolVar(&cfg.IndexOnly, "index-only", false, "Run the gateway in index-only mode which only allows querying the state and indexing, but disallows sending transactions.")

config/config.go

@@ -54,10 +54,8 @@ type Config struct {
 	COAAddress flow.Address
 	// COAKey is Flow key to the COA account. WARNING: do not use in production
 	COAKey crypto.PrivateKey
-	// COAKeys is a slice of all the keys that will be used in key-rotation mechanism.
-	COAKeys []crypto.PrivateKey
-	// COACloudKMSKeys is a slice of all the keys and their versions that will be used in Cloud KMS key-rotation mechanism.
-	COACloudKMSKeys []flowGoKMS.Key
+	// COACloudKMSKey is a Cloud KMS key that will be used for signing transactions.
+	COACloudKMSKey *flowGoKMS.Key
 	// GasPrice is a fixed gas price that will be used when submitting transactions.
 	GasPrice *big.Int
 	// InitCadenceHeight is used for initializing the database on a local emulator or a live network.

services/ingestion/event_subscriber.go

@@ -33,9 +33,10 @@ var _ EventSubscriber = &RPCEventSubscriber{}
 type RPCEventSubscriber struct {
 	logger zerolog.Logger
 
-	client *requester.CrossSporkClient
-	chain  flowGo.ChainID
-	height uint64
+	client   *requester.CrossSporkClient
+	chain    flowGo.ChainID
+	keystore *requester.Keystore
+	height   uint64
 
 	recovery        bool
 	recoveredEvents []flow.Event
@@ -45,15 +46,17 @@ func NewRPCEventSubscriber(
 	logger zerolog.Logger,
 	client *requester.CrossSporkClient,
 	chainID flowGo.ChainID,
+	keystore *requester.Keystore,
 	startHeight uint64,
 ) *RPCEventSubscriber {
 	logger = logger.With().Str("component", "subscriber").Logger()
 	return &RPCEventSubscriber{
 		logger: logger,
 
-		client: client,
-		chain:  chainID,
-		height: startHeight,
+		client:   client,
+		chain:    chainID,
+		keystore: keystore,
+		height:   startHeight,
 	}
 }
 
@@ -169,6 +172,9 @@ func (r *RPCEventSubscriber) subscribe(ctx context.Context, height uint64) <-cha
 						continue
 					}
 				}
+				for _, evt := range blockEvents.Events {
+					r.keystore.UnlockKey(evt.TransactionID)
+				}
 
 				eventsChan <- evmEvents