Lower integration tests memory usage

cmd/sonicd/app/launcher.go

@@ -65,6 +65,8 @@ func initFlags() {
 	}
 	performanceFlags = []cli.Flag{
 		flags.CacheFlag,
+		flags.LiveDbCacheFlag,
+		flags.ArchiveCacheFlag,
 	}
 	networkingFlags = []cli.Flag{
 		flags.BootnodesFlag,
@@ -222,6 +224,15 @@ func lachesisMain(ctx *cli.Context) error {
 	}
 
 	metrics.SetDataDir(cfg.Node.DataDir) // report disk space usage into metrics
+	liveCache := ctx.Int64(flags.LiveDbCacheFlag.Name)
+	if liveCache > 0 {
+		cfg.OperaStore.EVM.StateDb.LiveCache = liveCache
+	}
+
+	archiveCache := ctx.Int64(flags.ArchiveCacheFlag.Name)
+	if archiveCache > 0 {
+		cfg.OperaStore.EVM.StateDb.ArchiveCache = archiveCache
+	}
 
 	node, _, nodeClose, err := config.MakeNode(ctx, cfg)
 	if err != nil {

cmd/sonicd/app/run_test.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"fmt"
+	"github.com/Fantom-foundation/lachesis-base/utils/cachescale"
 	"os"
 	"strings"
 	"testing"
@@ -11,8 +12,6 @@ import (
 	"github.com/Fantom-foundation/go-opera/integration/makefakegenesis"
 	futils "github.com/Fantom-foundation/go-opera/utils"
 	"github.com/Fantom-foundation/lachesis-base/inter/idx"
-	"github.com/Fantom-foundation/lachesis-base/utils/cachescale"
-
 	"github.com/docker/docker/pkg/reexec"
 	"github.com/ethereum/go-ethereum/common"
 
@@ -26,7 +25,14 @@ func tmpdir(t *testing.T) string {
 func initFakenetDatadir(dataDir string, validatorsNum idx.Validator) {
 	genesisStore := makefakegenesis.FakeGenesisStore(validatorsNum, futils.ToFtm(1000000000), futils.ToFtm(5000000))
 	defer genesisStore.Close()
-	if err := genesis.ImportGenesisStore(genesisStore, dataDir, false, cachescale.Identity); err != nil {
+
+	if err := genesis.ImportGenesisStore(genesis.ImportParams{
+		GenesisStore: genesisStore,
+		DataDir:      dataDir,
+		CacheRatio:   cachescale.Identity,
+		LiveDbCache:  1, // Set lowest cache
+		ArchiveCache: 1, // Set lowest cache
+	}); err != nil {
 		panic(err)
 	}
 }

cmd/sonictool/app/chain.go

@@ -3,6 +3,7 @@ package app
 import (
 	"compress/gzip"
 	"fmt"
+	"github.com/Fantom-foundation/go-opera/cmd/sonictool/db"
 	"io"
 	"os"
 	"strconv"
@@ -57,8 +58,14 @@ func exportEvents(ctx *cli.Context) error {
 		to = idx.Epoch(n)
 	}
 
+	gdbParams := db.GossipDbParameters{
+		DataDir:      dataDir,
+		LiveDbCache:  ctx.Int64(flags.LiveDbCacheFlag.Name),
+		ArchiveCache: ctx.Int64(flags.ArchiveCacheFlag.Name),
+	}
+
 	log.Info("Exporting events to file", "file", fn)
-	err = chain.ExportEvents(writer, dataDir, from, to)
+	err = chain.ExportEvents(gdbParams, writer, dataDir, from, to)
 	if err != nil {
 		return fmt.Errorf("export error: %w", err)
 	}

cmd/sonictool/app/export_genesis.go

@@ -7,7 +7,6 @@ import (
 	"github.com/Fantom-foundation/go-opera/cmd/sonictool/genesis"
 	"github.com/Fantom-foundation/go-opera/config/flags"
 	"github.com/Fantom-foundation/go-opera/integration"
-	"github.com/Fantom-foundation/lachesis-base/utils/cachescale"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"gopkg.in/urfave/cli.v1"
 	"os"
@@ -48,7 +47,14 @@ func exportGenesis(ctx *cli.Context) error {
 	}
 	defer dbs.Close()
 
-	gdb, err := db.MakeGossipDb(dbs, dataDir, false, cachescale.Identity)
+	gdb, err := db.MakeGossipDb(db.GossipDbParameters{
+		Dbs:           dbs,
+		DataDir:       dataDir,
+		ValidatorMode: false,
+		CacheRatio:    cacheRatio,
+		LiveDbCache:   ctx.Int64(flags.LiveDbCacheFlag.Name),
+		ArchiveCache:  ctx.Int64(flags.ArchiveCacheFlag.Name),
+	})
 	if err != nil {
 		return err
 	}

cmd/sonictool/app/genesis.go

@@ -66,7 +66,14 @@ func gfileGenesisImport(ctx *cli.Context) error {
 			return fmt.Errorf("genesis file check failed: %w", err)
 		}
 	}
-	return genesis.ImportGenesisStore(genesisStore, dataDir, validatorMode, cacheRatio)
+	return genesis.ImportGenesisStore(genesis.ImportParams{
+		GenesisStore:  genesisStore,
+		DataDir:       dataDir,
+		ValidatorMode: validatorMode,
+		CacheRatio:    cacheRatio,
+		LiveDbCache:   ctx.Int64(flags.LiveDbCacheFlag.Name),
+		ArchiveCache:  ctx.Int64(flags.ArchiveCacheFlag.Name),
+	})
 }
 
 func jsonGenesisImport(ctx *cli.Context) error {
@@ -98,7 +105,14 @@ func jsonGenesisImport(ctx *cli.Context) error {
 		return fmt.Errorf("failed to prepare JSON genesis: %w", err)
 	}
 	defer genesisStore.Close()
-	return genesis.ImportGenesisStore(genesisStore, dataDir, validatorMode, cacheRatio)
+	return genesis.ImportGenesisStore(genesis.ImportParams{
+		GenesisStore:  genesisStore,
+		DataDir:       dataDir,
+		ValidatorMode: validatorMode,
+		CacheRatio:    cacheRatio,
+		LiveDbCache:   ctx.Int64(flags.LiveDbCacheFlag.Name),
+		ArchiveCache:  ctx.Int64(flags.ArchiveCacheFlag.Name),
+	})
 }
 
 func fakeGenesisImport(ctx *cli.Context) error {
@@ -127,7 +141,14 @@ func fakeGenesisImport(ctx *cli.Context) error {
 
 	genesisStore := makefakegenesis.FakeGenesisStore(idx.Validator(validatorsNumber), futils.ToFtm(1000000000), futils.ToFtm(5000000))
 	defer genesisStore.Close()
-	return genesis.ImportGenesisStore(genesisStore, dataDir, validatorMode, cacheRatio)
+	return genesis.ImportGenesisStore(genesis.ImportParams{
+		GenesisStore:  genesisStore,
+		DataDir:       dataDir,
+		ValidatorMode: validatorMode,
+		CacheRatio:    cacheRatio,
+		LiveDbCache:   ctx.Int64(flags.LiveDbCacheFlag.Name),
+		ArchiveCache:  ctx.Int64(flags.ArchiveCacheFlag.Name),
+	})
 }
 
 func isValidatorModeSet(ctx *cli.Context) (bool, error) {

cmd/sonictool/app/main.go

@@ -18,6 +18,8 @@ func Run() error {
 	app.Flags = []cli.Flag{
 		flags.DataDirFlag,
 		flags.CacheFlag,
+		flags.LiveDbCacheFlag,
+		flags.ArchiveCacheFlag,
 	}
 	app.Commands = []cli.Command{
 		{

cmd/sonictool/chain/export_events.go

@@ -24,15 +24,19 @@ var (
 // always print out progress. This avoids the user wondering what's going on.
 const statsReportLimit = 8 * time.Second
 
-func ExportEvents(w io.Writer, dataDir string, from, to idx.Epoch) (err error) {
+func ExportEvents(gdbParams db.GossipDbParameters, w io.Writer, dataDir string, from, to idx.Epoch) (err error) {
 	chaindataDir := filepath.Join(dataDir, "chaindata")
 	dbs, err := db.MakeDbProducer(chaindataDir, cachescale.Identity)
 	if err != nil {
 		return err
 	}
 	defer dbs.Close()
 
-	gdb, err := db.MakeGossipDb(dbs, dataDir, false, cachescale.Identity)
+	// Fill the rest of the params
+	gdbParams.Dbs = dbs
+	gdbParams.CacheRatio = cachescale.Identity
+
+	gdb, err := db.MakeGossipDb(gdbParams)
 	if err != nil {
 		return err
 	}

cmd/sonictool/db/dbutils.go

@@ -31,7 +31,7 @@ func makeDatabaseHandles() uint64 {
 	if err != nil {
 		panic(fmt.Errorf("failed to raise file descriptor allowance: %v", err))
 	}
-	return raised / 6 + 1
+	return raised/6 + 1
 }
 
 func AssertDatabaseNotInitialized(dataDir string) error {
@@ -61,16 +61,32 @@ func MakeDbProducer(chaindataDir string, cacheRatio cachescale.Func) (kvdb.FullD
 	})
 }
 
-func MakeGossipDb(dbs kvdb.FullDBProducer, dataDir string, validatorMode bool, cacheRatio cachescale.Func) (*gossip.Store, error) {
-	gdbConfig := gossip.DefaultStoreConfig(cacheRatio)
-	gdbConfig.EVM.StateDb.Directory = filepath.Join(dataDir, "carmen")
-	if validatorMode {
+// GossipDbParameters are parameters for GossipDb factory func.
+type GossipDbParameters struct {
+	Dbs                       kvdb.FullDBProducer
+	DataDir                   string
+	ValidatorMode             bool
+	CacheRatio                cachescale.Func
+	LiveDbCache, ArchiveCache int64 // in bytes
+}
+
+func MakeGossipDb(params GossipDbParameters) (*gossip.Store, error) {
+	gdbConfig := gossip.DefaultStoreConfig(params.CacheRatio)
+	if params.ValidatorMode {
 		gdbConfig.EVM.StateDb.Archive = carmen.NoArchive
 		gdbConfig.EVM.DisableLogsIndexing = true
 		gdbConfig.EVM.DisableTxHashesIndexing = true
 	}
 
-	gdb, err := gossip.NewStore(dbs, gdbConfig)
+	if params.LiveDbCache > 0 {
+		gdbConfig.EVM.StateDb.LiveCache = params.LiveDbCache
+	}
+	if params.ArchiveCache > 0 {
+		gdbConfig.EVM.StateDb.ArchiveCache = params.ArchiveCache
+	}
+	gdbConfig.EVM.StateDb.Directory = filepath.Join(params.DataDir, "carmen")
+
+	gdb, err := gossip.NewStore(params.Dbs, gdbConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create gossip store: %w", err)
 	}

cmd/sonictool/genesis/import.go

@@ -13,29 +13,45 @@ import (
 	"path/filepath"
 )
 
-func ImportGenesisStore(genesisStore *genesisstore.Store, dataDir string, validatorMode bool, cacheRatio cachescale.Func) error {
-	if err := db.AssertDatabaseNotInitialized(dataDir); err != nil {
+// ImportParams are parameters for ImportGenesisStore func.
+type ImportParams struct {
+	GenesisStore              *genesisstore.Store
+	DataDir                   string
+	ValidatorMode             bool
+	CacheRatio                cachescale.Func
+	LiveDbCache, ArchiveCache int64 // in bytes
+}
+
+func ImportGenesisStore(params ImportParams) error {
+	if err := db.AssertDatabaseNotInitialized(params.DataDir); err != nil {
 		return fmt.Errorf("database in datadir is already initialized: %w", err)
 	}
-	if err := db.RemoveDatabase(dataDir); err != nil {
+	if err := db.RemoveDatabase(params.DataDir); err != nil {
 		return fmt.Errorf("failed to remove existing data from the datadir: %w", err)
 	}
 
-	chaindataDir := filepath.Join(dataDir, "chaindata")
-	dbs, err := db.MakeDbProducer(chaindataDir, cacheRatio)
+	chaindataDir := filepath.Join(params.DataDir, "chaindata")
+	dbs, err := db.MakeDbProducer(chaindataDir, params.CacheRatio)
 	if err != nil {
 		return err
 	}
 	defer dbs.Close()
 	setGenesisProcessing(chaindataDir)
 
-	gdb, err := db.MakeGossipDb(dbs, dataDir, validatorMode, cacheRatio)
+	gdb, err := db.MakeGossipDb(db.GossipDbParameters{
+		Dbs:           dbs,
+		DataDir:       params.DataDir,
+		ValidatorMode: params.ValidatorMode,
+		CacheRatio:    params.CacheRatio,
+		LiveDbCache:   params.LiveDbCache,
+		ArchiveCache:  params.ArchiveCache,
+	})
 	if err != nil {
 		return err
 	}
 	defer gdb.Close()
 
-	err = gdb.ApplyGenesis(genesisStore.Genesis())
+	err = gdb.ApplyGenesis(params.GenesisStore.Genesis())
 	if err != nil {
 		return fmt.Errorf("failed to write Gossip genesis state: %v", err)
 	}
@@ -54,7 +70,7 @@ func ImportGenesisStore(genesisStore *genesisstore.Store, dataDir string, valida
 	abftCrit := func(err error) {
 		panic(fmt.Errorf("lachesis store error: %w", err))
 	}
-	cdb := abft.NewStore(cMainDb, cGetEpochDB, abftCrit, abft.DefaultStoreConfig(cacheRatio))
+	cdb := abft.NewStore(cMainDb, cGetEpochDB, abftCrit, abft.DefaultStoreConfig(params.CacheRatio))
 	defer cdb.Close()
 
 	err = cdb.ApplyGenesis(&abft.Genesis{

config/flags/flags.go

@@ -17,6 +17,7 @@
 package flags
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/Fantom-foundation/go-opera/evmcore"
@@ -348,4 +349,20 @@ var (
 		Name:  "lachesis.suppress-frame-panic",
 		Usage: "Suppress frame missmatch error (when testing on historical imported/synced events)",
 	}
+
+	// StateDb
+	LiveDbCacheFlag = cli.Int64Flag{
+		Name: "statedb.livecache",
+		Usage: fmt.Sprintf("Size of live db cache in bytes. Leaving this blank (which is generally recommended),"+
+			"or setting this to <1 will automatically allocate cache size depending on how much cache you use with %s."+
+			"Setting this value to <=2000 will result in pre-confugired cache capacity of 2KB", CacheFlag.Name),
+		Value: 0,
+	}
+	ArchiveCacheFlag = cli.IntFlag{
+		Name: "statedb.archivecache",
+		Usage: fmt.Sprintf("Size of archive cache in bytes. Leaving this blank (which is generally recommended),"+
+			"or setting this to <1 will automatically allocate cache size depending on how much cache you use with %s."+
+			"Setting this value to <=2000 will result in pre-confugired cache capacity of 2KB", CacheFlag.Name),
+		Value: 0,
+	}
 )

tests/integration_test_net.go

@@ -169,13 +169,20 @@ func startIntegrationTestNet(directory string, args []string) (*IntegrationTestN
 	// initialize the data directory for the single node on the test network
 	// equivalent to running `sonictool --datadir <dataDir> genesis fake 1`
 	originalArgs := os.Args
-	os.Args = append([]string{"sonictool", "--datadir", result.stateDir()}, args...)
+	os.Args = append([]string{
+		"sonictool",
+		"--datadir", result.stateDir(),
+		"--statedb.livecache", "1",
+		"--statedb.archivecache", "1",
+	}, args...)
 	if err := sonictool.Run(); err != nil {
 		os.Args = originalArgs
 		return nil, fmt.Errorf("failed to initialize the test network: %w", err)
 	}
 	os.Args = originalArgs
 
+	os.Args = originalArgs
+
 	if err := result.start(); err != nil {
 		return nil, fmt.Errorf("failed to start the test network: %w", err)
 	}
@@ -200,10 +207,17 @@ func (n *IntegrationTestNet) start() error {
 			"sonicd",
 			"--datadir", n.stateDir(),
 			"--fakenet", "1/1",
-			"--http", "--http.addr", "0.0.0.0", "--http.port", "18545",
+			"--http",
+			"--http.addr", "0.0.0.0",
+			"--http.port", "18545",
 			"--http.api", "admin,eth,web3,net,txpool,ftm,trace,debug",
-			"--ws", "--ws.addr", "0.0.0.0", "--ws.port", "18546", "--ws.api", "admin,eth,ftm",
+			"--ws",
+			"--ws.addr", "0.0.0.0",
+			"--ws.port", "18546",
+			"--ws.api", "admin,eth,ftm",
 			"--datadir.minfreedisk", "0",
+			"--statedb.livecache", "1",
+			"--statedb.archivecache", "1",
 		}
 		sonicd.Run()
 	}()