Load extensions into sqlite

cmd/debugger/main.go

@@ -42,7 +42,10 @@ func main() {
 	etherscanClient := etherscan.NewEtherscanClient(cfg, l)
 	client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l)
 
-	db := sqlite.NewSqlite(cfg.GetSqlitePath(), l)
+	db := sqlite.NewSqlite(&sqlite.SqliteConfig{
+		Path:           cfg.GetSqlitePath(),
+		ExtensionsPath: cfg.SqliteConfig.ExtensionsPath,
+	}, l)
 
 	grm, err := sqlite.NewGormSqliteFromSqlite(db)
 	if err != nil {

cmd/root.go

@@ -36,6 +36,7 @@ func init() {
 
 	rootCmd.PersistentFlags().Bool("sqlite.in-memory", false, `"true" or "false"`)
 	rootCmd.PersistentFlags().String("sqlite.db-file-path", "", `e.g. "/tmp/sidecar.db"`)
+	rootCmd.PersistentFlags().StringArray("sqlite.extensions-path", []string{}, `e.g. "./sqlite-extensions"`)
 
 	rootCmd.PersistentFlags().Int("rpc.grpc-port", 7100, `e.g. 7100`)
 	rootCmd.PersistentFlags().Int("rpc.http-port", 7101, `e.g. 7101`)

cmd/run.go

@@ -59,7 +59,10 @@ var runCmd = &cobra.Command{
 			}
 		}
 
-		db := sqlite.NewSqlite(cfg.GetSqlitePath(), l)
+		db := sqlite.NewSqlite(&sqlite.SqliteConfig{
+			Path:           cfg.GetSqlitePath(),
+			ExtensionsPath: cfg.SqliteConfig.ExtensionsPath,
+		}, l)
 
 		grm, err := sqlite.NewGormSqliteFromSqlite(db)
 		if err != nil {

cmd/sidecar/main.go

@@ -45,7 +45,10 @@ func main() {
 	etherscanClient := etherscan.NewEtherscanClient(cfg, l)
 	client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l)
 
-	db := sqlite.NewSqlite(cfg.GetSqlitePath(), l)
+	db := sqlite.NewSqlite(&sqlite.SqliteConfig{
+		Path:           cfg.GetSqlitePath(),
+		ExtensionsPath: cfg.SqliteConfig.ExtensionsPath,
+	}, l)
 
 	grm, err := sqlite.NewGormSqliteFromSqlite(db)
 	if err != nil {

internal/config/config.go

@@ -69,8 +69,9 @@ type EtherscanConfig struct {
 }
 
 type SqliteConfig struct {
-	InMemory   bool
-	DbFilePath string
+	InMemory       bool
+	DbFilePath     string
+	ExtensionsPath []string
 }
 
 type RpcConfig struct {
@@ -108,7 +109,8 @@ func NewConfig() *Config {
 		},
 
 		SqliteConfig: SqliteConfig{
-			InMemory: viper.GetBool(normalizeFlagName("sqlite.in_memory")),
+			InMemory:       viper.GetBool(normalizeFlagName("sqlite.in_memory")),
+			ExtensionsPath: parseListEnvVar(viper.GetString(normalizeFlagName("sqlite.extensions_path"))),
 		},
 
 		RpcConfig: RpcConfig{

internal/contractStore/sqliteContractStore/sqliteContractStore_test.go

@@ -6,6 +6,7 @@ import (
 	"github.com/Layr-Labs/go-sidecar/internal/logger"
 	"github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations"
 	"github.com/Layr-Labs/go-sidecar/internal/tests"
+	"github.com/Layr-Labs/go-sidecar/internal/tests/sqlite"
 	"github.com/stretchr/testify/assert"
 	"go.uber.org/zap"
 	"gorm.io/gorm"
@@ -22,7 +23,7 @@ func setup() (
 	cfg := tests.GetConfig()
 	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
-	db, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	db, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/eigenState/avsOperators/avsOperators_test.go

@@ -25,7 +25,7 @@ func setup() (
 	cfg := tests.GetConfig()
 	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
-	db, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	db, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/eigenState/eigenstate_test.go

@@ -23,7 +23,7 @@ func setup() (
 	cfg := tests.GetConfig()
 	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
-	db, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	db, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/eigenState/operatorShares/operatorShares_test.go

@@ -27,7 +27,7 @@ func setup() (
 	cfg := tests.GetConfig()
 	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
-	db, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	db, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/eigenState/rewardSubmissions/rewardSubmissions_test.go

@@ -27,7 +27,7 @@ func setup() (
 	cfg := tests.GetConfig()
 	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
-	db, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	db, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/eigenState/stakerDelegations/stakerDelegations_test.go

@@ -24,7 +24,7 @@ func setup() (
 	cfg := tests.GetConfig()
 	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
-	db, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	db, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/eigenState/stakerShares/stakerShares_test.go

@@ -27,7 +27,7 @@ func setup() (
 	cfg := tests.GetConfig()
 	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
-	db, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	db, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/eigenState/stateManager/stateManager_test.go

@@ -22,7 +22,7 @@ func setup() (
 	cfg := tests.GetConfig()
 	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
-	db, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	db, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/eigenState/submittedDistributionRoots/submittedDistributionRoots_test.go

@@ -24,7 +24,7 @@ func setup() (
 	cfg := tests.GetConfig()
 	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
-	db, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	db, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/indexer/restakedStrategies_test.go

@@ -41,7 +41,7 @@ func setup() (
 	cfg := tests.GetConfig()
 	l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
-	db, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	db, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/pipeline/pipeline_integration_test.go

@@ -62,7 +62,7 @@ func setup() (
 	client := ethereum.NewClient(rpcUrl, l)
 
 	// database
-	grm, err := tests.GetInMemorySqliteDatabaseConnection(l)
+	grm, err := sqlite.GetInMemorySqliteDatabaseConnection(l)
 	if err != nil {
 		panic(err)
 	}

internal/sqlite/sqlite.go

@@ -70,18 +70,15 @@ var hasRegisteredExtensions = false
 
 const SqliteInMemoryPath = "file::memory:?cache=shared"
 
-func NewInMemorySqlite(l *zap.Logger) gorm.Dialector {
-	return NewSqlite(SqliteInMemoryPath, l)
+type SqliteConfig struct {
+	Path           string
+	ExtensionsPath []string
 }
 
-func NewInMemorySqliteWithName(name string, l *zap.Logger) gorm.Dialector {
-	path := fmt.Sprintf("file:%s?mode=memory&cache=shared", name)
-	return NewSqlite(path, l)
-}
-
-func NewSqlite(path string, l *zap.Logger) gorm.Dialector {
+func NewSqlite(cfg *SqliteConfig, l *zap.Logger) gorm.Dialector {
 	if !hasRegisteredExtensions {
 		sql.Register("sqlite3_with_extensions", &goSqlite.SQLiteDriver{
+			Extensions: cfg.ExtensionsPath,
 			ConnectHook: func(conn *goSqlite.SQLiteConn) error {
 				// Generic functions
 				if err := conn.RegisterAggregator("sum_big", NewSumBigNumbers, true); err != nil {
@@ -115,10 +112,10 @@ func NewSqlite(path string, l *zap.Logger) gorm.Dialector {
 					l.Sugar().Errorw("Failed to register function PostNileTokensPerDay", "error", err)
 					return err
 				}
-				if err := conn.RegisterFunc("pre_nile_tokens_per_day", numbers.PreNileTokensPerDay, true); err != nil {
-					l.Sugar().Errorw("Failed to register function PreNileTokensPerDay", "error", err)
-					return err
-				}
+				// if err := conn.RegisterFunc("pre_nile_tokens_per_day", numbers.PreNileTokensPerDay, true); err != nil {
+				// 	l.Sugar().Errorw("Failed to register function PreNileTokensPerDay", "error", err)
+				// 	return err
+				// }
 				// Staker proportion and weight
 				if err := conn.RegisterFunc("calc_staker_proportion", numbers.CalculateStakerProportion, true); err != nil {
 					l.Sugar().Errorw("Failed to register function CalculateStakerProportion", "error", err)
@@ -129,27 +126,27 @@ func NewSqlite(path string, l *zap.Logger) gorm.Dialector {
 					return err
 				}
 				// Forked rewards for stakers
-				if err := conn.RegisterFunc("amazon_token_rewards", numbers.CalculateAmazonStakerTokenRewards, true); err != nil {
-					l.Sugar().Errorw("Failed to register function CalculateAmazonStakerTokenRewards", "error", err)
-					return err
-				}
-				if err := conn.RegisterFunc("nile_token_rewards", numbers.CalculateNileStakerTokenRewards, true); err != nil {
-					l.Sugar().Errorw("Failed to register function CalculateNileStakerTokenRewards", "error", err)
-					return err
-				}
-				if err := conn.RegisterFunc("post_nile_token_rewards", numbers.CalculatePostNileStakerTokenRewards, true); err != nil {
-					l.Sugar().Errorw("Failed to register function CalculatePostNileStakerTokenRewards", "error", err)
-					return err
-				}
+				//if err := conn.RegisterFunc("amazon_token_rewards", numbers.CalculateAmazonStakerTokenRewards, true); err != nil {
+				//	l.Sugar().Errorw("Failed to register function CalculateAmazonStakerTokenRewards", "error", err)
+				//	return err
+				//}
+				//if err := conn.RegisterFunc("nile_token_rewards", numbers.CalculateNileStakerTokenRewards, true); err != nil {
+				//	l.Sugar().Errorw("Failed to register function CalculateNileStakerTokenRewards", "error", err)
+				//	return err
+				//}
+				//if err := conn.RegisterFunc("post_nile_token_rewards", numbers.CalculatePostNileStakerTokenRewards, true); err != nil {
+				//	l.Sugar().Errorw("Failed to register function CalculatePostNileStakerTokenRewards", "error", err)
+				//	return err
+				//}
 				// Operator tokens
-				if err := conn.RegisterFunc("amazon_operator_tokens", numbers.CalculateAmazonOperatorTokens, true); err != nil {
-					l.Sugar().Errorw("Failed to register function CalculateAmazonOperatorTokens", "error", err)
-					return err
-				}
-				if err := conn.RegisterFunc("nile_operator_tokens", numbers.CalculateNileOperatorTokens, true); err != nil {
-					l.Sugar().Errorw("Failed to register function CalculateNileOperatorTokens", "error", err)
-					return err
-				}
+				//if err := conn.RegisterFunc("amazon_operator_tokens", numbers.CalculateAmazonOperatorTokens, true); err != nil {
+				//	l.Sugar().Errorw("Failed to register function CalculateAmazonOperatorTokens", "error", err)
+				//	return err
+				//}
+				//if err := conn.RegisterFunc("nile_operator_tokens", numbers.CalculateNileOperatorTokens, true); err != nil {
+				//	l.Sugar().Errorw("Failed to register function CalculateNileOperatorTokens", "error", err)
+				//	return err
+				//}
 				if err := conn.RegisterFunc("post_nile_operator_tokens", numbers.CalculatePostNileOperatorTokens, true); err != nil {
 					l.Sugar().Errorw("Failed to register function CalculatePostNileOperatorTokens", "error", err)
 					return err
@@ -162,7 +159,7 @@ func NewSqlite(path string, l *zap.Logger) gorm.Dialector {
 
 	return &sqlite.Dialector{
 		DriverName: "sqlite3_with_extensions",
-		DSN:        path,
+		DSN:        cfg.Path,
 	}
 }