refactor: support rewards table names from previous releases

handle matching table names

.testdataVersion

@@ -1 +1 @@
-e94b8e364dfddd0743746f089f89a3d570751f03
+d67ef5d895bdc0ccd6a006a1683ac3e58f820ad0

internal/tests/testdata/operatorAvsSplitSnapshots/README.md

@@ -0,0 +1,19 @@
+
+## seed data
+
+```sql
+select
+    lower(arguments #>> '{1, Value}') as operator,
+    lower(arguments #>> '{2, Value}') as avs,
+    to_timestamp((output_data ->> 'activatedAt')::integer)::timestamp(6) as activated_at,
+    output_data ->> 'oldOperatorAVSSplitBips' as old_operator_avs_split_bips,
+    output_data ->> 'newOperatorAVSSplitBips' as new_operator_avs_split_bips,
+    block_number,
+    transaction_hash,
+    log_index
+from transaction_logs
+where
+    address = '0xb22ef643e1e067c994019a4c19e403253c05c2b0'
+    and event_name = 'OperatorAVSSplitBipsSet'
+order by block_number desc
+```

internal/tests/testdata/operatorDirectedRewardSubmissions/README.md

@@ -0,0 +1,47 @@
+```sql
+WITH strategies AS (
+  SELECT
+    tl.*,
+    output_data->'operatorDirectedRewardsSubmission'->>'token' as token,
+    output_data->'operatorDirectedRewardsSubmission'->>'duration' as duration,
+    output_data->'operatorDirectedRewardsSubmission'->>'startTimestamp' as start_timestamp,
+    strategy_data,
+    strategy_idx - 1 as strategy_idx  -- Subtract 1 for 0-based indexing
+  FROM transaction_logs as tl,
+  jsonb_array_elements(output_data->'operatorDirectedRewardsSubmission'->'strategiesAndMultipliers')
+  WITH ORDINALITY AS t(strategy_data, strategy_idx)
+  where
+    address = '0xb22ef643e1e067c994019a4c19e403253c05c2b0'
+    and event_name = 'OperatorDirectedAVSRewardsSubmissionCreated'
+),
+operators AS (
+  SELECT
+    operator_data,
+    output_data->'operatorDirectedRewardsSubmission' as rewards_submission,
+    operator_idx - 1 as operator_idx  -- Subtract 1 to make it 0-based indexing
+  FROM transaction_logs,
+  jsonb_array_elements(output_data->'operatorDirectedRewardsSubmission'->'operatorRewards')
+  WITH ORDINALITY AS t(operator_data, operator_idx)
+  where
+    address = '0xb22ef643e1e067c994019a4c19e403253c05c2b0'
+    and event_name = 'OperatorDirectedAVSRewardsSubmissionCreated'
+)
+SELECT
+    lower(arguments #>> '{1, Value}') as avs,
+    lower(arguments #>> '{2, Value}') as reward_hash,
+    strategies.token,
+    operator_data->>'operator' as operator,
+    operator_idx as operator_index,
+    operator_data->>'amount' as amount,
+    strategy_data->>'strategy' as strategy,
+    strategy_idx as strategy_index,
+    strategy_data->>'multiplier' as multiplier,
+    (to_timestamp((rewards_submission->>'startTimestamp')::int))::timestamp(6) as start_timestamp,
+    (rewards_submission->>'duration')::int as duration,
+    to_timestamp((rewards_submission->>'startTimestamp')::int + (rewards_submission->>'duration')::int)::timestamp(6) as end_timestamp,
+    block_number,
+    transaction_hash,
+    log_index
+FROM strategies
+CROSS JOIN operators;
+```

internal/tests/testdata/operatorPISplitSnapshots/README.md

@@ -0,0 +1,16 @@
+
+```sql
+select
+    lower(arguments #>> '{1, Value}') as operator,
+    to_timestamp((output_data ->> 'activatedAt')::integer)::timestamp(6) as activated_at,
+    output_data ->> 'oldOperatorPISplitBips' as old_operator_pi_split_bips,
+    output_data ->> 'newOperatorPISplitBips' as new_operator_pi_split_bips,
+    block_number,
+    transaction_hash,
+    log_index
+from transaction_logs
+where
+    address = '0xb22ef643e1e067c994019a4c19e403253c05c2b0'
+    and event_name = 'OperatorPISplitBipsSet'
+order by block_number desc
+```

pkg/rewards/stakerOperators/1_stakerStrategyPayouts.go

@@ -137,17 +137,26 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert1StakerStrategyPayouts(cut
 		return err
 	}
 
+	rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{
+		rewardsUtils.Table_1_ActiveRewards:       rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_1_ActiveRewards],
+		rewardsUtils.Table_2_StakerRewardAmounts: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_2_StakerRewardAmounts],
+	}, cutoffDate)
+	if err != nil {
+		sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err)
+		return err
+	}
+
 	query, err := rewardsUtils.RenderQueryTemplate(_1_stakerStrategyPayoutsQuery, map[string]string{
 		"destTableName":            destTableName,
-		"activeRewardsTable":       allTableNames[rewardsUtils.Table_1_ActiveRewards],
-		"stakerRewardAmountsTable": allTableNames[rewardsUtils.Table_2_StakerRewardAmounts],
+		"activeRewardsTable":       rewardsTables[rewardsUtils.Table_1_ActiveRewards],
+		"stakerRewardAmountsTable": rewardsTables[rewardsUtils.Table_2_StakerRewardAmounts],
 	})
 	if err != nil {
 		sog.logger.Sugar().Errorw("Failed to render 1_stakerStrategyPayouts query", "error", err)
 		return err
 	}
 
-	res := sog.db.Debug().Exec(query,
+	res := sog.db.Exec(query,
 		sql.Named("amazonHardforkDate", forks[config.Fork_Amazon]),
 		sql.Named("nileHardforkDate", forks[config.Fork_Nile]),
 	)

pkg/rewards/stakerOperators/2_operatorStrategyRewards.go

@@ -106,10 +106,24 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert2OperatorStrategyRewards(c
 		"cutoffDate", cutoffDate,
 	)
 
+	if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil {
+		sog.logger.Sugar().Errorw("Failed to drop table", "error", err)
+		return err
+	}
+
+	rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{
+		rewardsUtils.Table_1_ActiveRewards:         rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_1_ActiveRewards],
+		rewardsUtils.Table_3_OperatorRewardAmounts: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_3_OperatorRewardAmounts],
+	}, cutoffDate)
+	if err != nil {
+		sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err)
+		return err
+	}
+
 	query, err := rewardsUtils.RenderQueryTemplate(_2_operatorStrategyRewardsQuery, map[string]string{
 		"destTableName":              destTableName,
-		"activeRewardsTable":         allTableNames[rewardsUtils.Table_1_ActiveRewards],
-		"operatorRewardAmountsTable": allTableNames[rewardsUtils.Table_3_OperatorRewardAmounts],
+		"activeRewardsTable":         rewardsTables[rewardsUtils.Table_1_ActiveRewards],
+		"operatorRewardAmountsTable": rewardsTables[rewardsUtils.Table_3_OperatorRewardAmounts],
 	})
 	if err != nil {
 		sog.logger.Sugar().Errorw("Failed to render 2_operatorStrategyRewards query", "error", err)

pkg/rewards/stakerOperators/3_rewardsForAllStrategyPayouts.go

@@ -81,9 +81,13 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert3RewardsForAllStrategyPayo
 		return err
 	}
 
+	rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{
+		rewardsUtils.Table_1_ActiveRewards: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_1_ActiveRewards],
+	}, cutoffDate)
+
 	query, err := rewardsUtils.RenderQueryTemplate(_3_rewardsForAllStrategyPayoutsQuery, map[string]string{
 		"destTableName":      destTableName,
-		"activeRewardsTable": allTableNames[rewardsUtils.Table_1_ActiveRewards],
+		"activeRewardsTable": rewardsTables[rewardsUtils.Table_1_ActiveRewards],
 	})
 	if err != nil {
 		sog.logger.Sugar().Errorw("Failed to render 3_rewardsForAllStrategyPayouts query", "error", err)

pkg/rewards/stakerOperators/4_rfaeStakerStrategyPayouts.go

@@ -123,10 +123,14 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert4RfaeStakerStrategyPayout(
 		return err
 	}
 
+	rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{
+		rewardsUtils.Table_1_ActiveRewards: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_1_ActiveRewards],
+		rewardsUtils.Table_5_RfaeStakers:   rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_5_RfaeStakers],
+	}, cutoffDate)
 	query, err := rewardsUtils.RenderQueryTemplate(_4_rfaeStakerStrategyPayoutsQuery, map[string]string{
 		"destTableName":      destTableName,
-		"activeRewardsTable": allTableNames[rewardsUtils.Table_1_ActiveRewards],
-		"rfaeStakersTable":   allTableNames[rewardsUtils.Table_5_RfaeStakers],
+		"activeRewardsTable": rewardsTables[rewardsUtils.Table_1_ActiveRewards],
+		"rfaeStakersTable":   rewardsTables[rewardsUtils.Table_5_RfaeStakers],
 	})
 	if err != nil {
 		sog.logger.Sugar().Errorw("Failed to render 4_rfaeStakerStrategyPayouts query", "error", err)

pkg/rewards/stakerOperators/5_rfaeOperatorStrategyPayout.go

@@ -108,10 +108,14 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert5RfaeOperatorStrategyPayou
 		return err
 	}
 
+	rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{
+		rewardsUtils.Table_1_ActiveRewards: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_1_ActiveRewards],
+		rewardsUtils.Table_6_RfaeOperators: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_6_RfaeOperators],
+	}, cutoffDate)
 	query, err := rewardsUtils.RenderQueryTemplate(_5_rfaeOperatorStrategyPayoutsQuery, map[string]string{
 		"destTableName":      destTableName,
-		"activeRewardsTable": allTableNames[rewardsUtils.Table_1_ActiveRewards],
-		"rfaeOperatorsTable": allTableNames[rewardsUtils.Table_6_RfaeOperators],
+		"activeRewardsTable": rewardsTables[rewardsUtils.Table_1_ActiveRewards],
+		"rfaeOperatorsTable": rewardsTables[rewardsUtils.Table_6_RfaeOperators],
 	})
 	if err != nil {
 		sog.logger.Sugar().Errorw("Failed to render 5_rfaeOperatorStrategyPayouts query", "error", err)

pkg/rewards/stakerOperators/6_stakerOperatorStaging.go

@@ -118,7 +118,6 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert6StakerOperatorStaging(cut
 		zap.String("destTableName", destTableName),
 		zap.String("cutoffDate", cutoffDate),
 	)
-
 	query, err := rewardsUtils.RenderQueryTemplate(_6_stakerOperatorsStaging, map[string]string{
 		"destTableName":                    destTableName,
 		"sot1StakerStrategyPayouts":        allTableNames[rewardsUtils.Sot_1_StakerStrategyPayouts],

pkg/rewards/stakerOperators/7_stakerOperator.go

@@ -71,7 +71,7 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert7StakerOperator(cutoffDate
 		return err
 	}
 
-	res := sog.db.Debug().Exec(query)
+	res := sog.db.Exec(query)
 	if res.Error != nil {
 		sog.logger.Sugar().Errorw("Failed to generate 7_stakerOperator",
 			zap.String("cutoffDate", cutoffDate),

pkg/rewards/stakerOperators/stakerOperator.go

@@ -2,6 +2,7 @@ package stakerOperators
 
 import (
 	"github.com/Layr-Labs/sidecar/internal/config"
+	"github.com/Layr-Labs/sidecar/pkg/rewardsUtils"
 	"go.uber.org/zap"
 	"gorm.io/gorm"
 )
@@ -95,3 +96,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str
 	}
 	return nil
 }
+
+func (sog *StakerOperatorsGenerator) FindRewardsTableNamesForSearchPattersn(patterns map[string]string, cutoffDate string) (map[string]string, error) {
+	return rewardsUtils.FindRewardsTableNamesForSearchPatterns(patterns, cutoffDate, sog.globalConfig.DatabaseConfig.SchemaName, sog.db)
+}

pkg/rewardsUtils/rewardsUtils.go

@@ -2,6 +2,7 @@ package rewardsUtils
 
 import (
 	"bytes"
+	"database/sql"
 	"fmt"
 	"github.com/Layr-Labs/sidecar/pkg/postgres/helpers"
 	"github.com/Layr-Labs/sidecar/pkg/utils"
@@ -47,6 +48,23 @@ var goldTableBaseNames = map[string]string{
 	Sot_6_StakerOperatorStaging:       Sot_6_StakerOperatorStaging,
 }
 
+var GoldTableNameSearchPattern = map[string]string{
+	Table_1_ActiveRewards:         "gold_%_active_rewards",
+	Table_2_StakerRewardAmounts:   "gold_%_staker_reward_amounts",
+	Table_3_OperatorRewardAmounts: "gold_%_operator_reward_amounts",
+	Table_4_RewardsForAll:         "gold_%_rewards_for_all",
+	Table_5_RfaeStakers:           "gold_%_rfae_stakers",
+	Table_6_RfaeOperators:         "gold_%_rfae_operators",
+	Table_7_GoldStaging:           "gold_%_staging",
+
+	Sot_1_StakerStrategyPayouts:       "sot_%_staker_strategy_payouts",
+	Sot_2_OperatorStrategyPayouts:     "sot_%_operator_strategy_payouts",
+	Sot_3_RewardsForAllStrategyPayout: "sot_%_rewards_for_all_strategy_payout",
+	Sot_4_RfaeStakers:                 "sot_%_rfae_stakers",
+	Sot_5_RfaeOperators:               "sot_%_rfae_operators",
+	Sot_6_StakerOperatorStaging:       "sot_%_staker_operator_staging",
+}
+
 func GetGoldTableNames(snapshotDate string) map[string]string {
 	tableNames := make(map[string]string)
 	for key, baseName := range goldTableBaseNames {
@@ -113,3 +131,48 @@ func DropTableIfExists(grm *gorm.DB, tableName string, l *zap.Logger) error {
 	}
 	return nil
 }
+
+func findTableByLikeName(likeName string, grm *gorm.DB, schemaName string) (string, error) {
+	if schemaName == "" {
+		schemaName = "public"
+	}
+	query := `
+			SELECT table_name
+			FROM information_schema.tables
+			WHERE table_schema = @schemaName
+				AND table_type='BASE TABLE'
+				and table_name like @pattern
+			limit 1
+		`
+	var tname string
+	res := grm.Debug().Raw(query,
+		sql.Named("schemaName", schemaName),
+		sql.Named("pattern", likeName)).
+		Scan(&tname)
+	if res.Error != nil {
+		return "", res.Error
+	}
+	if tname == "" {
+		return "", fmt.Errorf("table not found for key %s", likeName)
+	}
+	return tname, nil
+}
+
+// FindRewardsTableNamesForSearchPatterns finds the table names for the given search patterns
+//
+// As table names evolve over time due to adding more, the numerical index might change in the constants
+// in this file. This makes finding past table names difficult. This function helps to find the table names
+// using the base table name and the cutoff date with a wildcard at the front.
+func FindRewardsTableNamesForSearchPatterns(patterns map[string]string, cutoffDate string, schemaName string, grm *gorm.DB) (map[string]string, error) {
+	results := make(map[string]string)
+	for key, pattern := range patterns {
+		snakeCaseCutoffDate := utils.SnakeCase(cutoffDate)
+		p := fmt.Sprintf("%s_%s", pattern, snakeCaseCutoffDate)
+		tname, err := findTableByLikeName(p, grm, schemaName)
+		if err != nil {
+			return nil, err
+		}
+		results[key] = tname
+	}
+	return results, nil
+}