Merge pull request #1816 from c9s/c9s/fix/circuit-breaker-metrics-label

FIX: [circuit breaker] separate metrics label by symbol and add panic handler
c9s · Nov 14, 2024 · 2af2b4e · 2af2b4e
2 parents cd58793 + e47609e
commit 2af2b4e
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 15 deletions.
diff --git a/pkg/risk/circuitbreaker/basic.go b/pkg/risk/circuitbreaker/basic.go
@@ -13,53 +13,55 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+var metricsLabels = []string{"strategy", "strategyInstance", "symbol"}
+
 var consecutiveTotalLossMetrics = prometheus.NewGaugeVec(
 	prometheus.GaugeOpts{
 		Name: "bbgo_circuit_breaker_consecutive_total_loss",
 		Help: "",
-	}, []string{"strategy", "strategyInstance"})
+	}, metricsLabels)
 
 var consecutiveLossTimesCounterMetrics = prometheus.NewGaugeVec(
 	prometheus.GaugeOpts{
 		Name: "bbgo_circuit_breaker_consecutive_loss_times",
 		Help: "",
-	}, []string{"strategy", "strategyInstance"})
+	}, metricsLabels)
 
 var haltCounterMetrics = prometheus.NewGaugeVec(
 	prometheus.GaugeOpts{
 		Name: "bbgo_circuit_breaker_halt_counter",
 		Help: "",
-	}, []string{"strategy", "strategyInstance"})
+	}, metricsLabels)
 
 var haltMetrics = prometheus.NewGaugeVec(
 	prometheus.GaugeOpts{
 		Name: "bbgo_circuit_breaker_halt",
 		Help: "",
-	}, []string{"strategy", "strategyInstance"})
+	}, metricsLabels)
 
 var totalProfitMetrics = prometheus.NewGaugeVec(
 	prometheus.GaugeOpts{
 		Name: "bbgo_circuit_breaker_total_profit",
 		Help: "",
-	}, []string{"strategy", "strategyInstance"})
+	}, metricsLabels)
 
 var profitWinCounterMetrics = prometheus.NewCounterVec(
 	prometheus.CounterOpts{
 		Name: "bbgo_circuit_breaker_profit_win_counter",
 		Help: "profit winning counter",
-	}, []string{"strategy", "strategyInstance"})
+	}, metricsLabels)
 
 var profitLossCounterMetrics = prometheus.NewCounterVec(
 	prometheus.CounterOpts{
 		Name: "bbgo_circuit_breaker_profit_loss_counter",
-		Help: "profit los counter",
-	}, []string{"strategy", "strategyInstance"})
+		Help: "profit loss counter",
+	}, metricsLabels)
 
 var winningRatioMetrics = prometheus.NewGaugeVec(
 	prometheus.GaugeOpts{
 		Name: "bbgo_circuit_breaker_winning_ratio",
 		Help: "winning ratio",
-	}, []string{"strategy", "strategyInstance"})
+	}, metricsLabels)
 
 func init() {
 	prometheus.MustRegister(
@@ -74,6 +76,7 @@ func init() {
 	)
 }
 
+//go:generate callbackgen -type BasicCircuitBreaker
 type BasicCircuitBreaker struct {
 	Enabled bool `json:"enabled"`
 
@@ -94,7 +97,9 @@ type BasicCircuitBreaker struct {
 
 	HaltDuration types.Duration `json:"haltDuration"`
 
-	strategyID, strategyInstance string
+	strategyID, strategyInstance, symbol string
+
+	panicCallbacks []func()
 
 	haltCounter int
 	haltReason  string
@@ -117,7 +122,7 @@ type BasicCircuitBreaker struct {
 	metricsLabels prometheus.Labels
 }
 
-func NewBasicCircuitBreaker(strategyID, strategyInstance string) *BasicCircuitBreaker {
+func NewBasicCircuitBreaker(strategyID, strategyInstance, symbol string) *BasicCircuitBreaker {
 	b := &BasicCircuitBreaker{
 		Enabled:                       true,
 		MaximumConsecutiveLossTimes:   8,
@@ -127,19 +132,39 @@ func NewBasicCircuitBreaker(strategyID, strategyInstance string) *BasicCircuitBr
 		HaltDuration:     types.Duration(1 * time.Hour),
 		strategyID:       strategyID,
 		strategyInstance: strategyInstance,
-		metricsLabels:    prometheus.Labels{"strategy": strategyID, "strategyInstance": strategyInstance},
+		symbol:           symbol,
+		metricsLabels: prometheus.Labels{
+			"strategy":         strategyID,
+			"strategyInstance": strategyInstance,
+			"symbol":           symbol,
+		},
 	}
 
 	b.updateMetrics()
 	return b
 }
 
+func (b *BasicCircuitBreaker) SetMetricsInfo(strategyID, strategyInstance, symbol string) {
+	b.strategyID = strategyID
+	b.strategyInstance = strategyInstance
+	b.symbol = symbol
+	b.metricsLabels = prometheus.Labels{
+		"strategy":         b.strategyID,
+		"strategyInstance": b.strategyInstance,
+		"symbol":           b.symbol,
+	}
+}
+
 func (b *BasicCircuitBreaker) getMetricsLabels() prometheus.Labels {
 	if b.metricsLabels != nil {
 		return b.metricsLabels
 	}
 
-	return prometheus.Labels{"strategy": b.strategyID, "strategyInstance": b.strategyInstance}
+	return prometheus.Labels{
+		"strategy":         b.strategyID,
+		"strategyInstance": b.strategyInstance,
+		"symbol":           b.symbol,
+	}
 }
 
 func (b *BasicCircuitBreaker) updateMetrics() {
@@ -263,6 +288,8 @@ func (b *BasicCircuitBreaker) halt(now time.Time, reason string) {
 	defer b.updateMetrics()
 
 	if b.MaximumHaltTimesExceededPanic && b.haltCounter > b.MaximumHaltTimes {
+		b.EmitPanic()
+
 		panic(fmt.Errorf("total %d halt times > maximumHaltTimesExceededPanic %d", b.haltCounter, b.MaximumHaltTimes))
 	}
 }
diff --git a/pkg/risk/circuitbreaker/basiccircuitbreaker_callbacks.go b/pkg/risk/circuitbreaker/basiccircuitbreaker_callbacks.go
diff --git a/pkg/strategy/common/strategy.go b/pkg/strategy/common/strategy.go
@@ -81,7 +81,8 @@ func (s *Strategy) Initialize(
 
 	if !s.CircuitBreakLossThreshold.IsZero() {
 		log.Infof("circuitBreakLossThreshold is configured, setting up CircuitBreakRiskControl...")
-		s.circuitBreakRiskControl = circuitbreaker.NewBasicCircuitBreaker(strategyID, instanceID)
+		s.circuitBreakRiskControl = circuitbreaker.NewBasicCircuitBreaker(strategyID, instanceID, market.Symbol)
+
 		s.OrderExecutor.TradeCollector().OnProfit(func(trade types.Trade, profit *types.Profit) {
 			if profit != nil && s.circuitBreakRiskControl != nil {
 				s.circuitBreakRiskControl.RecordProfit(profit.Profit, trade.Time.Time())

diff --git a/pkg/strategy/xmaker/strategy.go b/pkg/strategy/xmaker/strategy.go
@@ -1406,7 +1406,9 @@ func (s *Strategy) Defaults() error {
 	}
 
 	if s.CircuitBreaker == nil {
-		s.CircuitBreaker = circuitbreaker.NewBasicCircuitBreaker(ID, s.InstanceID())
+		s.CircuitBreaker = circuitbreaker.NewBasicCircuitBreaker(ID, s.InstanceID(), s.Symbol)
+	} else {
+		s.CircuitBreaker.SetMetricsInfo(ID, s.InstanceID(), s.Symbol)
 	}
 
 	// circuitBreakerAlertLimiter is for CircuitBreaker alerts