Add retry on Delete silence

CHANGELOG.md

@@ -23,6 +23,7 @@
 * [ENHANCEMENT] Query Frontend/Scheduler: Time check in query priority now considers overall data select time window (including range selectors, modifiers and lookback delta). #5758
 * [ENHANCEMENT] Querier: Added `querier.store-gateway-query-stats-enabled` to enable or disable store gateway query stats log. #5749
 * [ENHANCEMENT] Upgrade to go 1.21.6. #5765
+* [ENHANCEMENT] AlertManager: Retrying AlertManager Delete Silence on error #5794
 * [BUGFIX] Distributor: Do not use label with empty values for sharding #5717
 * [BUGFIX] Query Frontend: queries with negative offset should check whether it is cacheable or not. #5719
 * [BUGFIX] Redis Cache: pass `cache_size` config correctly. #5734

pkg/alertmanager/distributor.go

@@ -247,14 +247,14 @@ func (d *Distributor) doUnary(userID string, w http.ResponseWriter, r *http.Requ
 	// we forward the request to only only of the alertmanagers.
 
 	var instances []ring.InstanceDesc
-	if req.GetMethod() == "GET" {
+	if req.GetMethod() == "GET" || req.GetMethod() == "DELETE" {
 		instances = replicationSet.Instances
 		// Randomize the list of instances to not always query the same one.
 		rand.Shuffle(len(instances), func(i, j int) {
 			instances[i], instances[j] = instances[j], instances[i]
 		})
 	} else {
-		//Picking 1 instance at Random for Non-Get Unary Read requests, as shuffling through large number of instances might increase complexity
+		//Picking 1 instance at Random for Non-Get and Non-Delete Unary Read requests, as shuffling through large number of instances might increase complexity
 		randN := rand.Intn(len(replicationSet.Instances))
 		instances = replicationSet.Instances[randN : randN+1]
 	}

pkg/alertmanager/distributor_test.go

@@ -200,6 +200,24 @@ func TestDistributor_DistributeRequest(t *testing.T) {
 			expStatusCode:      http.StatusOK,
 			expectedTotalCalls: 1,
 			route:              "/silence/id",
+		}, {
+			name:               "Delete /silence/id should try all alert managers on error",
+			numAM:              3,
+			numHappyAM:         0,
+			replicationFactor:  3,
+			isDelete:           true,
+			expStatusCode:      http.StatusInternalServerError,
+			expectedTotalCalls: 3,
+			route:              "/silence/id",
+		}, {
+			name:               "Delete /silence/id is sent to 3 AM when 2 are not happy",
+			numAM:              3,
+			numHappyAM:         1,
+			replicationFactor:  3,
+			isDelete:           true,
+			expStatusCode:      http.StatusOK,
+			expectedTotalCalls: 3,
+			route:              "/silence/id",
 		}, {
 			name:               "Read /status is sent to only 1 AM",
 			numAM:              5,