Add retry on Delete silence

Signed-off-by: Eunice Kim <[email protected]>

pkg/alertmanager/distributor.go

@@ -247,7 +247,7 @@ func (d *Distributor) doUnary(userID string, w http.ResponseWriter, r *http.Requ
 	// we forward the request to only only of the alertmanagers.
 
 	var instances []ring.InstanceDesc
-	if req.GetMethod() == "GET" {
+	if req.GetMethod() == "GET" || req.GetMethod() == "DELETE" {
 		instances = replicationSet.Instances
 		// Randomize the list of instances to not always query the same one.
 		rand.Shuffle(len(instances), func(i, j int) {

pkg/alertmanager/distributor_test.go

@@ -200,6 +200,24 @@ func TestDistributor_DistributeRequest(t *testing.T) {
 			expStatusCode:      http.StatusOK,
 			expectedTotalCalls: 1,
 			route:              "/silence/id",
+		}, {
+			name:               "Delete /silence/id should try all alert managers on error",
+			numAM:              3,
+			numHappyAM:         0,
+			replicationFactor:  3,
+			isDelete:           true,
+			expStatusCode:      http.StatusInternalServerError,
+			expectedTotalCalls: 3,
+			route:              "/silence/id",
+		}, {
+			name:               "Delete /silence/id is sent to 3 AM when 2 are not happy",
+			numAM:              3,
+			numHappyAM:         1,
+			replicationFactor:  3,
+			isDelete:           true,
+			expStatusCode:      http.StatusOK,
+			expectedTotalCalls: 3,
+			route:              "/silence/id",
 		}, {
 			name:               "Read /status is sent to only 1 AM",
 			numAM:              5,