diff --git a/integration/e2ecortex/client.go b/integration/e2ecortex/client.go index 2de73dbdd3..bc53f4dc58 100644 --- a/integration/e2ecortex/client.go +++ b/integration/e2ecortex/client.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "strconv" + "strings" "time" "github.com/gogo/protobuf/proto" @@ -74,7 +75,7 @@ func NewClient( querierAddress: querierAddress, alertmanagerAddress: alertmanagerAddress, rulerAddress: rulerAddress, - timeout: 5 * time.Second, + timeout: 30 * time.Second, httpClient: &http.Client{}, querierClient: promv1.NewAPI(querierAPIClient), orgID: orgID, @@ -105,7 +106,7 @@ func NewPromQueryClient(address string) (*Client, error) { } c := &Client{ - timeout: 5 * time.Second, + timeout: 30 * time.Second, httpClient: &http.Client{}, querierClient: promv1.NewAPI(querierAPIClient), } @@ -332,7 +333,26 @@ func (c *Client) OTLP(timeseries []prompb.TimeSeries) (*http.Response, error) { // Query runs an instant query. func (c *Client) Query(query string, ts time.Time) (model.Value, error) { - value, _, err := c.querierClient.Query(context.Background(), query, ts) + ctx := context.Background() + retries := backoff.New(ctx, backoff.Config{ + MinBackoff: 1 * time.Second, + MaxBackoff: 3 * time.Second, + MaxRetries: 5, + }) + var ( + value model.Value + err error + ) + for retries.Ongoing() { + value, _, err = c.querierClient.Query(context.Background(), query, ts) + if err == nil { + break + } + if !strings.Contains(err.Error(), "EOF") { + break + } + retries.Wait() + } return value, err } @@ -345,11 +365,31 @@ func (c *Client) QueryExemplars(query string, start, end time.Time) ([]promv1.Ex // QueryRange runs a query range. func (c *Client) QueryRange(query string, start, end time.Time, step time.Duration) (model.Value, error) { - value, _, err := c.querierClient.QueryRange(context.Background(), query, promv1.Range{ - Start: start, - End: end, - Step: step, + ctx := context.Background() + retries := backoff.New(ctx, backoff.Config{ + MinBackoff: 1 * time.Second, + MaxBackoff: 3 * time.Second, + MaxRetries: 5, }) + var ( + value model.Value + err error + ) + for retries.Ongoing() { + value, _, err = c.querierClient.QueryRange(context.Background(), query, promv1.Range{ + Start: start, + End: end, + Step: step, + }) + if err == nil { + break + } + if !strings.Contains(err.Error(), "EOF") { + break + } + retries.Wait() + } + return value, err } diff --git a/integration/query_fuzz_test.go b/integration/query_fuzz_test.go index 5c31e5089e..6029582f0f 100644 --- a/integration/query_fuzz_test.go +++ b/integration/query_fuzz_test.go @@ -515,7 +515,7 @@ func TestVerticalShardingFuzz(t *testing.T) { } ps := promqlsmith.New(rnd, lbls, opts...) - runQueryFuzzTestCases(t, ps, c1, c2, now, start, end, scrapeInterval, 100) + runQueryFuzzTestCases(t, ps, c1, c2, now, start, end, scrapeInterval, 1000) } // comparer should be used to compare promql results between engines. @@ -1065,7 +1065,7 @@ func TestBackwardCompatibilityQueryFuzz(t *testing.T) { } ps := promqlsmith.New(rnd, lbls, opts...) - runQueryFuzzTestCases(t, ps, c1, c2, end, start, end, scrapeInterval, 100) + runQueryFuzzTestCases(t, ps, c1, c2, end, start, end, scrapeInterval, 1000) } // TestPrometheusCompatibilityQueryFuzz compares Cortex with latest Prometheus release. @@ -1178,7 +1178,7 @@ func TestPrometheusCompatibilityQueryFuzz(t *testing.T) { } ps := promqlsmith.New(rnd, lbls, opts...) - runQueryFuzzTestCases(t, ps, c1, c2, end, start, end, scrapeInterval, 100) + runQueryFuzzTestCases(t, ps, c1, c2, end, start, end, scrapeInterval, 1000) } // waitUntilReady is a helper function to wait and check if both servers to test load the expected data. @@ -1201,8 +1201,11 @@ func waitUntilReady(t *testing.T, ctx context.Context, c1, c2 *e2ecortex.Client, labelSet2, err = c2.Series([]string{query}, start, end) require.NoError(t, err) - if cmp.Equal(labelSet1, labelSet2, labelSetsComparer) { - break + // Make sure series can be queried. + if len(labelSet1) > 0 { + if cmp.Equal(labelSet1, labelSet2, labelSetsComparer) { + break + } } retries.Wait()