Skip to content

Commit

Permalink
retry redirect to AlreadyVisitedUrl will loop error
Browse files Browse the repository at this point in the history
  • Loading branch information
Shinku-Chen committed Apr 8, 2024
1 parent b27534c commit 23e3a03
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion colly.go
Original file line number Diff line number Diff line change
Expand Up @@ -653,7 +653,11 @@ func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, c
}
// note: once 1.13 is minimum supported Go version,
// replace this with http.NewRequestWithContext
req = req.WithContext(context.WithValue(req.WithContext(c.Context).Context(), CheckRevisitKey, checkRevisit))
if c.Context == nil {
req = req.WithContext(context.WithValue(context.Background(), CheckRevisitKey, checkRevisit))
} else {
req = req.WithContext(context.WithValue(c.Context, CheckRevisitKey, checkRevisit))
}
if err := c.requestCheck(parsedURL, method, req.GetBody, depth, checkRevisit); err != nil {
return err
}
Expand Down

0 comments on commit 23e3a03

Please sign in to comment.