Revert "Respect request delay"

This reverts commit ef3115f.
spatie · Jul 30, 2024 · ae09480 · ae09480
1 parent ef3115f
commit ae09480
Showing 1 changed file with 5 additions and 6 deletions.
diff --git a/src/Handlers/CrawlRequestFulfilled.php b/src/Handlers/CrawlRequestFulfilled.php
@@ -51,7 +51,6 @@ public function __invoke(ResponseInterface $response, $index)
 
                 $this->crawler->getCrawlObservers()->crawlFailed($crawlUrl, $exception);
 
-                usleep($this->crawler->getDelayBetweenRequests());
                 return;
             }
 
@@ -65,13 +64,13 @@ public function __invoke(ResponseInterface $response, $index)
             $this->handleCrawled($responseWithCachedBody, $crawlUrl);
         }
 
-        if (!$this->crawler->getCrawlProfile() instanceof CrawlSubdomains) {
+        if (! $this->crawler->getCrawlProfile() instanceof CrawlSubdomains) {
             if ($crawlUrl->url->getHost() !== $this->crawler->getBaseUrl()->getHost()) {
                 return;
             }
         }
 
-        if (!$robots->mayFollow()) {
+        if (! $robots->mayFollow()) {
             return;
         }
 
@@ -103,7 +102,7 @@ protected function getBody(ResponseInterface $response): string
     {
         $contentType = $response->getHeaderLine('Content-Type');
 
-        if (!$this->isMimetypeAllowedToParse($contentType)) {
+        if (! $this->isMimetypeAllowedToParse($contentType)) {
             return '';
         }
 
@@ -127,7 +126,7 @@ protected function convertBodyToString(StreamInterface $bodyStream, $readMaximum
                 $newDataRead = null;
             }
 
-            if (!$newDataRead) {
+            if (! $newDataRead) {
                 break;
             }
 
@@ -152,7 +151,7 @@ protected function isMimetypeAllowedToParse($contentType): bool
             return true;
         }
 
-        if (!count($this->crawler->getParseableMimeTypes())) {
+        if (! count($this->crawler->getParseableMimeTypes())) {
             return true;
         }