From f4958b4f1e10113be2bc8d373afdd7b1d4fe0d72 Mon Sep 17 00:00:00 2001 From: Velka <59120414+Velka-DEV@users.noreply.github.com> Date: Tue, 2 Jan 2024 09:29:35 +0100 Subject: [PATCH] feat: custom link parser (#458) * feat: allow custom url parsers Previously named LinkAdder * fix: removed incorrect type check * feat: added sitemap url parser * tests: added sitemap url parsing test * chore: added readme documentation for custom url parsers * Update README.md * tests: added more consistent tests with sitemapindex and sitemap childs * Fix styling * chore: added documentation for the built-in SitemapUrlParser --------- Co-authored-by: Freek Van der Herten Co-authored-by: Velka-DEV --- README.md | 20 + src/CrawlObservers/CrawlObserver.php | 8 +- src/CrawlUrl.php | 4 +- src/Crawler.php | 19 +- src/Handlers/CrawlRequestFulfilled.php | 9 +- src/ResponseWithCachedBody.php | 2 +- .../LinkUrlParser.php} | 9 +- src/UrlParsers/SitemapUrlParser.php | 89 +++ src/UrlParsers/UrlParser.php | 13 + tests/CrawlObserverCollectionTest.php | 8 +- tests/SitemapUrlParserTest.php | 46 ++ tests/TestClasses/CrawlLogger.php | 10 +- tests/server/package-lock.json | 740 ++++++++++++------ tests/server/server.js | 72 ++ 14 files changed, 777 insertions(+), 272 deletions(-) rename src/{LinkAdder.php => UrlParsers/LinkUrlParser.php} (94%) create mode 100644 src/UrlParsers/SitemapUrlParser.php create mode 100644 src/UrlParsers/UrlParser.php create mode 100644 tests/SitemapUrlParserTest.php diff --git a/README.md b/README.md index d94a555a..818c0716 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,26 @@ This package comes with three `CrawlProfiles` out of the box: - `CrawlInternalUrls`: this profile will only crawl the internal urls on the pages of a host. - `CrawlSubdomains`: this profile will only crawl the internal urls and its subdomains on the pages of a host. +### Custom link extraction + +You can customize how links are extracted from a page by passing a custom `UrlParser` to the crawler. + +```php +Crawler::create() + ->setUrlParserClass(::class) + ... +``` + +By default, the `LinkUrlParser` is used. This parser will extract all links from the `href` attribute of `a` tags. + +There is also a built-in `SitemapUrlParser` that will extract & crawl all links from a sitemap. It does support sitemap index files. + +```php +Crawler::create() + ->setUrlParserClass(SitemapUrlParser::class) + ... +``` + ### Ignoring robots.txt and robots meta By default, the crawler will respect robots data. It is possible to disable these checks like so: diff --git a/src/CrawlObservers/CrawlObserver.php b/src/CrawlObservers/CrawlObserver.php index 8cd4ac4e..92a5c97a 100644 --- a/src/CrawlObservers/CrawlObserver.php +++ b/src/CrawlObservers/CrawlObserver.php @@ -21,8 +21,8 @@ public function willCrawl(UriInterface $url, ?string $linkText): void public function crawled( UriInterface $url, ResponseInterface $response, - UriInterface $foundOnUrl = null, - string $linkText = null, + ?UriInterface $foundOnUrl = null, + ?string $linkText = null, ): void { } @@ -32,8 +32,8 @@ public function crawled( public function crawlFailed( UriInterface $url, RequestException $requestException, - UriInterface $foundOnUrl = null, - string $linkText = null, + ?UriInterface $foundOnUrl = null, + ?string $linkText = null, ): void { } diff --git a/src/CrawlUrl.php b/src/CrawlUrl.php index a2596739..c52d43ea 100644 --- a/src/CrawlUrl.php +++ b/src/CrawlUrl.php @@ -16,9 +16,9 @@ class CrawlUrl public static function create( UriInterface $url, - UriInterface $foundOnUrl = null, + ?UriInterface $foundOnUrl = null, $id = null, - string $linkText = null, + ?string $linkText = null, ): static { $static = new static($url, $foundOnUrl, linkText: $linkText); diff --git a/src/Crawler.php b/src/Crawler.php index 996fe979..547d3b03 100644 --- a/src/Crawler.php +++ b/src/Crawler.php @@ -19,6 +19,7 @@ use Spatie\Crawler\Exceptions\InvalidCrawlRequestHandler; use Spatie\Crawler\Handlers\CrawlRequestFailed; use Spatie\Crawler\Handlers\CrawlRequestFulfilled; +use Spatie\Crawler\UrlParsers\LinkUrlParser; use Spatie\Robots\RobotsTxt; use Tree\Node\Node; @@ -62,6 +63,8 @@ class Crawler protected string $crawlRequestFailedClass; + protected string $urlParserClass; + protected int $delayBetweenRequests = 0; protected array $allowedMimeTypes = []; @@ -102,6 +105,8 @@ public function __construct( $this->crawlRequestFulfilledClass = CrawlRequestFulfilled::class; $this->crawlRequestFailedClass = CrawlRequestFailed::class; + + $this->urlParserClass = LinkUrlParser::class; } public function getDefaultScheme(): string @@ -345,6 +350,18 @@ public function setCrawlFailedHandlerClass(string $crawlRequestFailedClass): sel return $this; } + public function setUrlParserClass(string $urlParserClass): self + { + $this->urlParserClass = $urlParserClass; + + return $this; + } + + public function getUrlParserClass(): string + { + return $this->urlParserClass; + } + public function setBrowsershot(Browsershot $browsershot) { $this->browsershot = $browsershot; @@ -430,7 +447,7 @@ public function startCrawling(UriInterface|string $baseUrl) } } - public function addToDepthTree(UriInterface $url, UriInterface $parentUrl, Node $node = null): ?Node + public function addToDepthTree(UriInterface $url, UriInterface $parentUrl, ?Node $node = null): ?Node { if (is_null($this->maximumDepth)) { return new Node((string) $url); diff --git a/src/Handlers/CrawlRequestFulfilled.php b/src/Handlers/CrawlRequestFulfilled.php index 130e6f7b..4f45b491 100644 --- a/src/Handlers/CrawlRequestFulfilled.php +++ b/src/Handlers/CrawlRequestFulfilled.php @@ -13,16 +13,17 @@ use Spatie\Crawler\CrawlerRobots; use Spatie\Crawler\CrawlProfiles\CrawlSubdomains; use Spatie\Crawler\CrawlUrl; -use Spatie\Crawler\LinkAdder; use Spatie\Crawler\ResponseWithCachedBody; +use Spatie\Crawler\UrlParsers\UrlParser; class CrawlRequestFulfilled { - protected LinkAdder $linkAdder; + protected UrlParser $urlParser; public function __construct(protected Crawler $crawler) { - $this->linkAdder = new LinkAdder($this->crawler); + $urlParserClass = $this->crawler->getUrlParserClass(); + $this->urlParser = new $urlParserClass($this->crawler); } public function __invoke(ResponseInterface $response, $index) @@ -62,7 +63,7 @@ public function __invoke(ResponseInterface $response, $index) $baseUrl = $this->getBaseUrl($response, $crawlUrl); - $this->linkAdder->addFromHtml($body, $baseUrl); + $this->urlParser->addFromHtml($body, $baseUrl); usleep($this->crawler->getDelayBetweenRequests()); } diff --git a/src/ResponseWithCachedBody.php b/src/ResponseWithCachedBody.php index b7318da4..a44fd15d 100644 --- a/src/ResponseWithCachedBody.php +++ b/src/ResponseWithCachedBody.php @@ -20,7 +20,7 @@ public static function fromGuzzlePsr7Response(ResponseInterface $response): stat ); } - public function setCachedBody(string $body = null): void + public function setCachedBody(?string $body = null): void { $this->cachedBody = $body; } diff --git a/src/LinkAdder.php b/src/UrlParsers/LinkUrlParser.php similarity index 94% rename from src/LinkAdder.php rename to src/UrlParsers/LinkUrlParser.php index e2fcad92..69684cab 100644 --- a/src/LinkAdder.php +++ b/src/UrlParsers/LinkUrlParser.php @@ -1,15 +1,18 @@ getUri(), $linkText); } catch (InvalidArgumentException $exception) { - return; + return null; } }) ->filter(); diff --git a/src/UrlParsers/SitemapUrlParser.php b/src/UrlParsers/SitemapUrlParser.php new file mode 100644 index 00000000..04bf47ad --- /dev/null +++ b/src/UrlParsers/SitemapUrlParser.php @@ -0,0 +1,89 @@ +crawler = $crawler; + } + + public function addFromHtml(string $html, UriInterface $foundOnUrl): void + { + $allLinks = $this->extractLinksFromHtml($html, $foundOnUrl); + + collect($allLinks) + ->filter(fn (Url $url) => $this->hasCrawlableScheme($url)) + ->map(fn (Url $url) => $this->normalizeUrl($url)) + ->filter(function (Url $url) use ($foundOnUrl) { + if (! $node = $this->crawler->addToDepthTree($url, $foundOnUrl)) { + return false; + } + + return $this->shouldCrawl($node); + }) + ->filter(fn (Url $url) => ! str_contains($url->getPath(), '/tel:')) + ->each(function (Url $url) use ($foundOnUrl) { + $crawlUrl = CrawlUrl::create($url, $foundOnUrl, linkText: $url->linkText()); + + $this->crawler->addToCrawlQueue($crawlUrl); + }); + } + + protected function extractLinksFromHtml(string $html, UriInterface $foundOnUrl): ?Collection + { + $domCrawler = new DomCrawler($html, $foundOnUrl); + + return collect($domCrawler->filterXPath('//loc') + ->each(function (DomCrawler $node) { + try { + $linkText = $node->text(); + + if ($linkText) { + $linkText = substr($linkText, 0, 4000); + } + + return new Url($linkText, $linkText); + } catch (InvalidArgumentException $exception) { + return null; + } + })); + } + + protected function hasCrawlableScheme(UriInterface $uri): bool + { + return in_array($uri->getScheme(), ['http', 'https']); + } + + protected function normalizeUrl(UriInterface $url): UriInterface + { + return $url->withFragment(''); + } + + protected function shouldCrawl(Node $node): bool + { + if ($this->crawler->mustRespectRobots() && ! $this->crawler->getRobotsTxt()->allows($node->getValue(), $this->crawler->getUserAgent())) { + return false; + } + + $maximumDepth = $this->crawler->getMaximumDepth(); + + if (is_null($maximumDepth)) { + return true; + } + + return $node->getDepth() <= $maximumDepth; + } +} diff --git a/src/UrlParsers/UrlParser.php b/src/UrlParsers/UrlParser.php new file mode 100644 index 00000000..24ed5829 --- /dev/null +++ b/src/UrlParsers/UrlParser.php @@ -0,0 +1,13 @@ +crawled = true; } @@ -29,8 +29,8 @@ public function crawled( public function crawlFailed( UriInterface $url, RequestException $requestException, - UriInterface $foundOnUrl = null, - string $linkText = null, + ?UriInterface $foundOnUrl = null, + ?string $linkText = null, ): void { $this->failed = true; } diff --git a/tests/SitemapUrlParserTest.php b/tests/SitemapUrlParserTest.php new file mode 100644 index 00000000..d77f0da4 --- /dev/null +++ b/tests/SitemapUrlParserTest.php @@ -0,0 +1,46 @@ +setUrlParserClass(SitemapUrlParser::class) + ->startCrawling('http://localhost:8080/sitemap_index.xml'); + + expect(['url' => 'http://localhost:8080/sitemap1.xml', 'foundOn' => 'http://localhost:8080/sitemap_index.xml']) + ->toBeCrawledOnce(); + + expect(['url' => 'http://localhost:8080/sitemap2.xml', 'foundOn' => 'http://localhost:8080/sitemap_index.xml']) + ->toBeCrawledOnce(); +}); + +it('should extract urls from sitemaps trough sitemap index', function () { + createCrawler() + ->setUrlParserClass(SitemapUrlParser::class) + ->startCrawling('http://localhost:8080/sitemap_index.xml'); + + expect(['url' => 'http://localhost:8080/', 'foundOn' => 'http://localhost:8080/sitemap1.xml']) + ->toBeCrawledOnce(); + + expect(['url' => 'http://localhost:8080/link1', 'foundOn' => 'http://localhost:8080/sitemap1.xml']) + ->toBeCrawledOnce(); + + expect(['url' => 'http://localhost:8080/link1-next', 'foundOn' => 'http://localhost:8080/sitemap2.xml']) + ->toBeCrawledOnce(); + + expect(['url' => 'http://localhost:8080/link1-prev', 'foundOn' => 'http://localhost:8080/sitemap2.xml']) + ->toBeCrawledOnce(); + + expect(['url' => 'http://localhost:8080/link2', 'foundOn' => 'http://localhost:8080/sitemap2.xml']) + ->toBeCrawledOnce(); + + expect(['url' => 'http://localhost:8080/link3', 'foundOn' => 'http://localhost:8080/sitemap2.xml']) + ->toBeCrawledOnce(); +}); diff --git a/tests/TestClasses/CrawlLogger.php b/tests/TestClasses/CrawlLogger.php index c8935d95..a39c4d45 100644 --- a/tests/TestClasses/CrawlLogger.php +++ b/tests/TestClasses/CrawlLogger.php @@ -34,8 +34,8 @@ public function willCrawl(UriInterface $url, ?string $linkText): void public function crawled( UriInterface $url, ResponseInterface $response, - UriInterface $foundOnUrl = null, - string $linkText = null, + ?UriInterface $foundOnUrl = null, + ?string $linkText = null, ): void { $this->logCrawl($url, $foundOnUrl, $linkText); } @@ -43,13 +43,13 @@ public function crawled( public function crawlFailed( UriInterface $url, RequestException $requestException, - UriInterface $foundOnUrl = null, - string $linkText = null, + ?UriInterface $foundOnUrl = null, + ?string $linkText = null, ): void { $this->logCrawl($url, $foundOnUrl, $linkText); } - protected function logCrawl(UriInterface $url, ?UriInterface $foundOnUrl, string $linkText = null) + protected function logCrawl(UriInterface $url, ?UriInterface $foundOnUrl, ?string $linkText = null) { $logText = "{$this->observerId}hasBeenCrawled: {$url}"; diff --git a/tests/server/package-lock.json b/tests/server/package-lock.json index 5a885ce7..0a2af35f 100644 --- a/tests/server/package-lock.json +++ b/tests/server/package-lock.json @@ -1,75 +1,89 @@ { "name": "server", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 3, "requires": true, - "dependencies": { - "@types/node": { + "packages": { + "": { + "name": "server", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "body-parser": "^1.18.2", + "express": "^4.16.2", + "puppeteer": "^5.3.1" + } + }, + "node_modules/@types/node": { "version": "14.11.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz", "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==", "optional": true }, - "@types/yauzl": { + "node_modules/@types/yauzl": { "version": "2.9.1", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz", "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==", "optional": true, - "requires": { + "dependencies": { "@types/node": "*" } }, - "accepts": { + "node_modules/accepts": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.4.tgz", "integrity": "sha1-hiRnWMfdbSGmR0/whKR0DsBesh8=", - "requires": { + "dependencies": { "mime-types": "~2.1.16", "negotiator": "0.6.1" + }, + "engines": { + "node": ">= 0.6" } }, - "agent-base": { + "node_modules/agent-base": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", - "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" + "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==", + "engines": { + "node": ">= 6.0.0" + } }, - "array-flatten": { + "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, - "balanced-match": { + "node_modules/balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, - "base64-js": { + "node_modules/base64-js": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, - "bl": { + "node_modules/bl": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", - "requires": { + "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" - }, - "dependencies": { - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - } } }, - "body-parser": { + "node_modules/bl/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/body-parser": { "version": "1.18.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=", - "requires": { + "dependencies": { "bytes": "3.0.0", "content-type": "~1.0.4", "debug": "2.6.9", @@ -80,122 +94,149 @@ "qs": "6.5.1", "raw-body": "2.3.2", "type-is": "~1.6.15" + }, + "engines": { + "node": ">= 0.8" } }, - "brace-expansion": { + "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "requires": { + "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, - "buffer": { + "node_modules/buffer": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "requires": { + "dependencies": { "base64-js": "^1.0.2", "ieee754": "^1.1.4" } }, - "buffer-crc32": { + "node_modules/buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=", + "engines": { + "node": "*" + } }, - "bytes": { + "node_modules/bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=", + "engines": { + "node": ">= 0.8" + } }, - "chownr": { + "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, - "concat-map": { + "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, - "content-disposition": { + "node_modules/content-disposition": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", - "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" + "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=", + "engines": { + "node": ">= 0.6" + } }, - "content-type": { + "node_modules/content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "engines": { + "node": ">= 0.6" + } }, - "cookie": { + "node_modules/cookie": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", - "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" + "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=", + "engines": { + "node": ">= 0.6" + } }, - "cookie-signature": { + "node_modules/cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" }, - "debug": { + "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { + "dependencies": { "ms": "2.0.0" } }, - "depd": { + "node_modules/depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "engines": { + "node": ">= 0.6" + } }, - "destroy": { + "node_modules/destroy": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, - "devtools-protocol": { + "node_modules/devtools-protocol": { "version": "0.0.799653", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.799653.tgz", "integrity": "sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg==" }, - "ee-first": { + "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "encodeurl": { + "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "engines": { + "node": ">= 0.8" + } }, - "end-of-stream": { + "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "requires": { + "dependencies": { "once": "^1.4.0" } }, - "escape-html": { + "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, - "etag": { + "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "engines": { + "node": ">= 0.6" + } }, - "express": { + "node_modules/express": { "version": "4.16.2", "resolved": "https://registry.npmjs.org/express/-/express-4.16.2.tgz", "integrity": "sha1-41xt/i1kt9ygpc1PIXgb4ymeB2w=", - "requires": { + "dependencies": { "accepts": "~1.3.4", "array-flatten": "1.1.1", "body-parser": "1.18.2", @@ -227,58 +268,77 @@ "utils-merge": "1.0.1", "vary": "~1.1.2" }, - "dependencies": { - "setprototypeof": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" - }, - "statuses": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", - "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=" - } + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/express/node_modules/setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + }, + "node_modules/express/node_modules/statuses": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", + "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=", + "engines": { + "node": ">= 0.6" } }, - "extract-zip": { + "node_modules/extract-zip": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", - "requires": { - "@types/yauzl": "^2.9.1", + "dependencies": { "debug": "^4.1.1", "get-stream": "^5.1.0", "yauzl": "^2.10.0" }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extract-zip/node_modules/debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)", "dependencies": { - "debug": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", - "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true } } }, - "fd-slicer": { + "node_modules/extract-zip/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/fd-slicer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", - "requires": { + "dependencies": { "pend": "~1.2.0" } }, - "finalhandler": { + "node_modules/finalhandler": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.0.tgz", "integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=", - "requires": { + "dependencies": { "debug": "2.6.9", "encodeurl": "~1.0.1", "escape-html": "~1.0.3", @@ -287,302 +347,406 @@ "statuses": "~1.3.1", "unpipe": "~1.0.0" }, - "dependencies": { - "statuses": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", - "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=" - } + "engines": { + "node": ">= 0.8" } }, - "find-up": { + "node_modules/finalhandler/node_modules/statuses": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", + "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/find-up": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "requires": { + "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" } }, - "forwarded": { + "node_modules/forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=", + "engines": { + "node": ">= 0.6" + } }, - "fresh": { + "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "engines": { + "node": ">= 0.6" + } }, - "fs-constants": { + "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, - "fs.realpath": { + "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, - "get-stream": { + "node_modules/get-stream": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", - "requires": { + "dependencies": { "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "glob": { + "node_modules/glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "requires": { + "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.0.4", "once": "^1.3.0", "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "http-errors": { + "node_modules/http-errors": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=", - "requires": { + "dependencies": { "depd": "1.1.1", "inherits": "2.0.3", "setprototypeof": "1.0.3", "statuses": ">= 1.3.1 < 2" }, - "dependencies": { - "depd": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", - "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=" - } + "engines": { + "node": ">= 0.6" } }, - "https-proxy-agent": { + "node_modules/http-errors/node_modules/depd": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", + "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/https-proxy-agent": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", - "requires": { + "dependencies": { "agent-base": "5", "debug": "4" }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/https-proxy-agent/node_modules/debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)", "dependencies": { - "debug": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", - "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true } } }, - "iconv-lite": { + "node_modules/https-proxy-agent/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/iconv-lite": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", - "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" + "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==", + "engines": { + "node": ">=0.10.0" + } }, - "ieee754": { + "node_modules/ieee754": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, - "inflight": { + "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "requires": { + "dependencies": { "once": "^1.3.0", "wrappy": "1" } }, - "inherits": { + "node_modules/inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, - "ipaddr.js": { + "node_modules/ipaddr.js": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz", - "integrity": "sha1-4/o1e3c9phnybpXwSdBVxyeW+Gs=" + "integrity": "sha1-4/o1e3c9phnybpXwSdBVxyeW+Gs=", + "engines": { + "node": ">= 0.10" + } }, - "locate-path": { + "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "requires": { + "dependencies": { "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" } }, - "media-typer": { + "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "engines": { + "node": ">= 0.6" + } }, - "merge-descriptors": { + "node_modules/merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" }, - "methods": { + "node_modules/methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", + "engines": { + "node": ">= 0.6" + } }, - "mime": { + "node_modules/mime": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", - "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" + "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==", + "bin": { + "mime": "cli.js" + } }, - "mime-db": { + "node_modules/mime-db": { "version": "1.33.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", - "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==" + "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==", + "engines": { + "node": ">= 0.6" + } }, - "mime-types": { + "node_modules/mime-types": { "version": "2.1.18", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", - "requires": { + "dependencies": { "mime-db": "~1.33.0" + }, + "engines": { + "node": ">= 0.6" } }, - "minimatch": { + "node_modules/minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { + "dependencies": { "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" } }, - "mkdirp-classic": { + "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, - "ms": { + "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "negotiator": { + "node_modules/negotiator": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", - "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" + "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=", + "engines": { + "node": ">= 0.6" + } }, - "on-finished": { + "node_modules/on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", - "requires": { + "dependencies": { "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" } }, - "once": { + "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "requires": { + "dependencies": { "wrappy": "1" } }, - "p-limit": { + "node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "requires": { + "dependencies": { "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "p-locate": { + "node_modules/p-locate": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "requires": { + "dependencies": { "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" } }, - "p-try": { + "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "engines": { + "node": ">=6" + } }, - "parseurl": { + "node_modules/parseurl": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", - "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" + "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=", + "engines": { + "node": ">= 0.8" + } }, - "path-exists": { + "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "engines": { + "node": ">=8" + } }, - "path-is-absolute": { + "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "engines": { + "node": ">=0.10.0" + } }, - "path-to-regexp": { + "node_modules/path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "pend": { + "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, - "pkg-dir": { + "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "requires": { + "dependencies": { "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" } }, - "progress": { + "node_modules/progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "engines": { + "node": ">=0.4.0" + } }, - "proxy-addr": { + "node_modules/proxy-addr": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz", "integrity": "sha512-jQTChiCJteusULxjBp8+jftSQE5Obdl3k4cnmLA6WXtK6XFuWRnvVL7aCiBqaLPM8c4ph0S4tKna8XvmIwEnXQ==", - "requires": { + "dependencies": { "forwarded": "~0.1.2", "ipaddr.js": "1.6.0" + }, + "engines": { + "node": ">= 0.10" } }, - "proxy-from-env": { + "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, - "pump": { + "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "requires": { + "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, - "puppeteer": { + "node_modules/puppeteer": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.3.1.tgz", "integrity": "sha512-YTM1RaBeYrj6n7IlRXRYLqJHF+GM7tasbvrNFx6w1S16G76NrPq7oYFKLDO+BQsXNtS8kW2GxWCXjIMPvfDyaQ==", - "requires": { + "deprecated": "< 21.3.7 is no longer supported", + "hasInstallScript": true, + "dependencies": { "debug": "^4.1.0", "devtools-protocol": "0.0.799653", "extract-zip": "^2.0.0", @@ -595,71 +759,99 @@ "unbzip2-stream": "^1.3.3", "ws": "^7.2.3" }, + "engines": { + "node": ">=10.18.1" + } + }, + "node_modules/puppeteer/node_modules/debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)", "dependencies": { - "debug": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", - "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true } } }, - "qs": { + "node_modules/puppeteer/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/qs": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", - "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" + "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==", + "engines": { + "node": ">=0.6" + } }, - "range-parser": { + "node_modules/range-parser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", - "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" + "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=", + "engines": { + "node": ">= 0.6" + } }, - "raw-body": { + "node_modules/raw-body": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz", "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=", - "requires": { + "dependencies": { "bytes": "3.0.0", "http-errors": "1.6.2", "iconv-lite": "0.4.19", "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" } }, - "readable-stream": { + "node_modules/readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "requires": { + "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" } }, - "rimraf": { + "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "requires": { + "dependencies": { "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "safe-buffer": { + "node_modules/safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" }, - "send": { + "node_modules/send": { "version": "0.16.1", "resolved": "https://registry.npmjs.org/send/-/send-0.16.1.tgz", "integrity": "sha512-ElCLJdJIKPk6ux/Hocwhk7NFHpI3pVm/IZOYWqUmoxcgeyM+MpxHHKhb8QmlJDX1pU6WrgaHBkVNm73Sv7uc2A==", - "requires": { + "dependencies": { "debug": "2.6.9", "depd": "~1.1.1", "destroy": "~1.0.4", @@ -674,131 +866,183 @@ "range-parser": "~1.2.0", "statuses": "~1.3.1" }, - "dependencies": { - "statuses": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", - "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=" - } + "engines": { + "node": ">= 0.8.0" } }, - "serve-static": { + "node_modules/send/node_modules/statuses": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", + "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-static": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.1.tgz", "integrity": "sha512-hSMUZrsPa/I09VYFJwa627JJkNs0NrfL1Uzuup+GqHfToR2KcsXFymXSV90hoyw3M+msjFuQly+YzIH/q0MGlQ==", - "requires": { + "dependencies": { "encodeurl": "~1.0.1", "escape-html": "~1.0.3", "parseurl": "~1.3.2", "send": "0.16.1" + }, + "engines": { + "node": ">= 0.8.0" } }, - "setprototypeof": { + "node_modules/setprototypeof": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ=" }, - "statuses": { + "node_modules/statuses": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==", + "engines": { + "node": ">= 0.6" + } }, - "string_decoder": { + "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "requires": { - "safe-buffer": "~5.2.0" - }, "dependencies": { - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" - } + "safe-buffer": "~5.2.0" } }, - "tar-fs": { + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/tar-fs": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz", "integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==", - "requires": { + "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.0.0" } }, - "tar-stream": { + "node_modules/tar-stream": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz", "integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==", - "requires": { + "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" } }, - "through": { + "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, - "type-is": { + "node_modules/type-is": { "version": "1.6.16", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==", - "requires": { + "dependencies": { "media-typer": "0.3.0", "mime-types": "~2.1.18" + }, + "engines": { + "node": ">= 0.6" } }, - "unbzip2-stream": { + "node_modules/unbzip2-stream": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", - "requires": { + "dependencies": { "buffer": "^5.2.1", "through": "^2.3.8" } }, - "unpipe": { + "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "engines": { + "node": ">= 0.8" + } }, - "util-deprecate": { + "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, - "utils-merge": { + "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", + "engines": { + "node": ">= 0.4.0" + } }, - "vary": { + "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "engines": { + "node": ">= 0.8" + } }, - "wrappy": { + "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, - "ws": { + "node_modules/ws": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz", - "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==" + "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } }, - "yauzl": { + "node_modules/yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", - "requires": { + "dependencies": { "buffer-crc32": "~0.2.3", "fd-slicer": "~1.1.0" } diff --git a/tests/server/server.js b/tests/server/server.js index c20fa609..b0dd4550 100644 --- a/tests/server/server.js +++ b/tests/server/server.js @@ -136,6 +136,78 @@ app.get('/robots.txt', function (req, res) { res.end(html); }); +app.get('/sitemap_index.xml', function (req, res) { + var sitemapIndex = '\n' + + '\n' + + ' \n' + + ' http://localhost:8080/sitemap1.xml\n' + + ' 2024-01-01\n' + + ' \n' + + ' \n' + + ' http://localhost:8080/sitemap2.xml\n' + + ' 2024-01-01\n' + + ' \n' + + ''; + + res.contentType('text/xml; charset=utf-8'); + res.end(sitemapIndex); +}); + +app.get('/sitemap1.xml', function (req, res) { + var sitemap1 = '\n' + + '\n' + + ' \n' + + ' http://localhost:8080/\n' + + ' 2016-01-01\n' + + ' monthly\n' + + ' 0.8\n' + + ' \n' + + ' \n' + + ' http://localhost:8080/link1\n' + + ' 2016-01-01\n' + + ' monthly\n' + + ' 0.8\n' + + ' \n' + + ''; + + res.contentType('text/xml; charset=utf-8'); + res.end(sitemap1); +}); + +app.get('/sitemap2.xml', function (req, res) { + var sitemap2 = '\n' + + '\n' + + ' \n' + + ' http://localhost:8080/link1-next\n' + + ' 2016-01-01\n' + + ' monthly\n' + + ' 0.8\n' + + ' \n' + + ' \n' + + ' http://localhost:8080/link1-prev\n' + + ' 2016-01-01\n' + + ' monthly\n' + + ' 0.8\n' + + ' \n' + + ' \n' + + ' http://localhost:8080/link2\n' + + ' 2016-01-01\n' + + ' monthly\n' + + ' 0.8\n' + + ' \n' + + ' \n' + + ' http://localhost:8080/link3\n' + + ' 2016-01-01\n' + + ' monthly\n' + + ' 0.8\n' + + ' \n' + + ''; + + res.contentType('text/xml; charset=utf-8'); + res.end(sitemap2); +}); + + let server = app.listen(8080, function () { const host = 'localhost'; const port = server.address().port;