add: exclude archive.org & archive-it.org by default

internetarchive · Aug 21, 2024 · 1257ff4 · 1257ff4
1 parent f3ae9cf
commit 1257ff4
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/internal/pkg/crawl/config.go b/internal/pkg/crawl/config.go
@@ -246,7 +246,10 @@ func GenerateCrawlConfig(config *config.Config) (*Crawl, error) {
 	c.DomainsCrawl = config.DomainsCrawl
 	c.DisableAssetsCapture = config.DisableAssetsCapture
 	c.DisabledHTMLTags = config.DisableHTMLTag
-	c.ExcludedHosts = config.ExcludeHosts
+
+	// We exclude some hosts by default
+	c.ExcludedHosts = utils.DedupeStrings(append(config.ExcludeHosts, "archive.org", "archive-it.org"))
+
 	c.IncludedHosts = config.IncludeHosts
 	c.CaptureAlternatePages = config.CaptureAlternatePages
 	c.ExcludedStrings = config.ExcludeString