Add additional variable for the archive output URI

docs/settings.md

@@ -8,7 +8,7 @@
 
 ```python
 SW_EXPORT_URI = "s3://scrapy-webarchive/"
-SW_EXPORT_URI = "s3://scrapy-webarchive/{year}/{month}/{day}/"
+SW_EXPORT_URI = "s3://scrapy-webarchive/{year}/{month}/{day}/{spider}/"
 ```
 
 This is the output path of the WACZ file. Multiple variables can be added that allow dynamic generation of the output path. 

scrapy_webarchive/extensions.py

@@ -32,12 +32,15 @@ def __init__(self, settings: Settings, crawler: Crawler) -> None:
         if not self.settings["SW_EXPORT_URI"]:
             raise NotConfigured
 
-        self.store = self._get_store()
+        self.store = self._get_store(spider_name=crawler.spider.name)
         self.writer = WarcFileWriter(collection_name=crawler.spider.name)
 
-    def _get_store(self):
+    def _get_store(self, spider_name: str):
         archive_uri_template = self.settings["SW_EXPORT_URI"]
-        uri = archive_uri_template.format(**get_archive_uri_template_variables())
+        uri = archive_uri_template.format(**{
+            "spider": spider_name,
+            **get_archive_uri_template_dt_variables(),
+        })
         store_cls = self.STORE_SCHEMES[get_scheme_from_uri(uri)]
         return store_cls(uri)
 
@@ -105,7 +108,7 @@ def spider_closed(self, spider: Spider) -> None:
         WaczFileCreator(store=self.store, warc_fname=self.writer.warc_fname, collection_name=spider.name).create()
 
 
-def get_archive_uri_template_variables() -> dict:
+def get_archive_uri_template_dt_variables() -> dict:
     current_date = datetime.now()
 
     return {