diff --git a/README.md b/README.md index 3c33908..7198583 100644 --- a/README.md +++ b/README.md @@ -155,8 +155,8 @@ yandere: ``` ## Download location -By default, pictures will download to working directory. -If you want to change download location, you can add FILES_STORE option to config. +By default, pictures will download to `${FAVORS_HOME}/{site_name}` +If you want to change download location, you can update FILES_STORE. For example, if you want save pixiv files to `pictures/a`, and want save yandere files to `pictures/b`, you can modify config file like this: ```yaml pixiv: diff --git a/src/favorites_crawler/commands/crawl.py b/src/favorites_crawler/commands/crawl.py index 97c969f..3170f50 100644 --- a/src/favorites_crawler/commands/crawl.py +++ b/src/favorites_crawler/commands/crawl.py @@ -72,7 +72,7 @@ def crawl(name, **kwargs): """ spider = spider_loader.load(name) favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME) - overwrite_spider_settings(spider, scrapy_settings, load_config(favors_home)) + overwrite_spider_settings(spider, favors_home, load_config(favors_home)) process = CrawlerProcess(scrapy_settings) process.crawl(spider, **kwargs) for crawler in process.crawlers: diff --git a/src/favorites_crawler/constants/path.py b/src/favorites_crawler/constants/path.py index 07f4619..37cfdcb 100644 --- a/src/favorites_crawler/constants/path.py +++ b/src/favorites_crawler/constants/path.py @@ -1 +1,3 @@ -DEFAULT_FAVORS_HOME = '~/.favorites_crawler' +import os + +DEFAULT_FAVORS_HOME = os.path.join('~', '.favorites_crawler') diff --git a/src/favorites_crawler/utils/config.py b/src/favorites_crawler/utils/config.py index c3aa23a..d4ab627 100644 --- a/src/favorites_crawler/utils/config.py +++ b/src/favorites_crawler/utils/config.py @@ -1,7 +1,8 @@ import os -import yaml from copy import deepcopy +import yaml + DEFAULT_CONFIG = { 'global': { @@ -10,28 +11,28 @@ 'EXIF_TOOL_EXECUTABLE': None, }, 'pixiv': { - 'FILES_STORE': 'favorites_crawler_files/pixiv', + 'FILES_STORE': os.path.join('$FAVORS_HOME', 'pixiv'), 'USER_ID': '', 'ACCESS_TOKEN': '', 'REFRESH_TOKEN': '', }, 'yandere': { - 'FILES_STORE': 'favorites_crawler_files/yandere', + 'FILES_STORE': os.path.join('$FAVORS_HOME', 'yandere'), 'USERNAME': '', }, 'twitter': { - 'FILES_STORE': 'favorites_crawler_files/twitter', + 'FILES_STORE': os.path.join('$FAVORS_HOME', 'twitter'), 'USER_ID': '', 'AUTHORIZATION': '', 'LIKES_ID': '', 'X_CSRF_TOKEN': '', }, 'lemon': { - 'FILES_STORE': 'favorites_crawler_files/lemon', + 'FILES_STORE': os.path.join('$FAVORS_HOME', 'lemon'), }, 'nhentai': { 'USER_AGENT': '', - 'FILES_STORE': 'favorites_crawler_files/nhentai', + 'FILES_STORE': os.path.join('$FAVORS_HOME', 'nhentai'), } } @@ -63,22 +64,26 @@ def create_favors_home(path: str): os.makedirs(path, exist_ok=True) -def overwrite_spider_settings(spider, default_settings, user_config): +def overwrite_spider_settings(spider, home, user_config): """ Overwrite spider settings by user config Priority: favors spider config > favors global config > spider custom settings > scrapy settings :param spider: Spider class - :param default_settings: :class:`scrapy.settings.Settings` + :param home: favors home :param user_config: favorites crawler config """ global_config = user_config.get('global') if global_config: spider.custom_settings.update(global_config) - spider_config = user_config.get(spider.name) + spider_config = user_config.get(spider.name, {}) if spider_config: spider.custom_settings.update(spider_config) - default_files_store = os.path.join(default_settings.get('FILES_STORE', ''), spider.name) - spider.custom_settings.setdefault('FILES_STORE', default_files_store) + home = os.path.expanduser(home) + files_store = spider_config.get('FILES_STORE') + if files_store: + spider.custom_settings['FILES_STORE'] = files_store.replace('$FAVORS_HOME', home) + else: + spider.custom_settings['FILES_STORE'] = os.path.join(home, spider.name) diff --git a/tests/test_utils/test_config.py b/tests/test_utils/test_config.py index b32203a..f0e60c5 100644 --- a/tests/test_utils/test_config.py +++ b/tests/test_utils/test_config.py @@ -84,7 +84,7 @@ def test_overwrite_spider_settings(self): } spider = spider_loader.load('pixiv') - overwrite_spider_settings(spider, scrapy_settings, user_config) + overwrite_spider_settings(spider, '~', user_config) assert spider.custom_settings['FILES_STORE'] == user_config['pixiv']['FILES_STORE'] assert spider.custom_settings['ENABLE_ORGANIZE_BY_ARTIST'] == user_config['global']['ENABLE_ORGANIZE_BY_ARTIST'] @@ -100,7 +100,7 @@ def test_spider_config_priority_should_gt_global_config(self): } spider = spider_loader.load('yandere') - overwrite_spider_settings(spider, scrapy_settings, user_config) + overwrite_spider_settings(spider, '~', user_config) assert spider.custom_settings['ENABLE_ORGANIZE_BY_ARTIST'] == user_config['yandere']['ENABLE_ORGANIZE_BY_ARTIST'] @@ -108,6 +108,14 @@ def test_should_set_default_file_store_when_user_doesnt_config_it(self): user_config = {} spider = spider_loader.load('nhentai') - overwrite_spider_settings(spider, scrapy_settings, user_config) + overwrite_spider_settings(spider, '~', user_config) - assert spider.custom_settings['FILES_STORE'] == os.path.join(scrapy_settings.get('FILES_STORE', ''), 'nhentai') + assert spider.custom_settings['FILES_STORE'] == os.path.expanduser(os.path.join('~', 'nhentai')) + + def test_should_replace_favors_home_in_files_store(self): + user_config = {'twitter': {'FILES_STORE': os.path.join('$FAVORS_HOME', 'twitter')}} + spider = spider_loader.load('twitter') + + overwrite_spider_settings(spider, '~', user_config) + + assert spider.custom_settings['FILES_STORE'] == os.path.expanduser(os.path.join('~', 'twitter'))