From f31e1e078985108f2d8b334740c6ac423368dee0 Mon Sep 17 00:00:00 2001 From: wen Date: Mon, 2 Dec 2024 20:21:55 +0800 Subject: [PATCH 1/9] fix(spiders/x): fix cannot crawl pictures from x after login --- src/favorites_crawler/commands/crawl.py | 4 --- src/favorites_crawler/commands/login.py | 19 ++++-------- src/favorites_crawler/spiders/twitter.py | 30 ++++--------------- src/favorites_crawler/utils/auth.py | 20 +++++++++++++ tests/test_commands/test_login.py | 13 -------- tests/test_utils/test_auth.py | 38 ++++++++++++++++++++++++ 6 files changed, 69 insertions(+), 55 deletions(-) delete mode 100644 tests/test_commands/test_login.py create mode 100644 tests/test_utils/test_auth.py diff --git a/src/favorites_crawler/commands/crawl.py b/src/favorites_crawler/commands/crawl.py index 7c3b072..a550ba2 100644 --- a/src/favorites_crawler/commands/crawl.py +++ b/src/favorites_crawler/commands/crawl.py @@ -76,7 +76,3 @@ def crawl(name, **kwargs): for crawler in process.crawlers: crawler.signals.connect(spider_closed, signal=signals.spider_closed) process.start() - - -if __name__ == '__main__': - crawl('pixiv') diff --git a/src/favorites_crawler/commands/login.py b/src/favorites_crawler/commands/login.py index b495289..899d0eb 100644 --- a/src/favorites_crawler/commands/login.py +++ b/src/favorites_crawler/commands/login.py @@ -1,13 +1,11 @@ -import re from typing import Optional -from urllib.parse import unquote from webbrowser import open as open_url import typer from selenium.common import NoSuchWindowException from favorites_crawler.constants.endpoints import TWITTER_PROFILE_LIKES_URL -from favorites_crawler.utils.auth import CustomGetPixivToken +from favorites_crawler.utils.auth import CustomGetPixivToken, parse_twitter_likes_url, parser_twitter_likes_features from favorites_crawler.utils.config import dump_config, load_config @@ -98,16 +96,11 @@ def login_twitter( try: twitter_config['AUTHORIZATION'] = input('Authorization: ') twitter_config['X_CSRF_TOKEN'] = input('X-Csrf-Token: ') - twitter_config['LIKES_ID'], twitter_config['USER_ID'] = parse_twitter_likes_url(input('Request URL: ')) - except KeyboardInterrupt: - "Failed to login." + url = input('Request URL: ') + twitter_config['LIKES_ID'], twitter_config['USER_ID'] = parse_twitter_likes_url(url) + twitter_config['FEATURES'] = parser_twitter_likes_features(url) + except Exception as e: + print(f"Failed to login: {e!r}") return dump_config(config) print("Login successful.") - - -def parse_twitter_likes_url(url): - """Parse USER_ID and LIKES_ID from URL""" - url = unquote(url).replace(' ', '') - match = re.match(r'^.+?graphql/(.+?)/.+?userId":"(.+?)".+$', url) - return match.groups() diff --git a/src/favorites_crawler/spiders/twitter.py b/src/favorites_crawler/spiders/twitter.py index d0e2506..9516879 100644 --- a/src/favorites_crawler/spiders/twitter.py +++ b/src/favorites_crawler/spiders/twitter.py @@ -18,6 +18,7 @@ class TwitterSpider(BaseSpider): custom_settings = { 'CONCURRENT_REQUESTS': 2, 'ITEM_PIPELINES': {'favorites_crawler.pipelines.PicturePipeline': 0}, + 'HTTPERROR_ALLOWED_CODES': [400], } @property @@ -41,31 +42,7 @@ def __init__(self, *args, **kwargs): "withVoice": True, "withV2Timeline": True } - self.features = { - "rweb_tipjar_consumption_enabled": True, - "responsive_web_graphql_exclude_directive_enabled": True, - "verified_phone_label_enabled": False, "creator_subscriptions_tweet_preview_api_enabled": True, - "responsive_web_graphql_timeline_navigation_enabled": True, - "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False, - "communities_web_enable_tweet_community_results_fetch": True, - "c9s_tweet_anatomy_moderator_badge_enabled": True, - "articles_preview_enabled": True, - "tweetypie_unmention_optimization_enabled": True, - "responsive_web_edit_tweet_api_enabled": True, - "graphql_is_translatable_rweb_tweet_is_translatable_enabled": True, - "view_counts_everywhere_api_enabled": True, - "longform_notetweets_consumption_enabled": True, - "responsive_web_twitter_article_tweet_consumption_enabled": True, - "tweet_awards_web_tipping_enabled": False, - "creator_subscriptions_quote_tweet_preview_enabled": False, - "freedom_of_speech_not_reach_fetch_enabled": True, - "standardized_nudges_misinfo": True, - "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": True, - "rweb_video_timestamps_enabled": True, - "longform_notetweets_rich_text_read_enabled": True, - "longform_notetweets_inline_media_enabled": True, - "responsive_web_enhance_cards_enabled": False - } + self.features = self.custom_settings.get('FEATURES', {}) self.headers = { 'Authorization': self.custom_settings.get('AUTHORIZATION'), 'x-csrf-token': self.custom_settings.get('X_CSRF_TOKEN'), @@ -79,6 +56,9 @@ def parse_start_url(self, response, **kwargs): yield item_or_request def parse(self, response, **kwargs): + if response.status == 400: + self.logger.error('Failed to request x API, error message: %s', response.json()) + entries = ( entry['content'] for entry in DictRouter(response.json()).route_to( 'data.user.result.timeline_v2.timeline.instructions.0.entries', [], diff --git a/src/favorites_crawler/utils/auth.py b/src/favorites_crawler/utils/auth.py index 803e3c7..b4fe383 100644 --- a/src/favorites_crawler/utils/auth.py +++ b/src/favorites_crawler/utils/auth.py @@ -1,3 +1,7 @@ +import json +import re +from urllib.parse import unquote + from gppt import GetPixivToken from gppt.consts import REDIRECT_URI from selenium.common import TimeoutException @@ -31,3 +35,19 @@ def refresh_pixiv(): pixiv_config['ACCESS_TOKEN'] = access_token dump_config(config) return access_token + + +def parse_twitter_likes_url(url): + """Parse USER_ID and LIKES_ID from URL""" + url = unquote(url).replace(' ', '') + match = re.match(r'^.+?graphql/(.+?)/.+?userId":"(.+?)".+$', url) + return match.groups() + + +def parser_twitter_likes_features(url): + url = unquote(url).replace(' ', '') + features = re.match(r'^.+features=(\{.+?}).+$', url) + if features: + print(features.group(1)) + features = json.loads(features.group(1)) + return features diff --git a/tests/test_commands/test_login.py b/tests/test_commands/test_login.py deleted file mode 100644 index d02db85..0000000 --- a/tests/test_commands/test_login.py +++ /dev/null @@ -1,13 +0,0 @@ -import pytest - -from favorites_crawler.commands.login import parse_twitter_likes_url - - -@pytest.mark.parametrize('url, expected', ( - ( - 'https://twitter.com/i/api/graphql/xxxx/Likes?variables=%7B%22userId%22%3A%22xxxx%22%2C%22...', - ('xxxx', 'xxxx'), - ), -)) -def test_twitter_parse_likes_url(url, expected): - assert parse_twitter_likes_url(url) == expected diff --git a/tests/test_utils/test_auth.py b/tests/test_utils/test_auth.py new file mode 100644 index 0000000..4465f9a --- /dev/null +++ b/tests/test_utils/test_auth.py @@ -0,0 +1,38 @@ +import pytest + +from favorites_crawler.utils.auth import parse_twitter_likes_url, parser_twitter_likes_features + + +def test_parser_twitter_likes_features(): + url = 'https://x.com/i/api/graphql/xxx/Likes?variables=%7B%22userId%22%3A%22xxx%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withClientEventToken%22%3Afalse%2C%22withBirdwatchNotes%22%3Afalse%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22profile_label_improvements_pcf_label_in_post_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D' + + features = parser_twitter_likes_features(url) + + assert features == {'profile_label_improvements_pcf_label_in_post_enabled': False, + 'rweb_tipjar_consumption_enabled': True, + 'responsive_web_graphql_exclude_directive_enabled': True, 'verified_phone_label_enabled': False, + 'creator_subscriptions_tweet_preview_api_enabled': True, + 'responsive_web_graphql_timeline_navigation_enabled': True, + 'responsive_web_graphql_skip_user_profile_image_extensions_enabled': False, + 'communities_web_enable_tweet_community_results_fetch': True, + 'c9s_tweet_anatomy_moderator_badge_enabled': True, 'articles_preview_enabled': True, + 'responsive_web_edit_tweet_api_enabled': True, + 'graphql_is_translatable_rweb_tweet_is_translatable_enabled': True, + 'view_counts_everywhere_api_enabled': True, 'longform_notetweets_consumption_enabled': True, + 'responsive_web_twitter_article_tweet_consumption_enabled': True, + 'tweet_awards_web_tipping_enabled': False, + 'creator_subscriptions_quote_tweet_preview_enabled': False, + 'freedom_of_speech_not_reach_fetch_enabled': True, 'standardized_nudges_misinfo': True, + 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': True, + 'rweb_video_timestamps_enabled': True, 'longform_notetweets_rich_text_read_enabled': True, + 'longform_notetweets_inline_media_enabled': True, 'responsive_web_enhance_cards_enabled': False} + + +@pytest.mark.parametrize('url, expected', ( + ( + 'https://twitter.com/i/api/graphql/xxxx/Likes?variables=%7B%22userId%22%3A%22xxxx%22%2C%22...', + ('xxxx', 'xxxx'), + ), +)) +def test_twitter_parse_likes_url(url, expected): + assert parse_twitter_likes_url(url) == expected From faba7d589ba3b37f907140fcbd414bdd99b4a36c Mon Sep 17 00:00:00 2001 From: wen Date: Mon, 2 Dec 2024 21:12:44 +0800 Subject: [PATCH 2/9] refactor(spiders/x): refactor favors login twitter command --- README.md | 41 ++++++++++++-------- src/favorites_crawler/commands/login.py | 38 +++++++++++------- src/favorites_crawler/constants/endpoints.py | 1 - src/favorites_crawler/constants/path.py | 3 ++ src/favorites_crawler/utils/auth.py | 1 - src/favorites_crawler/utils/config.py | 3 +- 6 files changed, 53 insertions(+), 34 deletions(-) create mode 100644 src/favorites_crawler/constants/path.py diff --git a/README.md b/README.md index 6634990..f8d1542 100644 --- a/README.md +++ b/README.md @@ -64,11 +64,10 @@ favors login --help ``` ## Login Yandere -1. run command: - ``` - favors login yandere - ``` -2. input your username and hit the Enter key. +run command: +``` +favors login yandere -u {username} +``` ## Login NHentai 1. Open nhentai on browser and login. @@ -76,19 +75,27 @@ favors login --help 3. Copy cookie file to {user_home}/.favorites_crawler. ## Login Twitter -1. run command - ``` - favors login twitter - ``` -2. input your username, after press Enter, likes page will open in browser. -3. Open dev console (F12) and switch to network tab. -4. Enable persistent logging ("Preserve log"). -5. Type into the filter field: Likes? -6. Refresh Page. -7. Copy Authorization, X-Csrf-Token and RequestURL from request(Likes?variables...) input on terminal. -8. Use "Get cookies.txt" browser extension download cookie file. -9. Copy cookie file to {user_home}/.favorites_crawler. +1. Get Authorization, X-Csrf-Token RequestURL and Cookie File + 1. Open [x.com](https://x.com/) and login, get to your "Likes" page + 2. Open dev console (F12) and switch to network tab. + 3. Enable persistent logging ("Preserve log"). + 4. Type into the filter field: Likes? + 5. Refresh Page. + 6. Copy Authorization, X-Csrf-Token and RequestURL from request(Likes?variables...) + 7. Use "Get cookies.txt" browser extension download cookie file. +2. Execute command: + ```commandline + favors login x -at "{Authorization}" -ct "{X-Csrf-Token}" -u "{RequestURL}" -c "{Cookie File}" + ``` +Example: +```commandline +favors login x -at "Bearer AAAAAAAAAAAAA..." -ct ... -u "https://x.com/i/api/graphql/.../Likes?..." -c "C:\Users\xxx\Downloads\x.com_cookies.txt" +``` + +Note: Request URL will make the entire command very long. +If you cannot enter such a long command in the macOS terminal, +you can write the command in a sh file and execute it. # Crawl diff --git a/src/favorites_crawler/commands/login.py b/src/favorites_crawler/commands/login.py index 899d0eb..f205b59 100644 --- a/src/favorites_crawler/commands/login.py +++ b/src/favorites_crawler/commands/login.py @@ -1,12 +1,12 @@ +import shutil from typing import Optional -from webbrowser import open as open_url import typer from selenium.common import NoSuchWindowException -from favorites_crawler.constants.endpoints import TWITTER_PROFILE_LIKES_URL from favorites_crawler.utils.auth import CustomGetPixivToken, parse_twitter_likes_url, parser_twitter_likes_features from favorites_crawler.utils.config import dump_config, load_config +from favorites_crawler.constants.path import DEFAULT_FAVORS_HOME app = typer.Typer(help='Prepare auth information for crawling.', no_args_is_help=True) @@ -73,32 +73,42 @@ def login_yandere( @app.command('x') @app.command('twitter') def login_twitter( - username: str = typer.Option( - ..., '-u', '--username', - help="Your twitter username." + auth_token: str = typer.Option( + ..., '-at', '--auth-token', + help='Authorization Token (Copy from Dev console)' + ), + csrf_token: str = typer.Option( + ..., '-ct', '--csrf-token', + help='Authorization Token (Copy from Dev console)' + ), + likes_url: str = typer.Option( + ..., '-u', '--likes-url', + help='Request URL of Likes API (Copy from Dev console)' + ), + cookie_file: str = typer.Option( + ..., '-c', '--cookie-file', + help='Netscape HTTP Cookie File, you can download it by "Get cookies.txt" browser extension.' ) ): """ Login to twitter. - 1. After execute this command, likes page will open in browser.\n + 1. Open twitter and login, get to your "Likes" page.\n 2. Open dev console (F12) and switch to network tab.\n 3. Enable persistent logging ("Preserve log").\n 4. Type into the filter field: Likes?\n 5. Refresh Page.\n 6. Copy Authorization, X-Csrf-Token and RequestURL from request(Likes?variables...) input on terminal.\n - 7. Use "Get cookies.txt" browser extension download cookie file.\n - 8. Copy cookie file to {user_home}/.favorites_crawler. + 7. Use "Get cookies.txt" browser extension download cookie file. """ - open_url(TWITTER_PROFILE_LIKES_URL.format(username=username)) config = load_config() twitter_config = config.setdefault('twitter', {}) try: - twitter_config['AUTHORIZATION'] = input('Authorization: ') - twitter_config['X_CSRF_TOKEN'] = input('X-Csrf-Token: ') - url = input('Request URL: ') - twitter_config['LIKES_ID'], twitter_config['USER_ID'] = parse_twitter_likes_url(url) - twitter_config['FEATURES'] = parser_twitter_likes_features(url) + twitter_config['AUTHORIZATION'] = auth_token + twitter_config['X_CSRF_TOKEN'] = csrf_token + twitter_config['LIKES_ID'], twitter_config['USER_ID'] = parse_twitter_likes_url(likes_url) + twitter_config['FEATURES'] = parser_twitter_likes_features(likes_url) + shutil.copy(cookie_file, DEFAULT_FAVORS_HOME) except Exception as e: print(f"Failed to login: {e!r}") return diff --git a/src/favorites_crawler/constants/endpoints.py b/src/favorites_crawler/constants/endpoints.py index eeb9b2d..45de8dd 100644 --- a/src/favorites_crawler/constants/endpoints.py +++ b/src/favorites_crawler/constants/endpoints.py @@ -11,4 +11,3 @@ NHENTAI_USER_FAVORITES_URL = 'https://nhentai.net/favorites/' TWITTER_LIKES_URL = 'https://x.com/i/api/graphql/{id}/Likes' -TWITTER_PROFILE_LIKES_URL = 'https://x.com/{username}/likes' diff --git a/src/favorites_crawler/constants/path.py b/src/favorites_crawler/constants/path.py new file mode 100644 index 0000000..89d3598 --- /dev/null +++ b/src/favorites_crawler/constants/path.py @@ -0,0 +1,3 @@ +import os + +DEFAULT_FAVORS_HOME = os.path.expanduser('~/.favorites_crawler') diff --git a/src/favorites_crawler/utils/auth.py b/src/favorites_crawler/utils/auth.py index b4fe383..85762f6 100644 --- a/src/favorites_crawler/utils/auth.py +++ b/src/favorites_crawler/utils/auth.py @@ -48,6 +48,5 @@ def parser_twitter_likes_features(url): url = unquote(url).replace(' ', '') features = re.match(r'^.+features=(\{.+?}).+$', url) if features: - print(features.group(1)) features = json.loads(features.group(1)) return features diff --git a/src/favorites_crawler/utils/config.py b/src/favorites_crawler/utils/config.py index 1af06b8..b100a50 100644 --- a/src/favorites_crawler/utils/config.py +++ b/src/favorites_crawler/utils/config.py @@ -1,7 +1,8 @@ import os import yaml -DEFAULT_FAVORS_HOME = os.path.expanduser('~/.favorites_crawler') +from favorites_crawler.constants.path import DEFAULT_FAVORS_HOME + DEFAULT_CONFIG = { 'global': { 'ENABLE_ORGANIZE_BY_ARTIST': True, From 72174b7ec12a9dae0b95d083c84bf6be294fc047 Mon Sep 17 00:00:00 2001 From: wen Date: Mon, 2 Dec 2024 21:29:43 +0800 Subject: [PATCH 3/9] feature(spider/nhentai): add favors login nhentai command --- README.md | 13 +++++++--- src/favorites_crawler/commands/login.py | 32 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f8d1542..cd70a7f 100644 --- a/README.md +++ b/README.md @@ -70,9 +70,16 @@ favors login yandere -u {username} ``` ## Login NHentai -1. Open nhentai on browser and login. -2. Use "Get cookies.txt" browser extension download cookie file. -3. Copy cookie file to {user_home}/.favorites_crawler. +1. Get User-Agent and Cookie File + 1. Open nhentai and login. + 2. Open dev console (F12) and switch to network tab. + 3. Open any comic. + 4. Copy user-agent from any request. + 5. Use "Get cookies.txt" browser extension download cookie file. +2. Execute command: + ```commandline + favors login nhentai -u "{User-Agent}" -c "{Cookie File}" + ``` ## Login Twitter diff --git a/src/favorites_crawler/commands/login.py b/src/favorites_crawler/commands/login.py index f205b59..2ad09d0 100644 --- a/src/favorites_crawler/commands/login.py +++ b/src/favorites_crawler/commands/login.py @@ -114,3 +114,35 @@ def login_twitter( return dump_config(config) print("Login successful.") + + +@app.command("nhentai") +def login_nhentai( + user_agent: str = typer.Option( + ..., '-u', '--user-agent', + help='User Agent' + ), + cookie_file: str = typer.Option( + ..., '-c', '--cookie-file', + help='Netscape HTTP Cookie File, you can download it by "Get cookies.txt" browser extension.' + ) +): + """ + Login to nhentai. + + 1. Open nhentai and login.\n + 2. Open dev console (F12) and switch to network tab.\n + 3. Open any comic.\n + 4. Copy user-agent from any request.\n + 5. Use "Get cookies.txt" browser extension download cookie file. + """ + config = load_config() + nhentai_config = config.setdefault('nhentai', {}) + try: + nhentai_config['USER_AGENT'] = user_agent + shutil.copy(cookie_file, DEFAULT_FAVORS_HOME) + except Exception as e: + print(f"Failed to login: {e!r}") + return + dump_config(config) + print("Login successful.") From f8b747deccbc5459f1e5659f94f529a0a63d4d50 Mon Sep 17 00:00:00 2001 From: Wen Liang Date: Tue, 3 Dec 2024 17:18:00 +0800 Subject: [PATCH 4/9] refactor: read FAVORS_HOME from env, config save to FAVORS_HOME, default FAVORS_HOME is ~/.favorites_crawler --- src/favorites_crawler/commands/crawl.py | 4 ++- src/favorites_crawler/commands/login.py | 36 +++++++++++++---------- src/favorites_crawler/utils/config.py | 5 ++-- tests/test_commands/test_login.py | 39 +++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 20 deletions(-) create mode 100644 tests/test_commands/test_login.py diff --git a/src/favorites_crawler/commands/crawl.py b/src/favorites_crawler/commands/crawl.py index a550ba2..97c969f 100644 --- a/src/favorites_crawler/commands/crawl.py +++ b/src/favorites_crawler/commands/crawl.py @@ -9,6 +9,7 @@ from scrapy.spiderloader import SpiderLoader from favorites_crawler.utils.config import load_config, overwrite_spider_settings +from favorites_crawler.constants.path import DEFAULT_FAVORS_HOME app = typer.Typer(help='Crawl your favorites from websites.', no_args_is_help=True) @@ -70,7 +71,8 @@ def crawl(name, **kwargs): :param kwargs: kwargs passed to spider's __init__ method """ spider = spider_loader.load(name) - overwrite_spider_settings(spider, scrapy_settings, load_config()) + favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME) + overwrite_spider_settings(spider, scrapy_settings, load_config(favors_home)) process = CrawlerProcess(scrapy_settings) process.crawl(spider, **kwargs) for crawler in process.crawlers: diff --git a/src/favorites_crawler/commands/login.py b/src/favorites_crawler/commands/login.py index 2ad09d0..a46d4f3 100644 --- a/src/favorites_crawler/commands/login.py +++ b/src/favorites_crawler/commands/login.py @@ -1,8 +1,8 @@ +import os import shutil from typing import Optional import typer -from selenium.common import NoSuchWindowException from favorites_crawler.utils.auth import CustomGetPixivToken, parse_twitter_likes_url, parser_twitter_likes_features from favorites_crawler.utils.config import dump_config, load_config @@ -33,13 +33,14 @@ def login_pixiv( If you do not provide your username and password, you will login manually on the web page """ - config = load_config() + favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME) + config = load_config(favors_home) token_getter = CustomGetPixivToken() try: login_info = token_getter.login(username=username, password=password) - except NoSuchWindowException: - print('Failed to login.') - return + except Exception as e: + print(f'Failed to login. {e!r}') + exit(1) pixiv_config = config.setdefault('pixiv', {}) try: @@ -49,7 +50,7 @@ def login_pixiv( except KeyError as e: print(f'Failed to login. {e!r}') else: - dump_config(config) + dump_config(config, favors_home) print("Login successful.") @@ -63,10 +64,11 @@ def login_yandere( """ Login to yandere. """ - config = load_config() + favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME) + config = load_config(favors_home) yandere_config = config.setdefault('yandere', {}) yandere_config['USERNAME'] = username - dump_config(config) + dump_config(config, favors_home) print("Login successful.") @@ -101,18 +103,19 @@ def login_twitter( 6. Copy Authorization, X-Csrf-Token and RequestURL from request(Likes?variables...) input on terminal.\n 7. Use "Get cookies.txt" browser extension download cookie file. """ - config = load_config() + favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME) + config = load_config(favors_home) twitter_config = config.setdefault('twitter', {}) try: twitter_config['AUTHORIZATION'] = auth_token twitter_config['X_CSRF_TOKEN'] = csrf_token twitter_config['LIKES_ID'], twitter_config['USER_ID'] = parse_twitter_likes_url(likes_url) twitter_config['FEATURES'] = parser_twitter_likes_features(likes_url) - shutil.copy(cookie_file, DEFAULT_FAVORS_HOME) + shutil.copy(cookie_file, favors_home) except Exception as e: print(f"Failed to login: {e!r}") - return - dump_config(config) + exit(1) + dump_config(config, favors_home) print("Login successful.") @@ -136,13 +139,14 @@ def login_nhentai( 4. Copy user-agent from any request.\n 5. Use "Get cookies.txt" browser extension download cookie file. """ - config = load_config() + favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME) + config = load_config(favors_home) nhentai_config = config.setdefault('nhentai', {}) try: nhentai_config['USER_AGENT'] = user_agent - shutil.copy(cookie_file, DEFAULT_FAVORS_HOME) + shutil.copy(cookie_file, favors_home) except Exception as e: print(f"Failed to login: {e!r}") - return - dump_config(config) + exit(1) + dump_config(config, favors_home) print("Login successful.") diff --git a/src/favorites_crawler/utils/config.py b/src/favorites_crawler/utils/config.py index b100a50..ff63751 100644 --- a/src/favorites_crawler/utils/config.py +++ b/src/favorites_crawler/utils/config.py @@ -1,7 +1,6 @@ import os import yaml -from favorites_crawler.constants.path import DEFAULT_FAVORS_HOME DEFAULT_CONFIG = { 'global': { @@ -35,7 +34,7 @@ } -def load_config(home: str = DEFAULT_FAVORS_HOME) -> dict: +def load_config(home: str) -> dict: """Load config from user home""" create_favors_home(home) config_file = os.path.join(home, 'config.yml') @@ -46,7 +45,7 @@ def load_config(home: str = DEFAULT_FAVORS_HOME) -> dict: return yaml.safe_load(f) -def dump_config(data: dict, home: str = DEFAULT_FAVORS_HOME): +def dump_config(data: dict, home): """Dump config data to user home""" create_favors_home(home) config_file = os.path.join(home, 'config.yml') diff --git a/tests/test_commands/test_login.py b/tests/test_commands/test_login.py new file mode 100644 index 0000000..04af5e1 --- /dev/null +++ b/tests/test_commands/test_login.py @@ -0,0 +1,39 @@ +from typer.testing import CliRunner + +from favorites_crawler.commands.login import app +from favorites_crawler.utils.config import load_config + +runner = CliRunner() + + +class TestLoginNhentai: + def test_login_nhentai_success(self, tmp_path): + favors_home = tmp_path / 'home' + cookie = tmp_path / 'cookie.txt' + cookie.touch() + user_agent = 'Test-User-Agent' + + result = runner.invoke( + app, ['nhentai', '-c', str(cookie), '-u', user_agent], + env={'FAVORS_HOME': str(favors_home)} + ) + + assert result.exit_code == 0 + assert "success" in result.stdout + assert (favors_home / 'cookie.txt').exists() + assert (favors_home / 'config.yml').exists() + config = load_config(favors_home) + assert config['nhentai']['USER_AGENT'] == user_agent + + def test_login_nhentai_failed(self, tmp_path): + favors_home = tmp_path / 'home' + cookie = tmp_path / 'cookie.txt' + user_agent = 'Test-User-Agent' + + result = runner.invoke( + app, ['nhentai', '-c', str(cookie), '-u', user_agent], + env={'FAVORS_HOME': str(favors_home)} + ) + + assert result.exit_code == 1 + assert "Failed" in result.stdout From 30f0102e885a62c33b3d6469015af7e25c7ca534 Mon Sep 17 00:00:00 2001 From: Wen Liang Date: Tue, 3 Dec 2024 17:30:01 +0800 Subject: [PATCH 5/9] doc: update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cd70a7f..3c33908 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,11 @@ favors crawl twitter ``` # Config -Config file locate on `{your_home}/.favorites_crawler/config.yml`. + +Config file `config.yml` locate on `FAVORS_HOME`, +by default `FAVORS_HOME` is `{your_home}/.favorites_crawler`. +You can change `FAVORS_HOME` by set environment variable. + You can set any [scrapy built-in settings](https://docs.scrapy.org/en/latest/topics/settings.html#built-in-settings-reference) in this file. By default, file content likes this: From 468dbb97d3db5330a37c91ce0e11648849223b85 Mon Sep 17 00:00:00 2001 From: Wen Liang Date: Tue, 3 Dec 2024 20:17:58 +0800 Subject: [PATCH 6/9] test: test favors login twitter --- src/favorites_crawler/utils/config.py | 4 +- tests/test_commands/test_login.py | 110 ++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 9 deletions(-) diff --git a/src/favorites_crawler/utils/config.py b/src/favorites_crawler/utils/config.py index ff63751..315b809 100644 --- a/src/favorites_crawler/utils/config.py +++ b/src/favorites_crawler/utils/config.py @@ -1,5 +1,6 @@ import os import yaml +from copy import deepcopy DEFAULT_CONFIG = { @@ -29,6 +30,7 @@ 'FILES_STORE': 'favorites_crawler_files/lemon', }, 'nhentai': { + 'USER_AGENT': '', 'FILES_STORE': 'favorites_crawler_files/nhentai', } } @@ -40,7 +42,7 @@ def load_config(home: str) -> dict: config_file = os.path.join(home, 'config.yml') if not os.path.exists(config_file): dump_config(DEFAULT_CONFIG, home) - return DEFAULT_CONFIG + return deepcopy(DEFAULT_CONFIG) with open(config_file, encoding='utf8') as f: return yaml.safe_load(f) diff --git a/tests/test_commands/test_login.py b/tests/test_commands/test_login.py index 04af5e1..b609b61 100644 --- a/tests/test_commands/test_login.py +++ b/tests/test_commands/test_login.py @@ -1,20 +1,21 @@ from typer.testing import CliRunner -from favorites_crawler.commands.login import app +from favorites_crawler import app from favorites_crawler.utils.config import load_config runner = CliRunner() class TestLoginNhentai: + user_agent = 'Test-User-Agent' + def test_login_nhentai_success(self, tmp_path): favors_home = tmp_path / 'home' - cookie = tmp_path / 'cookie.txt' - cookie.touch() - user_agent = 'Test-User-Agent' + cookie_file = tmp_path / 'cookie.txt' + cookie_file.touch() result = runner.invoke( - app, ['nhentai', '-c', str(cookie), '-u', user_agent], + app, ['login', 'nhentai', '-c', str(cookie_file), '-u', self.user_agent], env={'FAVORS_HOME': str(favors_home)} ) @@ -23,17 +24,110 @@ def test_login_nhentai_success(self, tmp_path): assert (favors_home / 'cookie.txt').exists() assert (favors_home / 'config.yml').exists() config = load_config(favors_home) - assert config['nhentai']['USER_AGENT'] == user_agent + assert config['nhentai']['USER_AGENT'] == self.user_agent def test_login_nhentai_failed(self, tmp_path): favors_home = tmp_path / 'home' cookie = tmp_path / 'cookie.txt' - user_agent = 'Test-User-Agent' result = runner.invoke( - app, ['nhentai', '-c', str(cookie), '-u', user_agent], + app, ['login', 'nhentai', '-c', str(cookie), '-u', self.user_agent], env={'FAVORS_HOME': str(favors_home)} ) assert result.exit_code == 1 assert "Failed" in result.stdout + assert not (favors_home / 'cookie.txt').exists() + assert (favors_home / 'config.yml').exists() + config = load_config(favors_home) + assert config['nhentai']['USER_AGENT'] == '' + + +class TestLoginTwitter: + good_url = 'https://x.com/i/api/graphql/likes_id/Likes?variables=%7B%22userId%22%3A%22xxx%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withClientEventToken%22%3Afalse%2C%22withBirdwatchNotes%22%3Afalse%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22profile_label_improvements_pcf_label_in_post_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D' + bad_url = 'https://x.com/i/api/graphql/likion_feA%2C%22rweeabled%se%7D' + access_token = '"Bearer token"' + csrf_token = 'token' + + def test_login_twitter_success(self, tmp_path): + favors_home = tmp_path / 'home' + cookie_file = tmp_path / 'cookie.txt' + cookie_file.touch() + + result = runner.invoke( + app, ['login', 'x', '-at', self.access_token, '-ct', + self.csrf_token, '-u', self.good_url, '-c', str(cookie_file)], + env={'FAVORS_HOME': str(favors_home)} + ) + + assert result.exit_code == 0 + assert "success" in result.stdout + assert (favors_home / 'cookie.txt').exists() + assert (favors_home / 'config.yml').exists() + config = load_config(favors_home) + assert config['twitter']['AUTHORIZATION'] == self.access_token + assert config['twitter']['X_CSRF_TOKEN'] == self.csrf_token + assert config['twitter']['LIKES_ID'] == 'likes_id' + assert config['twitter']['FEATURES'] == { + 'profile_label_improvements_pcf_label_in_post_enabled': False, + 'rweb_tipjar_consumption_enabled': True, + 'responsive_web_graphql_exclude_directive_enabled': True, + 'verified_phone_label_enabled': False, + 'creator_subscriptions_tweet_preview_api_enabled': True, + 'responsive_web_graphql_timeline_navigation_enabled': True, + 'responsive_web_graphql_skip_user_profile_image_extensions_enabled': False, + 'communities_web_enable_tweet_community_results_fetch': True, + 'c9s_tweet_anatomy_moderator_badge_enabled': True, + 'articles_preview_enabled': True, + 'responsive_web_edit_tweet_api_enabled': True, + 'graphql_is_translatable_rweb_tweet_is_translatable_enabled': True, + 'view_counts_everywhere_api_enabled': True, + 'longform_notetweets_consumption_enabled': True, + 'responsive_web_twitter_article_tweet_consumption_enabled': True, + 'tweet_awards_web_tipping_enabled': False, + 'creator_subscriptions_quote_tweet_preview_enabled': False, + 'freedom_of_speech_not_reach_fetch_enabled': True, + 'standardized_nudges_misinfo': True, + 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': True, + 'rweb_video_timestamps_enabled': True, + 'longform_notetweets_rich_text_read_enabled': True, + 'longform_notetweets_inline_media_enabled': True, + 'responsive_web_enhance_cards_enabled': False + } + assert config['twitter']['USER_ID'] == 'xxx' + + def test_login_twitter_failed_when_given_bad_url(self, tmp_path): + favors_home = tmp_path / 'home' + cookie_file = tmp_path / 'cookie.txt' + cookie_file.touch() + + result = runner.invoke( + app, ['login', 'x', '-at', self.access_token, + '-ct', self.csrf_token, '-u', self.bad_url, '-c', str(cookie_file)], + env={'FAVORS_HOME': str(favors_home)} + ) + + self.assert_login_failed(result, favors_home) + + def test_login_twitter_failed_when_cookie_not_exists(self, tmp_path): + favors_home = tmp_path / 'home' + cookie_file = tmp_path / 'cookie.txt' + + result = runner.invoke( + app, ['login', 'x', '-at', self.access_token, + '-ct', self.csrf_token, '-u', self.good_url, '-c', str(cookie_file)], + env={'FAVORS_HOME': str(favors_home)} + ) + + self.assert_login_failed(result, favors_home) + + def assert_login_failed(self, result, favors_home): + assert result.exit_code == 1 + assert "Failed" in result.stdout + assert not (favors_home / 'cookie.txt').exists() + assert (favors_home / 'config.yml').exists() + config = load_config(favors_home) + assert config['twitter']['AUTHORIZATION'] == '' + assert config['twitter']['X_CSRF_TOKEN'] == '' + assert config['twitter']['LIKES_ID'] == '' + assert config['twitter']['USER_ID'] == '' From af64a2a5c10afdb7ca26ff97717d119d277dd759 Mon Sep 17 00:00:00 2001 From: Wen Liang Date: Tue, 3 Dec 2024 20:21:48 +0800 Subject: [PATCH 7/9] test: test favors login yandere --- tests/test_commands/test_login.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_commands/test_login.py b/tests/test_commands/test_login.py index b609b61..721e2b8 100644 --- a/tests/test_commands/test_login.py +++ b/tests/test_commands/test_login.py @@ -6,6 +6,23 @@ runner = CliRunner() +class TestLoginYandere: + def test_login_yandere_success(self, tmp_path): + favors_home = tmp_path / 'home' + username = 'username' + + result = runner.invoke( + app, ['login', 'yandere', '-u', username], + env={'FAVORS_HOME': str(favors_home)} + ) + + assert result.exit_code == 0 + assert "success" in result.stdout + assert (favors_home / 'config.yml').exists() + config = load_config(favors_home) + assert config['yandere']['USERNAME'] == username + + class TestLoginNhentai: user_agent = 'Test-User-Agent' From 6cd525dcbcd2d90b8b13f24c5507b10c9218a72a Mon Sep 17 00:00:00 2001 From: Wen Liang Date: Tue, 3 Dec 2024 20:50:57 +0800 Subject: [PATCH 8/9] test: test favors login pixiv --- src/favorites_crawler/commands/login.py | 1 + tests/test_commands/test_login.py | 107 ++++++++++++++++++++---- 2 files changed, 90 insertions(+), 18 deletions(-) diff --git a/src/favorites_crawler/commands/login.py b/src/favorites_crawler/commands/login.py index a46d4f3..b030a19 100644 --- a/src/favorites_crawler/commands/login.py +++ b/src/favorites_crawler/commands/login.py @@ -49,6 +49,7 @@ def login_pixiv( pixiv_config['REFRESH_TOKEN'] = login_info['refresh_token'] except KeyError as e: print(f'Failed to login. {e!r}') + exit(1) else: dump_config(config, favors_home) print("Login successful.") diff --git a/tests/test_commands/test_login.py b/tests/test_commands/test_login.py index 721e2b8..f00fecb 100644 --- a/tests/test_commands/test_login.py +++ b/tests/test_commands/test_login.py @@ -1,3 +1,5 @@ +from unittest.mock import patch + from typer.testing import CliRunner from favorites_crawler import app @@ -6,6 +8,74 @@ runner = CliRunner() +@patch('favorites_crawler.commands.login.CustomGetPixivToken') +class TestLoginPixiv: + username = 'username' + password = 'password' + user_id = 'user_id' + access_token = 'access_token' + refresh_token = 'refresh_token' + + def test_login_pixiv_success(self, mock_gppt, tmp_path): + favors_home = tmp_path / 'home' + mock_login = mock_gppt.return_value.login + mock_login.return_value = { + 'access_token': self.access_token, + 'refresh_token': self.refresh_token, + 'user': {'id': self.user_id} + } + + result = runner.invoke( + app, ['login', 'pixiv', '-u', self.username, '-p', self.password], + env={'FAVORS_HOME': str(favors_home)} + ) + + mock_login.assert_called_once_with(username=self.username, password=self.password) + assert result.exit_code == 0 + assert "success" in result.stdout + assert (favors_home / 'config.yml').exists() + config = load_config(favors_home)['pixiv'] + assert config['USER_ID'] == self.user_id + assert config['REFRESH_TOKEN'] == self.refresh_token + assert config['ACCESS_TOKEN'] == self.access_token + + def test_login_pixiv_should_failed_when_gppt_raise_exc(self, mock_gppt, tmp_path): + favors_home = tmp_path / 'home' + mock_login = mock_gppt.return_value.login + mock_login.side_effect = Exception + + result = runner.invoke( + app, ['login', 'pixiv', '-u', self.username, '-p', self.password], + env={'FAVORS_HOME': str(favors_home)} + ) + + mock_login.assert_called_once_with(username=self.username, password=self.password) + self.assert_login_failed(result, favors_home) + + def test_login_pixiv_should_failed_when_gppt_return_bad_resp(self, mock_gppt, tmp_path): + favors_home = tmp_path / 'home' + mock_login = mock_gppt.return_value.login + mock_login.return_value = {} + + result = runner.invoke( + app, ['login', 'pixiv', '-u', self.username, '-p', self.password], + env={'FAVORS_HOME': str(favors_home)} + ) + + mock_login.assert_called_once_with(username=self.username, password=self.password) + self.assert_login_failed(result, favors_home) + + @staticmethod + def assert_login_failed(result, favors_home): + assert result.exit_code == 1 + assert "Failed" in result.stdout + assert (favors_home / 'config.yml').exists() + config = load_config(favors_home)['pixiv'] + assert config['USER_ID'] == '' + assert config['REFRESH_TOKEN'] == '' + assert config['ACCESS_TOKEN'] == '' + + class TestLoginYandere: def test_login_yandere_success(self, tmp_path): favors_home = tmp_path / 'home' @@ -19,8 +89,8 @@ def test_login_yandere_success(self, tmp_path): assert result.exit_code == 0 assert "success" in result.stdout assert (favors_home / 'config.yml').exists() - config = load_config(favors_home) - assert config['yandere']['USERNAME'] == username + config = load_config(favors_home)['yandere'] + assert config['USERNAME'] == username class TestLoginNhentai: @@ -40,8 +110,8 @@ def test_login_nhentai_success(self, tmp_path): assert "success" in result.stdout assert (favors_home / 'cookie.txt').exists() assert (favors_home / 'config.yml').exists() - config = load_config(favors_home) - assert config['nhentai']['USER_AGENT'] == self.user_agent + config = load_config(favors_home)['nhentai'] + assert config['USER_AGENT'] == self.user_agent def test_login_nhentai_failed(self, tmp_path): favors_home = tmp_path / 'home' @@ -56,8 +126,8 @@ def test_login_nhentai_failed(self, tmp_path): assert "Failed" in result.stdout assert not (favors_home / 'cookie.txt').exists() assert (favors_home / 'config.yml').exists() - config = load_config(favors_home) - assert config['nhentai']['USER_AGENT'] == '' + config = load_config(favors_home)['nhentai'] + assert config['USER_AGENT'] == '' class TestLoginTwitter: @@ -81,11 +151,11 @@ def test_login_twitter_success(self, tmp_path): assert "success" in result.stdout assert (favors_home / 'cookie.txt').exists() assert (favors_home / 'config.yml').exists() - config = load_config(favors_home) - assert config['twitter']['AUTHORIZATION'] == self.access_token - assert config['twitter']['X_CSRF_TOKEN'] == self.csrf_token - assert config['twitter']['LIKES_ID'] == 'likes_id' - assert config['twitter']['FEATURES'] == { + config = load_config(favors_home)['twitter'] + assert config['AUTHORIZATION'] == self.access_token + assert config['X_CSRF_TOKEN'] == self.csrf_token + assert config['LIKES_ID'] == 'likes_id' + assert config['FEATURES'] == { 'profile_label_improvements_pcf_label_in_post_enabled': False, 'rweb_tipjar_consumption_enabled': True, 'responsive_web_graphql_exclude_directive_enabled': True, @@ -111,7 +181,7 @@ def test_login_twitter_success(self, tmp_path): 'longform_notetweets_inline_media_enabled': True, 'responsive_web_enhance_cards_enabled': False } - assert config['twitter']['USER_ID'] == 'xxx' + assert config['USER_ID'] == 'xxx' def test_login_twitter_failed_when_given_bad_url(self, tmp_path): favors_home = tmp_path / 'home' @@ -138,13 +208,14 @@ def test_login_twitter_failed_when_cookie_not_exists(self, tmp_path): self.assert_login_failed(result, favors_home) - def assert_login_failed(self, result, favors_home): + @staticmethod + def assert_login_failed(result, favors_home): assert result.exit_code == 1 assert "Failed" in result.stdout assert not (favors_home / 'cookie.txt').exists() assert (favors_home / 'config.yml').exists() - config = load_config(favors_home) - assert config['twitter']['AUTHORIZATION'] == '' - assert config['twitter']['X_CSRF_TOKEN'] == '' - assert config['twitter']['LIKES_ID'] == '' - assert config['twitter']['USER_ID'] == '' + config = load_config(favors_home)['twitter'] + assert config['AUTHORIZATION'] == '' + assert config['X_CSRF_TOKEN'] == '' + assert config['LIKES_ID'] == '' + assert config['USER_ID'] == '' From 27e47cf50b2364c71f23554109971a3a0e226e14 Mon Sep 17 00:00:00 2001 From: Wen Liang Date: Wed, 4 Dec 2024 11:29:57 +0800 Subject: [PATCH 9/9] refactor: FAVORS_HOME env, support replace ~ to user home --- src/favorites_crawler/constants/path.py | 4 +--- src/favorites_crawler/utils/config.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/favorites_crawler/constants/path.py b/src/favorites_crawler/constants/path.py index 89d3598..07f4619 100644 --- a/src/favorites_crawler/constants/path.py +++ b/src/favorites_crawler/constants/path.py @@ -1,3 +1 @@ -import os - -DEFAULT_FAVORS_HOME = os.path.expanduser('~/.favorites_crawler') +DEFAULT_FAVORS_HOME = '~/.favorites_crawler' diff --git a/src/favorites_crawler/utils/config.py b/src/favorites_crawler/utils/config.py index 315b809..c3aa23a 100644 --- a/src/favorites_crawler/utils/config.py +++ b/src/favorites_crawler/utils/config.py @@ -38,6 +38,7 @@ def load_config(home: str) -> dict: """Load config from user home""" + home = os.path.expanduser(home) create_favors_home(home) config_file = os.path.join(home, 'config.yml') if not os.path.exists(config_file): @@ -47,8 +48,9 @@ def load_config(home: str) -> dict: return yaml.safe_load(f) -def dump_config(data: dict, home): +def dump_config(data: dict, home: str): """Dump config data to user home""" + home = os.path.expanduser(home) create_favors_home(home) config_file = os.path.join(home, 'config.yml') with open(config_file, 'w', encoding='utf8') as f: