Skip to content

Commit

Permalink
refactor: read FAVORS_HOME from env, config save to FAVORS_HOME, defa…
Browse files Browse the repository at this point in the history
…ult FAVORS_HOME is ~/.favorites_crawler
  • Loading branch information
RyouMon committed Dec 3, 2024
1 parent 72174b7 commit f8b747d
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 20 deletions.
4 changes: 3 additions & 1 deletion src/favorites_crawler/commands/crawl.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from scrapy.spiderloader import SpiderLoader

from favorites_crawler.utils.config import load_config, overwrite_spider_settings
from favorites_crawler.constants.path import DEFAULT_FAVORS_HOME

app = typer.Typer(help='Crawl your favorites from websites.', no_args_is_help=True)

Expand Down Expand Up @@ -70,7 +71,8 @@ def crawl(name, **kwargs):
:param kwargs: kwargs passed to spider's __init__ method
"""
spider = spider_loader.load(name)
overwrite_spider_settings(spider, scrapy_settings, load_config())
favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME)
overwrite_spider_settings(spider, scrapy_settings, load_config(favors_home))

Check warning on line 75 in src/favorites_crawler/commands/crawl.py

View check run for this annotation

Codecov / codecov/patch

src/favorites_crawler/commands/crawl.py#L74-L75

Added lines #L74 - L75 were not covered by tests
process = CrawlerProcess(scrapy_settings)
process.crawl(spider, **kwargs)
for crawler in process.crawlers:
Expand Down
36 changes: 20 additions & 16 deletions src/favorites_crawler/commands/login.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import os
import shutil
from typing import Optional

import typer
from selenium.common import NoSuchWindowException

from favorites_crawler.utils.auth import CustomGetPixivToken, parse_twitter_likes_url, parser_twitter_likes_features
from favorites_crawler.utils.config import dump_config, load_config
Expand Down Expand Up @@ -33,13 +33,14 @@ def login_pixiv(
If you do not provide your username and password, you will login manually on the web page
"""
config = load_config()
favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME)
config = load_config(favors_home)

Check warning on line 37 in src/favorites_crawler/commands/login.py

View check run for this annotation

Codecov / codecov/patch

src/favorites_crawler/commands/login.py#L36-L37

Added lines #L36 - L37 were not covered by tests
token_getter = CustomGetPixivToken()
try:
login_info = token_getter.login(username=username, password=password)
except NoSuchWindowException:
print('Failed to login.')
return
except Exception as e:
print(f'Failed to login. {e!r}')
exit(1)

Check warning on line 43 in src/favorites_crawler/commands/login.py

View check run for this annotation

Codecov / codecov/patch

src/favorites_crawler/commands/login.py#L41-L43

Added lines #L41 - L43 were not covered by tests

pixiv_config = config.setdefault('pixiv', {})
try:
Expand All @@ -49,7 +50,7 @@ def login_pixiv(
except KeyError as e:
print(f'Failed to login. {e!r}')
else:
dump_config(config)
dump_config(config, favors_home)

Check warning on line 53 in src/favorites_crawler/commands/login.py

View check run for this annotation

Codecov / codecov/patch

src/favorites_crawler/commands/login.py#L53

Added line #L53 was not covered by tests
print("Login successful.")


Expand All @@ -63,10 +64,11 @@ def login_yandere(
"""
Login to yandere.
"""
config = load_config()
favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME)
config = load_config(favors_home)

Check warning on line 68 in src/favorites_crawler/commands/login.py

View check run for this annotation

Codecov / codecov/patch

src/favorites_crawler/commands/login.py#L67-L68

Added lines #L67 - L68 were not covered by tests
yandere_config = config.setdefault('yandere', {})
yandere_config['USERNAME'] = username
dump_config(config)
dump_config(config, favors_home)

Check warning on line 71 in src/favorites_crawler/commands/login.py

View check run for this annotation

Codecov / codecov/patch

src/favorites_crawler/commands/login.py#L71

Added line #L71 was not covered by tests
print("Login successful.")


Expand Down Expand Up @@ -101,18 +103,19 @@ def login_twitter(
6. Copy Authorization, X-Csrf-Token and RequestURL from request(Likes?variables...) input on terminal.\n
7. Use "Get cookies.txt" browser extension download cookie file.
"""
config = load_config()
favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME)
config = load_config(favors_home)

Check warning on line 107 in src/favorites_crawler/commands/login.py

View check run for this annotation

Codecov / codecov/patch

src/favorites_crawler/commands/login.py#L106-L107

Added lines #L106 - L107 were not covered by tests
twitter_config = config.setdefault('twitter', {})
try:
twitter_config['AUTHORIZATION'] = auth_token
twitter_config['X_CSRF_TOKEN'] = csrf_token
twitter_config['LIKES_ID'], twitter_config['USER_ID'] = parse_twitter_likes_url(likes_url)
twitter_config['FEATURES'] = parser_twitter_likes_features(likes_url)
shutil.copy(cookie_file, DEFAULT_FAVORS_HOME)
shutil.copy(cookie_file, favors_home)
except Exception as e:
print(f"Failed to login: {e!r}")
return
dump_config(config)
exit(1)
dump_config(config, favors_home)

Check warning on line 118 in src/favorites_crawler/commands/login.py

View check run for this annotation

Codecov / codecov/patch

src/favorites_crawler/commands/login.py#L110-L118

Added lines #L110 - L118 were not covered by tests
print("Login successful.")


Expand All @@ -136,13 +139,14 @@ def login_nhentai(
4. Copy user-agent from any request.\n
5. Use "Get cookies.txt" browser extension download cookie file.
"""
config = load_config()
favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME)
config = load_config(favors_home)
nhentai_config = config.setdefault('nhentai', {})
try:
nhentai_config['USER_AGENT'] = user_agent
shutil.copy(cookie_file, DEFAULT_FAVORS_HOME)
shutil.copy(cookie_file, favors_home)
except Exception as e:
print(f"Failed to login: {e!r}")
return
dump_config(config)
exit(1)
dump_config(config, favors_home)
print("Login successful.")
5 changes: 2 additions & 3 deletions src/favorites_crawler/utils/config.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import os
import yaml

from favorites_crawler.constants.path import DEFAULT_FAVORS_HOME

DEFAULT_CONFIG = {
'global': {
Expand Down Expand Up @@ -35,7 +34,7 @@
}


def load_config(home: str = DEFAULT_FAVORS_HOME) -> dict:
def load_config(home: str) -> dict:
"""Load config from user home"""
create_favors_home(home)
config_file = os.path.join(home, 'config.yml')
Expand All @@ -46,7 +45,7 @@ def load_config(home: str = DEFAULT_FAVORS_HOME) -> dict:
return yaml.safe_load(f)


def dump_config(data: dict, home: str = DEFAULT_FAVORS_HOME):
def dump_config(data: dict, home):
"""Dump config data to user home"""
create_favors_home(home)
config_file = os.path.join(home, 'config.yml')
Expand Down
39 changes: 39 additions & 0 deletions tests/test_commands/test_login.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
from typer.testing import CliRunner

from favorites_crawler.commands.login import app
from favorites_crawler.utils.config import load_config

runner = CliRunner()


class TestLoginNhentai:
def test_login_nhentai_success(self, tmp_path):
favors_home = tmp_path / 'home'
cookie = tmp_path / 'cookie.txt'
cookie.touch()
user_agent = 'Test-User-Agent'

result = runner.invoke(
app, ['nhentai', '-c', str(cookie), '-u', user_agent],
env={'FAVORS_HOME': str(favors_home)}
)

assert result.exit_code == 0
assert "success" in result.stdout
assert (favors_home / 'cookie.txt').exists()
assert (favors_home / 'config.yml').exists()
config = load_config(favors_home)
assert config['nhentai']['USER_AGENT'] == user_agent

def test_login_nhentai_failed(self, tmp_path):
favors_home = tmp_path / 'home'
cookie = tmp_path / 'cookie.txt'
user_agent = 'Test-User-Agent'

result = runner.invoke(
app, ['nhentai', '-c', str(cookie), '-u', user_agent],
env={'FAVORS_HOME': str(favors_home)}
)

assert result.exit_code == 1
assert "Failed" in result.stdout

0 comments on commit f8b747d

Please sign in to comment.