Skip to content

Commit

Permalink
Merge pull request #57 from RyouMon/fix-pixiv
Browse files Browse the repository at this point in the history
Fix: update twitters domain, fix 400 when start request
  • Loading branch information
RyouMon authored Aug 1, 2024
2 parents 1cb66c1 + fe8e37c commit fe60c46
Show file tree
Hide file tree
Showing 5 changed files with 14 additions and 9 deletions.
2 changes: 1 addition & 1 deletion src/favorites_crawler/constants/domains.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
YANDERE_DOMAIN = 'yande.re'
NHENTAI_DOMAIN = 'nhentai.net'
LMMPIC_DOMAIN = 'emonl.com'
TWITTER_DOMAIN = 'twitter.com'
TWITTER_DOMAIN = 'x.com'
4 changes: 2 additions & 2 deletions src/favorites_crawler/constants/endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@
NHENTAI_LOGIN_URL = 'https://nhentai.net/login/?next=/favorites/'
NHENTAI_USER_FAVORITES_URL = 'https://nhentai.net/favorites/'

TWITTER_LIKES_URL = 'https://twitter.com/i/api/graphql/{id}/Likes'
TWITTER_PROFILE_LIKES_URL = 'https://twitter.com/{username}/likes'
TWITTER_LIKES_URL = 'https://x.com/i/api/graphql/{id}/Likes'
TWITTER_PROFILE_LIKES_URL = 'https://x.com/{username}/likes'
2 changes: 1 addition & 1 deletion src/favorites_crawler/pipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def file_path(self, request, response=None, info=None, *, item=None):
def item_completed(self, results, item, info):
for result in info.downloaded.values():
if isinstance(result, Failure):
logger.error('Error when downloading file: %s', result.value)
logger.error('Error when downloading file: %r', result.value)
return super().item_completed(results, item, info)


Expand Down
3 changes: 3 additions & 0 deletions src/favorites_crawler/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,6 @@
# ExifTool settings
ENABLE_WRITE_IPTC_KEYWORDS = True
EXIF_TOOL_EXECUTABLE = None

# Set to True when DEBUGGING
HTTPERROR_ALLOW_ALL = False
12 changes: 7 additions & 5 deletions src/favorites_crawler/spiders/twitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,26 +42,28 @@ def __init__(self, *args, **kwargs):
"withV2Timeline": True
}
self.features = {
"rweb_tipjar_consumption_enabled": True,
"responsive_web_graphql_exclude_directive_enabled": True,
"verified_phone_label_enabled": False,
"creator_subscriptions_tweet_preview_api_enabled": True,
"verified_phone_label_enabled": False, "creator_subscriptions_tweet_preview_api_enabled": True,
"responsive_web_graphql_timeline_navigation_enabled": True,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": False,
"communities_web_enable_tweet_community_results_fetch": True,
"c9s_tweet_anatomy_moderator_badge_enabled": True,
"articles_preview_enabled": True,
"tweetypie_unmention_optimization_enabled": True,
"responsive_web_edit_tweet_api_enabled": True,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": True,
"view_counts_everywhere_api_enabled": True,
"longform_notetweets_consumption_enabled": True,
"responsive_web_twitter_article_tweet_consumption_enabled": False,
"responsive_web_twitter_article_tweet_consumption_enabled": True,
"tweet_awards_web_tipping_enabled": False,
"creator_subscriptions_quote_tweet_preview_enabled": False,
"freedom_of_speech_not_reach_fetch_enabled": True,
"standardized_nudges_misinfo": True,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": True,
"rweb_video_timestamps_enabled": True,
"longform_notetweets_rich_text_read_enabled": True,
"longform_notetweets_inline_media_enabled": True,
"responsive_web_media_download_video_enabled": False,
"responsive_web_enhance_cards_enabled": False
}
self.headers = {
Expand All @@ -79,7 +81,7 @@ def parse_start_url(self, response, **kwargs):
def parse(self, response, **kwargs):
entries = (
entry['content'] for entry in DictRouter(response.json()).route_to(
'data.user.result.timeline_v2.timeline.instructions.0.entries', [],
'data.user.result.timeline_v2.timeline.instructions.0.entries', [],
))
cursor_value = None
timeline_item_count = 0
Expand Down

0 comments on commit fe60c46

Please sign in to comment.