Fetch user activity from GitHub using httpx & asyncio (#355)

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: stable
+    rev: 21.7b0
     hooks:
     - id: black
 -   repo: https://gitlab.com/pycqa/flake8

Makefile

@@ -31,6 +31,9 @@ attach-worker: ## attach to worker process for debugging purposes
 requirements: ## generate requirements.txt using piptools
 	pip-compile --output-file=requirements.txt requirements.in
 
+install:  ## install development requirements
+	pip install -r requirements_dev.txt
+
 s3-bucket: ## create s3 bucket in localstack
 	docker-compose exec -T localstack bash /tmp/dev_scripts/create_s3_bucket.sh
 

busy_beaver/app.py

@@ -6,6 +6,7 @@
 
 from . import models
 from .blueprints import cfps_bp, events_bp, github_bp, healthcheck_bp, slack_bp, web_bp
+from .clients import github_async
 from .common.oauth import OAuthError
 from .common.wrappers import SlackClient
 from .config import DATABASE_URI, ENVIRONMENT, REDIS_URI, SECRET_KEY
@@ -74,7 +75,7 @@ def set_secure_headers(response):
 
     @app.shell_context_processor
     def create_shell_context():
-        context = {"app": app, "db": db, "rq": rq}
+        context = {"app": app, "db": db, "github": github_async, "rq": rq}
 
         db_models = inspect.getmembers(models, lambda member: inspect.isclass(member))
         for name, reference in db_models:

busy_beaver/apps/github_integration/summary/blocks.py

@@ -1,42 +1,24 @@
-from datetime import datetime, timedelta
+from datetime import datetime
 import logging
-from typing import List, NamedTuple
 
-from .summary import GitHubUserEvents
-from busy_beaver.models import GitHubSummaryUser
 from busy_beaver.toolbox.slack_block_kit import Divider, Section
 from busy_beaver.toolbox.slack_block_kit.elements import Image
 
 logger = logging.getLogger(__name__)
 
 
-class UserEvents(NamedTuple):
-    user: GitHubSummaryUser
-    events: GitHubUserEvents
-
-
 class GitHubSummaryPost:
-    def __init__(self, users: List[GitHubSummaryUser], boundary_dt: datetime):
-        self.users = users
-        self.boundary_dt = boundary_dt
-        self.now = boundary_dt + timedelta(days=1)
+    def __init__(self, all_user_events):
+        self.all_user_events = all_user_events
 
     def __repr__(self):  # pragma: no cover
         return "<GitHubSummaryPost>"
 
-    def create(self):
-        all_user_events = []
-        for idx, user in enumerate(self.users):
-            logger.info("Compiling stats for {0}".format(user))
-            user_events = GitHubUserEvents(user, self.boundary_dt)
-
-            if len(user_events) > 0:
-                all_user_events.append(UserEvents(user, user_events))
-
-        self.all_user_events = all_user_events
-
     def as_blocks(self):
-        output = [Section(f"*Daily GitHub Summary -- {self.now:%B %d, %Y}*"), Divider()]
+        output = [
+            Section(f"*Daily GitHub Summary -- {datetime.now():%B %d, %Y}*"),
+            Divider(),
+        ]
 
         if not self.all_user_events:
             output.append(Section(text="No activity to report."))

busy_beaver/apps/github_integration/summary/summary.py

@@ -1,5 +1,3 @@
-from datetime import datetime, timedelta
-
 from .event_list import (
     CommitsList,
     CreatedReposList,
@@ -10,14 +8,17 @@
     ReleasesPublishedList,
     StarredReposList,
 )
-from busy_beaver.clients import github
 from busy_beaver.models import GitHubSummaryUser
 
 
 class GitHubUserEvents:
-    def __init__(self, user: GitHubSummaryUser, boundary_dt: datetime):
+    def __init__(self, user: GitHubSummaryUser, classified_events: list):
         self.user = user
-        self.event_lists = [  # this is the order of summary output
+        self.classified_events = classified_events
+
+    @classmethod
+    def classify_events_by_type(cls, user: GitHubSummaryUser, events: list):
+        tracked_event_types = [  # this is the order of summary output
             ReleasesPublishedList(),
             CreatedReposList(),
             PublicizedReposList(),
@@ -28,22 +29,16 @@ def __init__(self, user: GitHubSummaryUser, boundary_dt: datetime):
             StarredReposList(),
         ]
 
-        self.username = user.github_username
-        start_dt = boundary_dt
-        end_dt = start_dt + timedelta(days=1)
-
-        timeline = github.user_activity_during_range(self.username, start_dt, end_dt)
-        for event in timeline:
-            for event_list in self.event_lists:
+        for event in events:
+            for event_list in tracked_event_types:
                 if event_list.matches_event(event):
                     event_list.append(event)
 
-    def __len__(self):
-        return sum(len(events) for events in self.event_lists)
+        return cls(user, tracked_event_types)
 
     def generate_summary_text(self):
         summary = ""
-        for event_list in self.event_lists:
+        for event_list in self.classified_events:
             summary += event_list.generate_summary_text()
 
         if not summary:

busy_beaver/apps/github_integration/summary/workflow.py

@@ -1,35 +1,43 @@
 from datetime import timedelta
 import logging
 import random
-from typing import List
+from typing import List, NamedTuple
 
 from sqlalchemy import and_
 
 from .blocks import GitHubSummaryPost
+from .summary import GitHubUserEvents
+from busy_beaver.clients import github_async
 from busy_beaver.common.wrappers import SlackClient
 from busy_beaver.exceptions import ValidationError
 from busy_beaver.extensions import rq
 from busy_beaver.models import GitHubSummaryUser, SlackInstallation
-from busy_beaver.toolbox import set_task_progress, utc_now_minus
+from busy_beaver.toolbox import generate_range_utc_now_minus, set_task_progress
 
 logger = logging.getLogger(__name__)
 
 
+class UserEvents(NamedTuple):
+    user: GitHubSummaryUser
+    events: GitHubUserEvents
+
+
 @rq.job
 def post_github_summary_message(workspace_id: str):
     installation = SlackInstallation.query.filter_by(workspace_id=workspace_id).first()
     if not installation:
         raise ValidationError("workspace not found")
 
-    boundary_dt = utc_now_minus(timedelta(days=1))
-    fetch_github_summary_post_to_slack(installation, boundary_dt)
+    start_dt, end_dt = generate_range_utc_now_minus(timedelta(days=1))
+    fetch_github_summary_post_to_slack(installation, start_dt, end_dt)
     set_task_progress(100)
 
 
-def fetch_github_summary_post_to_slack(installation, boundary_dt):
+def fetch_github_summary_post_to_slack(installation, start_dt, end_dt):
     channel = installation.github_summary_config.channel
     slack = SlackClient(installation.bot_access_token)
 
+    # Step 1: find active users
     channel_members = slack.get_channel_members(channel)
     users: List[GitHubSummaryUser] = GitHubSummaryUser.query.filter(
         and_(
@@ -40,9 +48,25 @@ def fetch_github_summary_post_to_slack(installation, boundary_dt):
     ).all()
     random.shuffle(users)
 
-    github_summary_post = GitHubSummaryPost(users, boundary_dt)
-    github_summary_post.create()
+    # Step 2: get GitHub activity for users
+    users_by_github_username = {user.github_username: user for user in users}
+    usernames = users_by_github_username.keys()
+    activity_by_user = github_async.get_activity_for_users(usernames, start_dt, end_dt)
+
+    # Step 3: classify activity by event type
+    all_user_events = []
+    for _, user in enumerate(users):
+        user_activity = activity_by_user[user.github_username]
+
+        logger.info("Compiling stats for {0}".format(user))
+        if len(user_activity) == 0:
+            continue
+
+        user_events = GitHubUserEvents.classify_events_by_type(user, user_activity)
+        all_user_events.append(UserEvents(user, user_events))
 
+    # Step 4: format message and post to Slack
+    github_summary_post = GitHubSummaryPost(all_user_events)
     slack.post_message(
         blocks=github_summary_post.as_blocks(),
         channel=channel,

busy_beaver/clients.py

@@ -8,7 +8,13 @@
 integration.
 """
 
-from .common.wrappers import GitHubClient, MeetupClient, S3Client, SlackClient
+from .common.wrappers import (
+    AsyncGitHubClient,
+    GitHubClient,
+    MeetupClient,
+    S3Client,
+    SlackClient,
+)
 from .config import (
     DIGITALOCEAN_SPACES_KEY,
     DIGITALOCEAN_SPACES_SECRET,
@@ -28,6 +34,7 @@
 
 chipy_slack = SlackClient(SLACK_TOKEN)  # Default Workspace -- this is being phased out
 github = GitHubClient(GITHUB_OAUTH_TOKEN)
+github_async = AsyncGitHubClient(GITHUB_OAUTH_TOKEN)
 meetup = MeetupClient(MEETUP_API_KEY)
 
 slack_install_oauth = SlackInstallationOAuthFlow(SLACK_CLIENT_ID, SLACK_CLIENT_SECRET)

busy_beaver/common/wrappers/__init__.py

@@ -1,5 +1,5 @@
 from .aws_s3 import S3Client  # noqa
-from .github import GitHubClient  # noqa
+from .github import AsyncGitHubClient, GitHubClient  # noqa
 from .meetup import MeetupClient  # noqa
 from .requests_client import RequestsClient  # noqa
 from .slack import SlackClient  # noqa