Moved Recommender to Its Own Repo (#8)

* refactor db to use sqlalchemy * tests: updated tests for refactored db * added sqlalchemy to requirements * blackified * fix process_result_value type annotation * tests: fix importing db in test_db * change dialect annotation type to Any * tests: replace db tests with new_dv tests * rename create_session to init_db, move get_session definition to module level * add sqlalchemy-stubs for mypy tests * fix initializing db in init * move db initialization to bot.main * mock database to avoid network requests in tests * fix references to database in bot.py * make session a keyword param to pass tests * update reference to database in __init__.get_user * fix KeyError in __init__.get_user * tests: added test for __init__.get_user * replace "Allerter" with "allerter" * add newline to geniust.VERSION * added comparing geniust.VERSION with git tag version * rename push branch in github action * bump version to 2.4.0 * added executing workflow on tags * fix tags syntax error * really fix tags syntax error * really really fix tags syntax error * fix tags syntax error * use double asterisk * added/updated docstring in api.py * added/updated docstring in bot.py * added/updated docstring in db.py * added/updated docstring in recommender.py * added/updated docstring in song.py * added/updated docstring in server.py * added/updated docstring in utils.py * blackified * bump runtime python to 3.8.8 * fix comparing tag_version and file_version in workflow, removed debug step * remove recommender data and class * removed recommender handlers from server * added typing.Dict import in db * added Recommender to API * removed lastfm api key * update recommender references * Complete api.Recommender * Added access token to api.Recommender * tests: added and updated tests * added recommender token to GH workflow * Blackified * Bump pyyaml to 5.4
allerter · Apr 1, 2021 · 469078a · 469078a
1 parent f6b7302
commit 469078a
Show file tree

Hide file tree

Showing 40 changed files with 900 additions and 55,012 deletions.
diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml
@@ -5,17 +5,33 @@ name: build
 
 on:
   push:
-    branches: [ master, development ]
+    branches: [ master, develop ]
+    tags:        
+      - '**'
   pull_request:
     branches: [ master ]
 
+
 jobs:
   build-and-test:
 
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
+    - name: Compare version with git tag
+      if: startsWith(github.ref, 'refs/tags/')
+      run: |
+        file_version=$(cat geniust/VERSION)
+        tag_version=${GITHUB_REF#refs/*/}
+        if test "$file_version" = "$tag_version";
+        then
+          echo "Versions match! >> $file_version"
+        else
+          echo "Versions don't match! >> FILE=$file_version != TAG=$tag_version"
+          exit 1
+        fi
+
     - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
@@ -44,6 +60,7 @@ jobs:
         SPOTIFY_REDIRECT_URI: ${{ secrets.SPOTIFY_REDIRECT_URI }}
         SPOTIFY_CLIENT_SECRET: ${{ secrets.SPOTIFY_CLIENT_SECRET }}
         LASTFM_API_KEY: ${{ secrets.LASTFM_API_KEY }}
+        RECOMMENDER_TOKEN: TEST_TOKEN
       run: tox
 
     - name: Archive code coverage results
@@ -81,10 +98,6 @@ jobs:
         with:
           name: code-coverage-report
 
-      - name: Check coverage file
-        run: |
-          ls
-
       - name: Upload results to Code Climate
         run: |
           ./cc-test-reporter after-build -t=coverage.py
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,11 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 20.8b1
+    hooks:
+    -   id: black
diff --git a/LICENSE b/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Allerter
+Copyright (c) 2020 allerter
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

diff --git a/README.md b/README.md
@@ -6,10 +6,10 @@
 
 ![status](https://img.shields.io/uptimerobot/status/m786636302-b2fa3edeb9237ae327f70d06)
 ![GitHub release (latest by
-date)](https://img.shields.io/github/v/release/Allerter/geniust)
-![build](https://github.com/Allerter/geniust/workflows/build/badge.svg)
+date)](https://img.shields.io/github/v/release/allerter/geniust)
+![build](https://github.com/allerter/geniust/workflows/build/badge.svg)
 [![Test
-Coverage](https://api.codeclimate.com/v1/badges/74d5611d77cb26f4ed16/test_coverage)](https://codeclimate.com/github/Allerter/geniust/test_coverage)
+Coverage](https://api.codeclimate.com/v1/badges/74d5611d77cb26f4ed16/test_coverage)](https://codeclimate.com/github/allerter/geniust/test_coverage)
 [![Code style:
 black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Telegram

diff --git a/geniust/VERSION b/geniust/VERSION
@@ -1 +1 @@
-2.3.0
+2.4.0
diff --git a/geniust/__init__.py b/geniust/__init__.py
@@ -9,7 +9,6 @@
 from lyricsgenius import OAuth2
 import yaml
 
-from geniust.db import Database
 from geniust.constants import (
     GENIUS_CLIENT_ID,
     GENIUS_REDIRECT_URI,
@@ -21,8 +20,6 @@
 
 username: str = "genius_the_bot"  # Bot(BOT_TOKEN).get_me().username
 
-database = Database("user_data", "user_preferences")
-
 RT = TypeVar("RT")
 
 
@@ -43,7 +40,7 @@ def wrapper(*args, **kwargs) -> RT:
         else:
             chat_id = update.inline_query.from_user.id
         if "bot_lang" not in context.user_data:
-            database.user(chat_id, context.user_data)
+            context.bot_data["db"].user(chat_id, context.user_data)
         result = func(*args, **kwargs)
         return result
 

diff --git a/geniust/api.py b/geniust/api.py
@@ -4,11 +4,13 @@
 import os
 import asyncio
 import queue
-import time
+from json.decoder import JSONDecodeError
 from typing import Any, Tuple, Optional, Union, List, Dict
+from dataclasses import dataclass
 
 import requests
 import telethon
+from requests.exceptions import HTTPError, Timeout
 from bs4 import BeautifulSoup
 from lyricsgenius import Genius, PublicAPI
 from lyricsgenius.utils import clean_str
@@ -17,65 +19,18 @@
 from telethon import types
 
 from geniust.constants import (
+    RECOMMENDER_TOKEN,
     TELETHON_API_ID,
+    Preferences,
     TELETHON_API_HASH,
     TELETHON_SESSION_STRING,
     ANNOTATIONS_CHANNEL_HANDLE,
     GENIUS_TOKEN,
-    LASTFM_API_KEY,
 )
 
 logger = logging.getLogger("geniust")
 
 
-def lastfm(method: str, parameters_input: dict) -> dict:  # pragma: no cover
-    api_key_lastfm = LASTFM_API_KEY
-    user_agent_lastfm = "GeniusT"
-    api_url_lastfm = "http://ws.audioscrobbler.com/2.0/"
-    # Last.fm API header and default parameters
-    headers = {"user-agent": user_agent_lastfm}
-    parameters = {"method": method, "api_key": api_key_lastfm, "format": "json"}
-    parameters.update(parameters_input)
-    # Responses and error codes
-    state = False
-    while state is False:
-        try:
-            response = requests.get(
-                api_url_lastfm, headers=headers, params=parameters, timeout=10
-            )
-            if response.status_code == 200:
-                logger.debug(
-                    ("Last.fm API: 200" " - Response was successfully received.")
-                )
-                state = True
-            elif response.status_code == 401:
-                logger.debug(
-                    ("Last.fm API: 401" " - Unauthorized. Please check your API key.")
-                )
-            elif response.status_code == 429:
-                logger.debug(
-                    ("Last.fm API: 429" " - Too many requests. Waiting 60 seconds.")
-                )
-                time.sleep(5)
-                state = False
-            else:
-                logger.debug(
-                    (
-                        "Last.fm API: Unspecified error %s."
-                        " No response was received."
-                        " Trying again after 60 seconds..."
-                    ),
-                    response.status_code,
-                )
-                time.sleep(1)
-                state = False
-        except OSError as err:
-            logger.debug("Error: %s. Trying again...", str(err))
-            time.sleep(3)
-            state = False
-    return response.json()
-
-
 def get_channel() -> types.TypeInputPeer:
     """Returns telethon Input Peer for the annotations channel
 
@@ -521,7 +476,12 @@ def song_annotations(
         return all_annotations
 
     def fetch(self, track: Dict[str, Any], include_annotations: bool) -> None:
-        """fetches song from Genius adds it to the artist object"""
+        """fetches song from Genius adds it to the artist objecty
+
+        Args:
+            track (Dict[str, Any]): Track dict including track information.
+            include_annotations (bool): True or False.
+        """
         song = track["song"]
 
         annotations: Dict[int, str] = {}
@@ -590,7 +550,16 @@ async def search_album(
     def async_album_search(
         self, album_id: int, include_annotations: bool = False
     ) -> Dict[str, Any]:
-        """gets the album from Genius and returns a dictionary"""
+        """gets the album from Genius and returns a dictionary
+
+        Args:
+            album_id (int): Album ID.
+            include_annotations (bool, optional): Include annotations
+                in album. Defaults to False.
+
+        Returns:
+            Dict[str, Any]: Album data and lyrics.
+        """
         q: queue.Queue = queue.Queue(1)
         new_loop = asyncio.new_event_loop()
         asyncio.set_event_loop(new_loop)
@@ -600,3 +569,156 @@ def async_album_search(
         new_loop.run_until_complete(future)
         new_loop.close()
         return q.get()
+
+
+@dataclass
+class SimpleArtist:
+    """An artist without full info"""
+
+    id: int
+    name: str
+
+    def __repr__(self):
+        return f"SimpleArtist(id={self.id})"
+
+
+@dataclass
+class Artist(SimpleArtist):
+    """A Artist from the Recommender"""
+
+    description: str
+
+    def __repr__(self):
+        return f"Artist(id={self.id})"
+
+
+@dataclass
+class Song:
+    """A Song from the Recommender"""
+
+    id: int
+    artist: str
+    name: str
+    genres: List[str]
+    id_spotify: Optional[str]
+    isrc: Optional[str]
+    cover_art: Optional[str]
+    preview_url: Optional[str]
+    download_url: Optional[str]
+
+    def __repr__(self):
+        return f"Song(id={self.id})"
+
+
+class Recommender:
+    API_ROOT = "https://geniust-recommender.herokuapp.com/"
+
+    def __init__(
+        self, genres: Optional[List[str]] = None, num_songs: Optional[int] = None
+    ):
+        self._sender = Sender(self.API_ROOT, access_token=RECOMMENDER_TOKEN, retries=3)
+        self.num_songs: int = (
+            self._sender.request("songs/len")["len"] if num_songs is None else num_songs
+        )
+        self.genres: List[str] = (
+            self._sender.request("genres")["genres"] if genres is None else genres
+        )
+        self.genres_by_number = {}
+        for i, genre in enumerate(self.genres):
+            self.genres_by_number[i] = genre
+
+    def artist(self, id: int) -> Artist:
+        res = self._sender.request(f"artists/{id}")["artist"]
+        return Artist(**res)
+
+    def genres_by_age(self, age: int) -> List[str]:
+        return self._sender.request("genres", params={"age": age})["genres"]
+
+    def preferences_from_platform(
+        self, token: str, platform: str
+    ) -> Optional[Preferences]:
+        res = self._sender.request(
+            "preferences", params={"token": token, "platform": platform}
+        )["preferences"]
+        return Preferences(**res) if res["genres"] else None
+
+    def search_artist(self, q: str) -> List[SimpleArtist]:
+        res = self._sender.request("search/artists", params={"q": q})["hits"]
+        return [SimpleArtist(**x) for x in res]
+
+    def shuffle(self, pref: Preferences) -> List[Song]:
+        params = {"genres": ",".join(pref.genres)}
+        artists = ",".join(pref.artists)
+        if artists:
+            params["artists"] = artists
+        res = self._sender.request(
+            "recommendations",
+            params=params,
+        )["recommendations"]
+        return [Song(**x) for x in res]
+
+    def song(self, id: int) -> Song:
+        res = self._sender.request(f"songs/{id}")["song"]
+        return Song(**res)
+
+
+class Sender:
+    """Sends requests to the GeniusT Recommender."""
+
+    def __init__(
+        self,
+        api_root: str,
+        access_token: str = None,
+        timeout: int = 5,
+        retries: int = 0,
+    ):
+        self.api_root = api_root
+        self._session = requests.Session()
+        self._session.headers = {
+            "application": "GeniusT TelegramBot",
+            "User-Agent": "https://github.com/allerter/geniust",
+        }  # type: ignore
+        if access_token:
+            self._session.headers["Authorization"] = f"Bearer {access_token}"
+        self.timeout: int = timeout
+        self.retries: int = retries
+
+    def request(
+        self, path: str, method: str = "GET", params: dict = None, **kwargs
+    ) -> dict:
+        """Makes a request to Genius."""
+        uri = self.api_root
+        uri += path
+        params = params if params else {}
+
+        # Make the request
+        response = None
+        tries = 0
+        while response is None and tries <= self.retries:
+            tries += 1
+            try:
+                response = self._session.request(
+                    method, uri, timeout=self.timeout, params=params, **kwargs
+                )
+                response.raise_for_status()
+            except Timeout as e:  # pragma: no cover
+                error = "Request timed out:\n{e}".format(e=e)
+                logger.warn(error)
+                if tries > self.retries:
+                    raise Timeout(error)
+            except HTTPError as e:  # pragma: no cover
+                error = get_description(e)
+                if response.status_code < 500 or tries > self.retries:
+                    raise HTTPError(response.status_code, error)
+        return response.json()
+
+
+def get_description(e: HTTPError) -> str:  # pragma: no cover
+    error = str(e)
+    try:
+        res = e.response.json()
+    except JSONDecodeError:
+        res = {}
+    description = res["detail"] if res.get("detail") else res.get("error_description")
+    error += "\n{}".format(description) if description else ""
+    return error