v2.3 (#5)

* add recommender.preferences_from_platform

* get preferences using recommender in process_preferences

* supply correct parameter to get_user_token

* added PreferencesHandler

* fix preferences_from_platform type hint

* blackified

* add missing import

* send the right token for spotify in process_preferences

* convert genres to list,
check genres before searching for artists

* tests: update test_token_handler

* tests: set html parser for bs4

* tests: speed up test_get_album

* tests: fix test_main requiring network connection

* blackified

* fix getting wrong token in PreferencesHandler

* fix issue where token is not declared

* fix assigning response to wrong key

* change conftest fixtures scope to session,
added fixtures for preferences_from_platform test

* tests: remove test_login keyboard assert

* tests: added test_preferences_from_platform

* tests: added lasftm track top tags and user pyongs files

* tests: added more cases

* tests: added PreferencesHandler tests

* tests: fix wrong value fro empty genre

empty genres returns None, not an empty string.

* tests: fix wrong value fro empty artists

also updated the test to reflect the corrections

* tests: fix wrong assert for empty artist

* added repr method to exclusion

* fix user_pyongs.json structure

* fix typo

* tests: improve recommender tests

added actually testing API return values in preferences_from_platform
added search query to select_artists

* tests: improve test_process_preferences

* added supplying chat id when updating preferences

* added lastfm to include

* tests: added inline query search songs tests

* tests: added upsert tests for db

* tests: add table to insert and upsert tables

* tests: added case to test_select

* tests: added tables to test_update

* tests: added select_language to coverage excluded

* tests: fixed patching lastfm

* tests: removed unused variable

* tests: fix flake8 errors

* blackified

* added shuffle to main menu for new users

* tests: added cases for shuffle option in main menu

geniust/bot.py

@@ -131,6 +131,8 @@ def main_menu(update: Update, context: CallbackContext) -> int:
 
     if context.user_data["preferences"] is not None:
         buttons.append([IButton(text["reset_shuffle"], callback_data="shuffle_reset")])
+    else:
+        buttons.append([IButton(text["shuffle"], callback_data="shuffle")])
 
     keyboard = IBKeyboard(buttons)
 
@@ -574,6 +576,11 @@ def main():
                 recommender.welcome_to_shuffle,
                 NewShuffleUser(user_data=dp.user_data),
             ),
+            CallbackQueryHandler(
+                recommender.welcome_to_shuffle,
+                "shuffle",
+                pattern=NewShuffleUser(user_data=dp.user_data),
+            ),
             CommandHandler("shuffle", recommender.display_recommendations),
             CallbackQueryHandler(recommender.reset_shuffle, pattern=r"^shuffle_reset$"),
         ],

geniust/data/en.yaml

@@ -10,6 +10,7 @@ main_menu:
   change_language: Change Language
   # login: Log into Genius
   view_accounts: View Account
+  shuffle: Get Song Recommednations
   reset_shuffle: Reset Shuffle Preferences
 
 error: "Sorry. Something went wrong :(\nStart again by clicking /start"

geniust/data/fa.yaml

@@ -10,6 +10,7 @@ main_menu:
   change_language: تغییر زبان ربات
   # login: لاگین‌کردن در جینیس
   view_accounts: مشاهده حساب کاربری
+  shuffle: دریافت آهنگ پیشنهادی
   reset_shuffle: بازنشانی تنظیمات شافل
 
 error: "ببخشید، یه چیزی درست کار نکرد :()\nمی‌تونی با دستور روبرو از اول شروع کنی /start"

geniust/functions/recommender.py

@@ -2,7 +2,7 @@
 import difflib
 from os.path import join
 from itertools import zip_longest
-from typing import Tuple, List, Union, Dict
+from typing import List, Optional, Dict
 from dataclasses import dataclass, asdict
 
 import tekore as tk
@@ -146,6 +146,72 @@ def genres_by_age(self, age: int) -> List[str]:
             age_group = age_groups[-1]
         return self.genres_by_age_group[age_group]
 
+    @log
+    def preferences_from_platform(
+        self, token: str, platform: str
+    ) -> Optional[Preferences]:
+        if platform == "genius":
+            user_genius = api.GeniusT(token)
+            account = user_genius.account()["user"]
+            pyongs = user_genius.user_pyongs(account["id"])
+            pyonged_songs = []
+            for contribution in pyongs["contribution_groups"]:
+                pyong = contribution["contributions"][0]
+                if pyong["pyongable_type"] == "song":
+                    api_path = pyong["pyongable"]["api_path"]
+                    pyonged_songs.append(int(api_path[api_path.rfind("/") + 1 :]))
+
+            public_genius = lg.PublicAPI(timeout=10)
+
+            genres = []
+            artists = []
+            for song_id in pyonged_songs:
+                song = public_genius.song(song_id)["song"]
+                artists.append(song["primary_artist"]["name"])
+                for tag in song["tags"]:
+                    for genre in self.genres:
+                        if genre in tag:
+                            genres.append(genre)
+        else:
+            user_spotify = tk.Spotify(token, sender=tk.RetryingSender())
+            top_tracks = user_spotify.current_user_top_tracks("short_term")
+            top_artists = user_spotify.current_user_top_artists(limit=5)
+            user_spotify.close()
+
+            # Add track genres to genres list
+            genres = []
+            for track in top_tracks.items:
+                track_genres = api.lastfm(
+                    "Track.getTopTags",
+                    {"artist": track.artists[0], "track": track.name},
+                )
+                if "toptags" in track_genres:
+                    for tag in track_genres["toptags"]["tag"]:
+                        for genre in self.genres:
+                            if genre in tag:
+                                genres.append(genre)
+
+            artists = [artist.name for artist in top_artists.items]
+
+        # get count of genres and only keep genres with a >=30% occurance
+        unique_elements, counts_elements = np.unique(genres, return_counts=True)
+        counts_elements = counts_elements.astype(float)
+        counts_elements /= counts_elements.sum()
+        genres = np.asarray((unique_elements, counts_elements))
+        genres = genres[0][genres[1] >= 0.30].tolist()
+
+        if genres:
+            # find user artists in recommender artists
+            found_artists = []
+            for artist in artists:
+                found_artist = self.artists[self.artists.name == artist].name.values
+                if found_artist.size > 0:
+                    found_artists.append(found_artist[0])
+        else:
+            found_artists = []
+
+        return Preferences(genres, found_artists) if genres else None
+
     def search_artist(self, artist: str) -> List[str]:
         artist = artist.lower()
         matches = difflib.get_close_matches(artist, self.lowered_artists_names.keys())
@@ -500,7 +566,7 @@ def select_artists(update: Update, context: CallbackContext):
 
 @log
 @get_user
-def select_language(update: Update, context: CallbackContext):  # pragam: no cover
+def select_language(update: Update, context: CallbackContext):  # pragma: no cover
     language = context.user_data["bot_lang"]
     text = context.bot_data["texts"][language]["select_language"]
     bd = context.bot_data
@@ -534,73 +600,20 @@ def process_preferences(update: Update, context: CallbackContext):
     message = bot.send_message(chat_id, text["getting_data"].format(platform_text))
 
     if platform == "genius":
-        genius_token = context.user_data["genius_token"]
-        user_genius = api.GeniusT(genius_token)
-        account = user_genius.account()["user"]
-        pyongs = user_genius.user_pyongs(account["id"])
-        pyonged_songs = []
-        for contribution in pyongs["contribution_groups"]:
-            pyong = contribution["contributions"][0]
-            if pyong["pyongable_type"] == "song":
-                api_path = pyong["pyongable"]["api_path"]
-                pyonged_songs.append(int(api_path[api_path.rfind("/") + 1 :]))
-
-        public_genius = lg.PublicAPI(timeout=10)
-
-        genres = []
-        artists = []
-        for song_id in pyonged_songs:
-            song = public_genius.song(song_id)["song"]
-            artists.append(song["primary_artist"]["name"])
-            for tag in song["tags"]:
-                for genre in recommender.genres:
-                    if genre in tag:
-                        genres.append(genre)
+        token = context.user_data["genius_token"]
     else:
-        spotify_token = context.user_data["spotify_token"]
-        token = spotify_token
         cred = tk.RefreshingCredentials(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)
-        token = cred.refresh_user_token(spotify_token)
-        user_spotify = tk.Spotify(token, sender=tk.RetryingSender())
-        top_tracks = user_spotify.current_user_top_tracks("short_term")
-        top_artists = user_spotify.current_user_top_artists(limit=5)
-        user_spotify.close()
-
-        # Add track genres to genres list
-        genres = []
-        for track in top_tracks.items:
-            track_genres = api.lastfm(
-                "Track.getTopTags", {"artist": track.artists[0], "track": track.name}
-            )
-            if "toptags" in track_genres:
-                for tag in track_genres["toptags"]["tag"]:
-                    for genre in recommender.genres:
-                        if genre in tag:
-                            genres.append(genre)
+        token = cred.refresh_user_token(context.user_data["spotify_token"])
+
+    preferences = recommender.preferences_from_platform(token, platform)
 
-        artists = [artist.name for artist in top_artists.items]
-
-    # get count of genres and only keep genres with a >=30% occurance
-    unique_elements, counts_elements = np.unique(genres, return_counts=True)
-    counts_elements = counts_elements.astype(float)
-    counts_elements /= counts_elements.sum()
-    genres = np.asarray((unique_elements, counts_elements))
-    genres = genres[0][genres[1] >= 0.30]
-
-    # find user artists in recommender artists
-    found_artists = []
-    for artist in artists:
-        found_artist = recommender.artists[
-            recommender.artists.name == artist
-        ].name.values
-        if found_artist.size > 0:
-            found_artists.append(found_artist[0])
-
-    if not genres:
+    if preferences is None:
         message.edit_text(text["insufficient_data"].format(platform_text))
     else:
-        context.user_data["preferences"] = Preferences(genres, found_artists)
-        context.bot_data["db"].update_preferences(context.user_data["preferences"])
+        context.user_data["preferences"] = preferences
+        context.bot_data["db"].update_preferences(
+            chat_id, context.user_data["preferences"]
+        )
         message.edit_text(text["done"])
 
     return END

geniust/server.py

@@ -102,7 +102,7 @@ def get(self):
         )
         if platform == "genius":
             try:
-                token = self.auths["genius"].get_user_token(redirected_url)
+                token = self.auths["genius"].get_user_token(url=redirected_url)
             except HTTPError as e:
                 self.logger.debug("%s for %s", str(e), state)
                 return
@@ -187,6 +187,47 @@ def get(self):
         self.write(res)
 
 
+class PreferencesHandler(RequestHandler):
+    def initialize(self, auths, recommender) -> None:
+        self.auths = auths
+        self.recommender = recommender
+        self.logger = logging.getLogger(__name__)
+
+    def set_default_headers(self):
+        self.set_header("Content-Type", "application/json")
+
+    @log
+    def get(self):
+        genius_code = self.get_argument("genius_code", default=None)
+        spotify_code = self.get_argument("spotify_code", default=None)
+        response = {"response": {"status_code": 200}}
+        r = response["response"]
+        if genius_code is None and spotify_code is None:
+            self.set_status(404)
+            r["error"] = "404 Not Found"
+            r["status_code"] = 404
+            token = None
+        elif genius_code:
+            token = self.auths["genius"].get_user_token(code=genius_code)
+            platform = "genius"
+        else:
+            token = self.auths["spotify"]._cred.request_user_token(spotify_code)
+            token = token.access_token
+            platform = "spotify"
+
+        if token is not None:
+            preferences = self.recommender.preferences_from_platform(token, platform)
+            if preferences is not None:
+                r["genres"] = preferences.genres
+                r["artists"] = preferences.artists
+            else:
+                r["genres"] = None
+                r["artists"] = None
+
+        res = json.dumps(response)
+        self.write(res)
+
+
 class RecommendationsHandler(RequestHandler):
     def initialize(self, recommender) -> None:
         self.recommender = recommender
@@ -297,6 +338,11 @@ def __init__(
                 ),
                 url(r"/api/genres", GenresHandler, dict(recommender=recommender)),
                 url(r"/api/search", SearchHandler, dict(recommender=recommender)),
+                url(
+                    r"/api/preferences",
+                    PreferencesHandler,
+                    dict(auths=auths, recommender=recommender),
+                ),
                 url(
                     r"/api/recommendations",
                     RecommendationsHandler,

tests/conftest.py

@@ -73,13 +73,13 @@ def search_users_dict(data_path):
         return json.load(f)
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def album_dict(data_path):
     with open(join(data_path, "album.json"), "r") as f:
         return json.load(f)
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def artist_dict(data_path):
     with open(join(data_path, "artist.json"), "r") as f:
         return json.load(f)
@@ -91,18 +91,30 @@ def annotation(data_path):
         return json.load(f)
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def song_dict(data_path):
     with open(join(data_path, "song.json"), "r") as f:
         return json.load(f)
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def user_dict(data_path):
     with open(join(data_path, "user.json"), "r") as f:
         return json.load(f)
 
 
+@pytest.fixture(scope="session")
+def user_pyongs_dict(data_path):
+    with open(join(data_path, "user_pyongs.json"), "r", encoding="utf8") as f:
+        return json.load(f)
+
+
+@pytest.fixture(scope="session")
+def lastfm_track_toptags(data_path):
+    with open(join(data_path, "lastfm_track_toptags.json"), "r") as f:
+        return json.load(f)
+
+
 @pytest.fixture(scope="session")
 def annotations(data_path):
     with open(join(data_path, "annotations.json"), "r") as f:
@@ -115,7 +127,7 @@ def page(data_path):
         return f.read()
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="session")
 def account_dict(data_path):
     with open(join(data_path, "account.json"), "r") as f:
         return json.load(f)

tests/data/lastfm_track_toptags.json

@@ -0,0 +1 @@
+{"toptags":{"tag":[{"count":100,"name":"alternative","url":"https://www.last.fm/tag/alternative"},{"count":87,"name":"alternative rock","url":"https://www.last.fm/tag/alternative+rock"},{"count":69,"name":"rock","url":"https://www.last.fm/tag/rock"},{"count":56,"name":"radiohead","url":"https://www.last.fm/tag/radiohead"},{"count":48,"name":"indie","url":"https://www.last.fm/tag/indie"},{"count":30,"name":"90s","url":"https://www.last.fm/tag/90s"},{"count":28,"name":"british","url":"https://www.last.fm/tag/british"},{"count":18,"name":"experimental","url":"https://www.last.fm/tag/experimental"},{"count":16,"name":"indie rock","url":"https://www.last.fm/tag/indie+rock"},{"count":12,"name":"electronic","url":"https://www.last.fm/tag/electronic"},{"count":12,"name":"britpop","url":"https://www.last.fm/tag/britpop"},{"count":11,"name":"epic","url":"https://www.last.fm/tag/epic"},{"count":11,"name":"Progressive rock","url":"https://www.last.fm/tag/Progressive+rock"},{"count":10,"name":"favorites","url":"https://www.last.fm/tag/favorites"},{"count":9,"name":"melancholic","url":"https://www.last.fm/tag/melancholic"},{"count":8,"name":"beautiful","url":"https://www.last.fm/tag/beautiful"},{"count":7,"name":"paranoid android","url":"https://www.last.fm/tag/paranoid+android"},{"count":6,"name":"psychedelic","url":"https://www.last.fm/tag/psychedelic"},{"count":6,"name":"amazing","url":"https://www.last.fm/tag/amazing"},{"count":6,"name":"Awesome","url":"https://www.last.fm/tag/Awesome"},{"count":5,"name":"UK","url":"https://www.last.fm/tag/UK"},{"count":5,"name":"melancholy","url":"https://www.last.fm/tag/melancholy"},{"count":5,"name":"art rock","url":"https://www.last.fm/tag/art+rock"},{"count":4,"name":"Favourites","url":"https://www.last.fm/tag/Favourites"},{"count":4,"name":"Masterpiece","url":"https://www.last.fm/tag/Masterpiece"},{"count":4,"name":"ergo proxy","url":"https://www.last.fm/tag/ergo+proxy"},{"count":4,"name":"favorite songs","url":"https://www.last.fm/tag/favorite+songs"},{"count":4,"name":"Experimental Rock","url":"https://www.last.fm/tag/Experimental+Rock"},{"count":4,"name":"1997","url":"https://www.last.fm/tag/1997"},{"count":4,"name":"ok computer","url":"https://www.last.fm/tag/ok+computer"},{"count":4,"name":"seen live","url":"https://www.last.fm/tag/seen+live"},{"count":3,"name":"sad","url":"https://www.last.fm/tag/sad"},{"count":3,"name":"post-rock","url":"https://www.last.fm/tag/post-rock"},{"count":3,"name":"Love","url":"https://www.last.fm/tag/Love"},{"count":3,"name":"genius","url":"https://www.last.fm/tag/genius"},{"count":3,"name":"Mellow","url":"https://www.last.fm/tag/Mellow"},{"count":3,"name":"Favorite","url":"https://www.last.fm/tag/Favorite"},{"count":3,"name":"thom yorke","url":"https://www.last.fm/tag/thom+yorke"},{"count":3,"name":"guitar","url":"https://www.last.fm/tag/guitar"},{"count":3,"name":"male vocalists","url":"https://www.last.fm/tag/male+vocalists"},{"count":3,"name":"anime","url":"https://www.last.fm/tag/anime"},{"count":3,"name":"great lyrics","url":"https://www.last.fm/tag/great+lyrics"},{"count":3,"name":"best song ever","url":"https://www.last.fm/tag/best+song+ever"},{"count":3,"name":"ambient","url":"https://www.last.fm/tag/ambient"},{"count":3,"name":"Favourite Songs","url":"https://www.last.fm/tag/Favourite+Songs"},{"count":2,"name":"Soundtrack","url":"https://www.last.fm/tag/Soundtrack"},{"count":2,"name":"favourite","url":"https://www.last.fm/tag/favourite"},{"count":2,"name":"perfect","url":"https://www.last.fm/tag/perfect"},{"count":2,"name":"FUCKING AWESOME","url":"https://www.last.fm/tag/FUCKING+AWESOME"},{"count":2,"name":"electronica","url":"https://www.last.fm/tag/electronica"},{"count":2,"name":"pop","url":"https://www.last.fm/tag/pop"},{"count":2,"name":"Progressive","url":"https://www.last.fm/tag/Progressive"},{"count":2,"name":"brilliant","url":"https://www.last.fm/tag/brilliant"},{"count":2,"name":"All time favourites","url":"https://www.last.fm/tag/All+time+favourites"},{"count":2,"name":"classic rock","url":"https://www.last.fm/tag/classic+rock"},{"count":2,"name":"chillout","url":"https://www.last.fm/tag/chillout"},{"count":2,"name":"chill","url":"https://www.last.fm/tag/chill"},{"count":2,"name":"classic","url":"https://www.last.fm/tag/classic"},{"count":2,"name":"brit rock","url":"https://www.last.fm/tag/brit+rock"},{"count":2,"name":"paranoid","url":"https://www.last.fm/tag/paranoid"},{"count":2,"name":"best","url":"https://www.last.fm/tag/best"},{"count":2,"name":"Alternative  Punk","url":"https://www.last.fm/tag/Alternative++Punk"},{"count":2,"name":"moody","url":"https://www.last.fm/tag/moody"},{"count":2,"name":"great song","url":"https://www.last.fm/tag/great+song"},{"count":2,"name":"loved","url":"https://www.last.fm/tag/loved"},{"count":2,"name":"alt rock","url":"https://www.last.fm/tag/alt+rock"},{"count":2,"name":"instrumental","url":"https://www.last.fm/tag/instrumental"},{"count":2,"name":"memories","url":"https://www.last.fm/tag/memories"},{"count":2,"name":"dramatic","url":"https://www.last.fm/tag/dramatic"},{"count":2,"name":"best songs ever","url":"https://www.last.fm/tag/best+songs+ever"},{"count":2,"name":"i had to change my pants after this song","url":"https://www.last.fm/tag/i+had+to+change+my+pants+after+this+song"},{"count":2,"name":"ridiculously awesomely good","url":"https://www.last.fm/tag/ridiculously+awesomely+good"},{"count":2,"name":"haunting","url":"https://www.last.fm/tag/haunting"},{"count":2,"name":"emotional","url":"https://www.last.fm/tag/emotional"},{"count":2,"name":"Psychedelic Rock","url":"https://www.last.fm/tag/Psychedelic+Rock"},{"count":2,"name":"male vocalist","url":"https://www.last.fm/tag/male+vocalist"},{"count":2,"name":"want to see live","url":"https://www.last.fm/tag/want+to+see+live"},{"count":2,"name":"psycho","url":"https://www.last.fm/tag/psycho"},{"count":2,"name":"1990s","url":"https://www.last.fm/tag/1990s"},{"count":2,"name":"Good Stuff","url":"https://www.last.fm/tag/Good+Stuff"},{"count":2,"name":"brit pop","url":"https://www.last.fm/tag/brit+pop"},{"count":2,"name":"british rock","url":"https://www.last.fm/tag/british+rock"},{"count":2,"name":"Soundtrack Of My Life","url":"https://www.last.fm/tag/Soundtrack+Of+My+Life"},{"count":1,"name":"dark","url":"https://www.last.fm/tag/dark"},{"count":1,"name":"post rock","url":"https://www.last.fm/tag/post+rock"},{"count":1,"name":"Ballad","url":"https://www.last.fm/tag/Ballad"},{"count":1,"name":"Awesome Guitar Jams","url":"https://www.last.fm/tag/Awesome+Guitar+Jams"},{"count":1,"name":"love at first listen","url":"https://www.last.fm/tag/love+at+first+listen"},{"count":1,"name":"nostalgia","url":"https://www.last.fm/tag/nostalgia"},{"count":1,"name":"grey storia","url":"https://www.last.fm/tag/grey+storia"}],"@attr":{"artist":"Radiohead","track":"Paranoid Android"}}}