v2.2 (#3)

* archive coverage.xml and run upload_coverage only if pushing to master * removed lint job and added it in tox * added handling invalid code parameter in Genius callback * reduced test time significantly by resetting mocks instead of creating new ones for every test (context and update fixtures) * added comment lines to visually separate fixtures in conftest * added creating users in context.user_data dynamically by language * add development branch to github workflow * apply black formatting * make black check all files * fix TokenHandler callback endpoint * fix test_token_handler * applied black formatting * - added bot's Telegram link to readme - fixed typo in fa.yaml * - fixed accessing tests using bot instead of bot_data - fixed typo in en texts * - added music recommender - refactored some parts to fit the music recommender in * added scikit-learn to requirements * removed test raise * added logging for badrequest error * fixed missing argument error in token request handler * - fixed token value error for spotify in token handler - fixed updating correct token in database * switch to refreshing credentials to get refreshing token * fix insert error * use tuple instead of list in upsert * add recommender api to webserver * fix wrong import * fix attribute access * improve matches for artist search by lowering possibilities and decreasing cutoff * - added parameters to the recommendations endpoint - added more fields to the tracks * fix concat error * fix comparison issue for np.nan values * replace np.nan with None in tracks * fixed key in response * return artist with correct case in recommender.search_artist * removed redundant search for song file * - store download_url in user data since datasets already have it - fixed issue where song artist wasn't found (songs with no corresponding artist in artists) * songs dataframe keeps genre column, fixed shuffle returning wrong results * removed has_download_url and has_preview_url and replaced it with song_type parameter * fix typo * added temporary fix for when displaying lyrics fails * - recommender now returns Song objects which have ids (row index) - language parameter was removed in favor of the "persian" genre - added recommender.song method * added preview and download link when displaying recommended songs * add persian to genres if user language is "fa" * fix display of song and its urls * end all returns ending in fallback for shuffle conv handler * disabled error handler * fixed sending artist object instead of name * added debug statement * fixed getting song by id_spotify from recommender * improve audio info when sending song * fix spotify artist name in download_song * fix typo * add caption when sending audio from display_song * add parameter to function logger * Revert "add parameter to function logger" This reverts commit 4f77442. * removed logging function return value * convert it to int * add download button for song from external source * fix spotify search issues * fixed UnboundLocalError * improve spotify search matching * fix typo * update outdated info * fix RuntimeWarning await error * add debug statement * fix typo * fixed passing bs4 object to bs4 * improve spotify search matching by removing space char * added all activities count to account display * changed forcing all user genres in song to only forcing one of user genres in song * added user endpoint * added user endpoint to main menu, added TYPING_USER constant * added searching songs by lyrics * fixed message handler redirecting user back to typing user, fixed search lyrics not in inline menu choices, fixed lyrics search hits number, * fixed no effective_chat from inline queries * fixed empty input text in inline query * fixed no pageview in stats for inline queries * added quotes to hrefs * moved thumb image to beginning of caption * fixed inline query result bugs * remove extra value for user query data * removed extra argument to user_caption * added cleaning user description * remove last message after user clicks on a search hit, don't remove inline buttons of entities so user can use them again * added head method to cron handler * fix deleting callback query message * fix displaying user in persian, improve lyrics search formatting * update persian inline query formatting * improved "hot" translation in persian * fixed typos * moved web server to server.py * updated readme * fix img link in readme * fix title link and add more info to Shuffle * applied black formatting * removed unused var in loop * change db.update type hint to Any, ensure values in a tuple/list * rewrite genres_by_age to not use list comprehension for finding item * add type hint to display_user buttons var * use genres_by_age in input_age * fixed keyboard button in select_language * remove extra if statement in display_artist * remove extra if statement * change format_language return type to Any to stop mypy complaints * applied black formatting * add allowing redefinition to mypy * added spotify credentials to github workflow * remove lastfm api key * tests: add new objects to bot_data * move server tests to test_server.py * add test_server.py * updated test_server tests * tests: moved recommender to own fixture * tests: added tests for server * tests: add Sender and FaMusic to no coverage * tests: update user_data and fix bot tests * tests: update and add tests to db * tests: update and add tests to utils * added returning END to login_choices * tests: update and add tests to account * tests: update album tests * tests: update annotations tests * tests: update artist tests * fix displaying user with no roles * tests: update and added tests to inline_query * remove extra append * remove extra append * fixed selecting song with id = 0 * tests: added tests for recommender * tests: add test_recommender.py * tests: update and add song tests * tests: added user tests * tests: added missing import to test_user * applied black formatting * fixed mypy errors (ignored error lines) * fixed flake8 errors * applied black formatting to tests * tests: added checking login_choices return value * removed lastfm, FaMusic and Sender * change lyricsgenius dep to fork * revert removing lastfm * add no coverage to lastfm * add lastfm key to constants * remove FaMusic in bot * declare lastfm key in github workflow * fix merge issue of last commit * bump version * bump cryptography to 3.4.5
allerter · Feb 16, 2021 · 7142a46 · 7142a46
1 parent e70a846
commit 7142a46
Show file tree

Hide file tree

Showing 48 changed files with 57,437 additions and 765 deletions.
diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml
@@ -40,6 +40,10 @@ jobs:
         GENIUS_CLIENT_ID: ${{ secrets.GENIUS_CLIENT_ID }}
         GENIUS_REDIRECT_URI: ${{ secrets.GENIUS_REDIRECT_URI }}
         GENIUS_CLIENT_SECRET: ${{ secrets.GENIUS_CLIENT_SECRET }}
+        SPOTIFY_CLIENT_ID: ${{ secrets.SPOTIFY_CLIENT_ID }}
+        SPOTIFY_REDIRECT_URI: ${{ secrets.SPOTIFY_REDIRECT_URI }}
+        SPOTIFY_CLIENT_SECRET: ${{ secrets.SPOTIFY_CLIENT_SECRET }}
+        LASTFM_API_KEY: ${{ secrets.LASTFM_API_KEY }}
       run: tox
 
     - name: Archive code coverage results

diff --git a/README.md b/README.md
@@ -1,67 +1,160 @@
-<h1 align="center"><img src="logo.png" width="200" align="center"/>Genius T</h1>
+<h1>
+  <img src="logo.png" alt="GeniusT logo" width="200" align="center"/>
+  Genius T
+</h1>
+
 
 ![status](https://img.shields.io/uptimerobot/status/m786636302-b2fa3edeb9237ae327f70d06)
-![GitHub release (latest by date)](https://img.shields.io/github/v/release/Allerter/geniust)
+![GitHub release (latest by
+date)](https://img.shields.io/github/v/release/Allerter/geniust)
 ![build](https://github.com/Allerter/geniust/workflows/build/badge.svg)
-[![Test Coverage](https://api.codeclimate.com/v1/badges/74d5611d77cb26f4ed16/test_coverage)](https://codeclimate.com/github/Allerter/geniust/test_coverage)
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Test
+Coverage](https://api.codeclimate.com/v1/badges/74d5611d77cb26f4ed16/test_coverage)](https://codeclimate.com/github/Allerter/geniust/test_coverage)
+[![Code style:
+black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Telegram
+Link](https://img.shields.io/static/v1?label=Telegram&message=Click%20Here&color=blue&logo=telegram)](https://t.me/genius_the_bot)
 
 A Telegram bot that provides music data and lyrics from Genius.
 
 ## Features
-> -   [Music Data](#music-data)
-> -   [Logging Into Genius](#logging-into-genius)
+
+> -   [Genius/Music Data](#geniusmusic-data)
+> -   [Genius Account](#genius-account)
+> -   [GeniusT Shuffle](#geniust-shuffle)
 > -   [Song Lyircs](#song-lyircs)
 > -   [Album Lyrics](#album-lyrics)
-> -   [Telegram features](#deploying-to-heroku)
+> -   [Telegram features](#telegram-features)
+
+### Genius/Music Data
+
+GeniusT allows searching for songs, albums, artists, users, and song
+lyrics. Furthermore, users can access tracks in albums, information
+about those tracks, view the artist info of those tracks, and about any
+information that one could want! All entities (albums, artists, and
+songs) are deep-linked; for example, users can view the artist of a song
+just by clicking on the artist\'s name, and from there they can go on to
+viewing that artist\'s songs, albums, the features on those songs and
+all that can be discovered!
+
+### Genius Account
+
+Using Genius\'s OAuth2, users can easily log into Genius in the bot and
+view their account\'s details (including unread messages), and even vote
+on album/song descriptions when viewing them.
+
+### GeniusT Shuffle
+<img src="geniust/data/shuffle.jpg" alt="GeniusT Shuffle logo" width="480"/>
+GeniusT offers a basic genre-based music recommendation system that
+offers users song recommendations from about 20K songs based on their
+favorite genres and artists. Users can get their preferences from their
+Genius or Spotify account or enter them manually. By logging into their
+account through OAuth2, GeniusT will try to generate user's preferences
+based on their account activity. If the user chooses to go the manual route,
+they either input their age (and let the bot guess
+the genres) or select them manually. Afterward, users can also add their
+favorite artists from the available ones or finish without any favorite
+artists (each user must have at least one favorite genre). Available
+genres:
+
+-   Classical
+-   Country
+-   Instrumental
+-   Persian
+-   Pop
+-   Rap
+-   R&B
+-   Rock
+-   Traditional
 
+The *Persian* genre acts as a language parameter and a user with that
+genre only gets songs that have the Persian genre; otherwise would get
+non-Persian (mostly English) songs. The English songs were taken
+randomly from the [LastFM 2020
+dataset](https://github.com/renesemela/lastfm-dataset-2020). The Persian
+songs are also retrieved from LastFM, but I neglected to save the
+scripts I used to get the songs. Further information for songs was
+retrieved from Spotify, Deezer, and Genius.
 
-### Music Data
-GeniusT allows searching for songs, albums and artists. Furthermore you can access tracks in albums, information about those tracks, view the artist info of those tracks and about any information that you could want! All entities (albums, artists and songs) are deep-linked; for example you can view the artist of a song just by clicking on the artist's name and from there you can go on to viewing that artist's songs, albums, the features on those songs and all that you can discover!
+When a user\'s preferences are sent to the recommender, it will get all
+songs that have one of the user\'s genres. Then the recommender chooses
+a random subset of those songs to avoid sending the same songs each
+time. Afterward, if the user had any favorite artists, the user
+artists\' descriptions are compared with the subset songs artists\'
+descriptions using TF-IDS to sort the songs based on the most
+similarity. Then the 5 most similar songs will be returned.
 
-### Logging Into Genius
-Using Genius's OAuth2, you can easily log into Genius in the bot and view your account's details (including unread messages) and even vote on album/song descriptions when you view them.
+All of the English songs have a preview URL, but because of copyright
+laws have no download URL (full song URL). Although the users can
+download the song from a link to another Telegram bot. Some Persian
+songs have a preview URL and some a direct download URL for a 128-bit
+MP3 file.
+
+Users can start using shuffle by sending the /shuffle command. After
+submitting their genres and adding their favorite artists, their
+preferences are saved. From then on, sending /shuffle will return 5
+recommended songs based on their preferences. For any of the 5 songs, if
+they can be found on Genius, they\'ll be deep-linked to be viewed. There
+also be deep-links for preview and download URLs if the song has a
+preview URL for song previews or a download URL or ISRC ID for
+downloads. The bot will send the full song if the song has a direct
+download URL, but if it has an ISRC ID, users will be directed to
+another bot.
 
 ### Song Lyrics
-The bot provides song lyrics using Telegram messages. The annotated fragments of songs are linked to their annotations which are uploaded by the bot in a separate channel meant for annotations. Meaning when you view lyrics for a song, you can click on highlighted lyrics that will lead you to their annotations right in Telegram.
 
+The bot provides song lyrics using Telegram messages. The annotated
+fragments of songs are linked to their annotations which are uploaded by
+the bot in a separate channel meant for annotations. Meaning when users
+view lyrics for a song, they can click on highlighted lyrics that will
+lead them to the annotations right there in Telegram.
 
 ### Album Lyrics
-As for the albums, you can either view the tracks and their lyrics one-by-one (as you normally would) or you could have the lyrics of all the tracks all in one place in three formats:
-- PDF: A PDF containing the album description, a table of contents containing song titles linked to the page of the song lyrics, and the lyrics of all the songs in the album.
-- ZIP: A ZIP file containing TXT files of each song's lyrics.
-- TELEGRA.PH: Returns a link to a *telegra.ph* page which in turn has the album's description and links to other *telegra.ph* pages that each has the description and the lyrics of a song.
-This feature isn't maintained like other parts of the bot, so some things might look off.
+
+As for the albums, users can either view the tracks and their lyrics
+one-by-one (as they normally would) or they could have the lyrics of all
+the tracks all in one place in three formats:
+
+-   PDF: A PDF containing the album description, a table of contents
+    containing song titles linked to the page of the song lyrics, and
+    the lyrics of all the songs in the album.
+-   ZIP: A ZIP file containing TXT files of each song\'s lyrics.
+-   TELEGRA.PH: Returns a link to a *telegra.ph* page which in turn has
+    the album\'s description and links to other *telegra.ph* pages that
+    each has the description and the lyrics of a song. This feature
+    isn\'t maintained, so some things might look off.
 
 ### Lyrics Customizations
-The bot allows users to choose between including and excluding the annotations from the lyrics. Users can also choose not to include ASCII or non-ASCII characters in the lyrics. One of the uses of this could be to remove English lines from songs that have been translated into Arabic or Persian so that only the translated lines remain.
 
+The bot allows users to choose between including and excluding the
+annotations from the lyrics. Users can also choose not to include ASCII
+or non-ASCII characters in the lyrics. One of the uses of this could be
+to remove English lines from songs that have been translated into Arabic
+or Persian so that only the translated lines remain.
 
 ### Telegram Features
-Users can search Genius navigating the iline menu which can be accessed using the `/start` command. Alternatively, you could directly reach to the desired feature using commands:
-- **/start**
-  start the bot
-- **/song**:
-  search for a song
-- **/album**:
-  search for an album
-- **/artist**:
-  search for an artist
-- **/lyrics_language**:
-  set lyrics language
-- **/include_annotations**:
-  include annotations
-- **/bot_language**:
-  set bot language
-- **/cancel**:
-  cancel the current task
-- **/help**:
-  more info about the bot
-- **/contact_us**:
-  send us a message
-
-You can also perform searches by using the inline search feature. For example:
-Just type in the bot's username like `@the_bot ` to get a help menu. Alternatively:
-- searching songs: `@the_bot .song we will rock you`
-- searching albums: `@the_bot .album hotel diablo`
-- searching artists: `@the_bot .artist Queen`
+
+Users can search Genius navigating the iline menu which can be accessed
+using the `/start` command. Alternatively, you could directly reach to
+the desired feature using commands:
+
+-   **/start** start the bot
+-   **/song**: search for a song
+-   **/album**: search for an album
+-   **/artist**: search for an artist
+-   **/user**: search for a user
+-   **/shuffle**: Get music recommendation
+-   **/lyrics_language**: set lyrics language
+-   **/include_annotations**: include annotations
+-   **/bot_language**: set bot language
+-   **/cancel**: cancel the current task
+-   **/help**: more info about the bot
+-   **/contact_us**: send us a message
+
+Users can also perform searches by using the inline search feature. For
+example:
+
+-   searching songs: `@genius_the_bot .song we will rock you`
+-   searching albums: `@genius_the_bot .album hotel diablo`
+-   searching artists: `@genius_the_bot .artist Queen`
+-   searching users: `@genius_the_bot .user lemonade`
diff --git a/geniust/VERSION b/geniust/VERSION
@@ -1 +1 @@
-2.0.0
+2.2.0
diff --git a/geniust/__init__.py b/geniust/__init__.py
@@ -5,23 +5,23 @@
 from typing import TypeVar, Callable
 
 # from telegram import Bot
+import tekore as tk
 from lyricsgenius import OAuth2
 import yaml
 
 from geniust.db import Database
-from geniust.api import GeniusT
 from geniust.constants import (
-    # BOT_TOKEN,
     GENIUS_CLIENT_ID,
     GENIUS_REDIRECT_URI,
     GENIUS_CLIENT_SECRET,
+    SPOTIFY_CLIENT_ID,
+    SPOTIFY_CLIENT_SECRET,
+    SPOTIFY_REDIRECT_URI,
 )
 
 username: str = "genius_the_bot"  # Bot(BOT_TOKEN).get_me().username
 
-database = Database(table="user_data")
-
-genius = GeniusT()
+database = Database("user_data", "user_preferences")
 
 RT = TypeVar("RT")
 
@@ -36,8 +36,12 @@ def get_user(func: Callable[..., RT]) -> Callable[..., RT]:
 
     @functools.wraps(func)
     def wrapper(*args, **kwargs) -> RT:
-        chat_id = args[0].effective_chat.id
+        update = args[0]
         context = args[1]
+        if update.effective_chat:
+            chat_id = update.effective_chat.id
+        else:
+            chat_id = update.inline_query.from_user.id
         if "bot_lang" not in context.user_data:
             database.user(chat_id, context.user_data)
         result = func(*args, **kwargs)
@@ -46,15 +50,23 @@ def wrapper(*args, **kwargs) -> RT:
     return wrapper
 
 
-here = pathlib.Path(__file__).parent.resolve()
-path = here / "text"
-files = [f for f in listdir(path) if isfile(join(path, f)) and f.endswith(".yaml")]
+data_path = pathlib.Path(__file__).parent.resolve() / "data"
+files = [
+    f for f in listdir(data_path) if isfile(join(data_path, f)) and f.endswith(".yaml")
+]
 texts = {}
 for file in files:
-    with open(join(path, file), "r", encoding="utf8") as f:
+    with open(join(data_path, file), "r", encoding="utf8") as f:
         language = file[:2]
         texts[language] = yaml.full_load(f)
 
-auth = OAuth2.full_code_exchange(
+genius_auth = OAuth2.full_code_exchange(
     GENIUS_CLIENT_ID, GENIUS_REDIRECT_URI, GENIUS_CLIENT_SECRET, scope=("me", "vote")
 )
+spotify_auth = tk.UserAuth(
+    tk.RefreshingCredentials(
+        SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET, SPOTIFY_REDIRECT_URI
+    ),
+    scope=tk.scope.user_top_read,
+)
+auths = {"genius": genius_auth, "spotify": spotify_auth}