Add support for new languages

notebook/yohane.ipynb

@@ -167,7 +167,7 @@
       "outputs": [],
       "source": [
         "# @title Generate\n",
-        "# @markdown **Replace the song filename here if you uploaded it manually**\n",
+        "# @markdown **Replace the song filename here if you uploaded it manually, and don't forget to change the language of the song.**\n",
         "\n",
         "import logging\n",
         "from pathlib import Path\n",
@@ -176,10 +176,11 @@
         "logging.basicConfig(level=\"INFO\", force=True)\n",
         "\n",
         "song_filename = \"song\" # @param {type:\"string\"}\n",
+        "language = \"jp\" # @param {type: \"string\"}\n",
         "\n",
         "yohane = Yohane(vocals_extractor)\n",
         "yohane.load_song(Path(song_filename))\n",
-        "yohane.load_lyrics(lyrics_area.value)\n",
+        "yohane.load_lyrics(lyrics_area.value, language)\n",
         "yohane.extract_vocals()\n",
         "yohane.force_align()\n",
         "subs = yohane.make_subs()\n"
@@ -236,7 +237,7 @@
     },
     "language_info": {
       "name": "python",
-      "version": "3.11.6"
+      "version": "3.11.0"
     }
   },
   "nbformat": 4,

pyproject.toml

@@ -18,6 +18,7 @@ vocal-remover = {git = "https://github.com/Japan7/vocal-remover.git", rev = "24f
 pysubs2 = "^1.6.1"
 regex = "^2023.10.3"
 click = "^8.1.7"
+pyphen = "^0.14.0"
 
 [tool.poetry.extras]
 torch = ["torch", "torchaudio"]

yohane/__main__.py

@@ -5,6 +5,7 @@
 
 import click
 import torchaudio
+from pyphen import LANGUAGES
 
 from yohane.audio import (
     HybridDemucsVocalsExtractor,
@@ -46,7 +47,14 @@
     show_default=True,
     help="Vocals extractor to use. 'None' to disable.",
 )
-def cli(song_file: Path, lyrics_file: TextIOWrapper, vocals_extractor: str):
+@click.option(
+    "-l",
+    "--language",
+    type=click.Choice(list(LANGUAGES.keys()) + ["jp"], case_sensitive=False),
+    default="jp",
+    help="Language of the lyrics from the available LibreOffice dictionaries. Default is Japanese (jp)."
+)
+def cli(song_file: Path, lyrics_file: TextIOWrapper, vocals_extractor: str, language: str):
     logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO").upper())
 
     extractor_cls = CLI_VOCALS_EXTRACTORS_OPTS[vocals_extractor]
@@ -55,7 +63,7 @@ def cli(song_file: Path, lyrics_file: TextIOWrapper, vocals_extractor: str):
     yohane = Yohane(extractor)
 
     yohane.load_song(song_file)
-    yohane.load_lyrics(lyrics_file.read())
+    yohane.load_lyrics(lyrics_file.read(), language)
 
     yohane.extract_vocals()
     if yohane.vocals is not None:

yohane/lyrics.py

@@ -2,11 +2,12 @@
 from functools import cached_property
 
 import regex as re
-
+import pyphen
 
 @dataclass
 class _Text:
     raw: str
+    language: str
 
     @cached_property
     def normalized(self):
@@ -21,21 +22,28 @@ def transcript(self):
 class Lyrics(_Text):
     @cached_property
     def lines(self):
-        return [Line(line) for line in filter(None, self.raw.splitlines())]
+        return [Line(line, self.language) for line in filter(None, self.raw.splitlines())]
 
 
 @dataclass
 class Line(_Text):
     @cached_property
     def words(self):
-        return [Word(word) for word in filter(None, self.transcript)]
+        return [Word(word, self.language) for word in filter(None, self.transcript)]
 
 
 @dataclass
 class Word(_Text):
     @cached_property
     def syllables(self):
-        return auto_split(self.normalized)
+        res = self.normalized
+        if self.language == 'jp':
+            res = auto_split(res)
+        else:
+            dic = pyphen.Pyphen(lang=self.language)
+            res = dic.inserted(res)
+            res = res.split("-")
+        return res
 
 
 def normalize_uroman(text: str):

yohane/pipeline.py

@@ -34,9 +34,9 @@ def extract_vocals(self):
             assert self.song
             self.vocals = self.vocals_extractor(*self.song)
 
-    def load_lyrics(self, lyrics_str: str):
+    def load_lyrics(self, lyrics_str: str, language: str):
         logger.info("Loading lyrics")
-        self.lyrics = Lyrics(lyrics_str)
+        self.lyrics = Lyrics(lyrics_str, language)
 
     def force_align(self):
         logger.info("Computing forced alignment")