From ff156dd2f2add4779830170cf5937ead168e799e Mon Sep 17 00:00:00 2001 From: andy840119 Date: Mon, 17 Jul 2023 23:01:01 +0800 Subject: [PATCH 1/8] Separate the method. --- .../RomajiTags/Ja/JaRomajiTagGenerator.cs | 27 ++++++++++++------- .../Lyrics/RubyTags/Ja/JaRubyTagGenerator.cs | 20 +++++++------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RomajiTags/Ja/JaRomajiTagGenerator.cs b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RomajiTags/Ja/JaRomajiTagGenerator.cs index 8261ce4f3..0faca61ab 100644 --- a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RomajiTags/Ja/JaRomajiTagGenerator.cs +++ b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RomajiTags/Ja/JaRomajiTagGenerator.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; +using System.Linq; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Ja; using Lucene.Net.Analysis.TokenAttributes; @@ -29,12 +30,17 @@ public JaRomajiTagGenerator(JaRomajiTagGeneratorConfig config) protected override RomajiTag[] GenerateFromItem(Lyric item) { - string text = item.Text; - var processingTags = new List(); - // Tokenize the text + string text = item.Text; var tokenStream = analyzer.GetTokenStream("dummy", new StringReader(text)); + var processingRomajies = getProcessingRomajies(text, tokenStream, Config).ToArray(); + + return Convert(processingRomajies).ToArray(); + } + + private static IEnumerable getProcessingRomajies(string text, TokenStream tokenStream, JaRomajiTagGeneratorConfig config) + { // Get result and offset var result = tokenStream.GetAttribute(); var offsetAtt = tokenStream.GetAttribute(); @@ -58,11 +64,11 @@ protected override RomajiTag[] GenerateFromItem(Lyric item) // Convert to romaji. string romaji = JpStringUtils.ToRomaji(katakana); - if (Config.Uppercase.Value) + if (config.Uppercase.Value) romaji = romaji.ToUpper(); // Make tag - processingTags.Add(new RomajiTagGeneratorParameter + yield return new RomajiTagGeneratorParameter { FromKanji = fromKanji, RomajiTag = new RomajiTag @@ -71,19 +77,22 @@ protected override RomajiTag[] GenerateFromItem(Lyric item) StartIndex = offsetAtt.StartOffset, EndIndex = offsetAtt.EndOffset - 1, }, - }); + }; } // Dispose tokenStream.End(); tokenStream.Dispose(); + } + internal static IEnumerable Convert(RomajiTagGeneratorParameter[] tags) + { var romajiTags = new List(); - foreach (var processingTag in processingTags) + foreach (var processingTag in tags) { // combine romajies of they are not from kanji. - var previousProcessingTag = processingTags.GetPrevious(processingTag); + var previousProcessingTag = tags.GetPrevious(processingTag); bool fromKanji = processingTag.FromKanji; if (previousProcessingTag != null && !fromKanji) @@ -98,7 +107,7 @@ protected override RomajiTag[] GenerateFromItem(Lyric item) } } - return romajiTags.ToArray(); + return romajiTags; } internal class RomajiTagGeneratorParameter diff --git a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RubyTags/Ja/JaRubyTagGenerator.cs b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RubyTags/Ja/JaRubyTagGenerator.cs index b380216e5..a5a26ac65 100644 --- a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RubyTags/Ja/JaRubyTagGenerator.cs +++ b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RubyTags/Ja/JaRubyTagGenerator.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; +using System.Linq; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Ja; using Lucene.Net.Analysis.TokenAttributes; @@ -27,12 +28,15 @@ public JaRubyTagGenerator(JaRubyTagGeneratorConfig config) protected override RubyTag[] GenerateFromItem(Lyric item) { - string text = item.Text; - var tags = new List(); - // Tokenize the text + string text = item.Text; var tokenStream = analyzer.GetTokenStream("dummy", new StringReader(text)); + return getProcessingRubyTags(text, tokenStream, Config).ToArray(); + } + + private static IEnumerable getProcessingRubyTags(string text, TokenStream tokenStream, JaRubyTagGeneratorConfig config) + { // Get result and offset var result = tokenStream.GetAttribute(); var offsetAtt = tokenStream.GetAttribute(); @@ -54,7 +58,7 @@ protected override RubyTag[] GenerateFromItem(Lyric item) // Convert to Hiragana as default. string hiragana = JpStringUtils.ToHiragana(katakana); - if (!Config.EnableDuplicatedRuby.Value) + if (!config.EnableDuplicatedRuby.Value) { // Not add duplicated ruby if same as parent. string parentText = text[offsetAtt.StartOffset..offsetAtt.EndOffset]; @@ -63,18 +67,16 @@ protected override RubyTag[] GenerateFromItem(Lyric item) } // Make tag - tags.Add(new RubyTag + yield return new RubyTag { - Text = Config.RubyAsKatakana.Value ? katakana : hiragana, + Text = config.RubyAsKatakana.Value ? katakana : hiragana, StartIndex = offsetAtt.StartOffset, EndIndex = offsetAtt.EndOffset - 1, - }); + }; } // Dispose tokenStream.End(); tokenStream.Dispose(); - - return tags.ToArray(); } } From dff4aed43ca651d338bc26b591bc6dea10ecefe2 Mon Sep 17 00:00:00 2001 From: andy840119 Date: Sun, 23 Jul 2023 00:41:41 +0800 Subject: [PATCH 2/8] Refactor the order. --- .../Lyrics/RomajiTags/Ja/JaRomajiTagGenerator.cs | 16 ++++++++-------- .../Lyrics/RubyTags/Ja/JaRubyTagGenerator.cs | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RomajiTags/Ja/JaRomajiTagGenerator.cs b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RomajiTags/Ja/JaRomajiTagGenerator.cs index 0faca61ab..b7720e581 100644 --- a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RomajiTags/Ja/JaRomajiTagGenerator.cs +++ b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RomajiTags/Ja/JaRomajiTagGenerator.cs @@ -41,10 +41,6 @@ protected override RomajiTag[] GenerateFromItem(Lyric item) private static IEnumerable getProcessingRomajies(string text, TokenStream tokenStream, JaRomajiTagGeneratorConfig config) { - // Get result and offset - var result = tokenStream.GetAttribute(); - var offsetAtt = tokenStream.GetAttribute(); - // Reset the stream and convert all result tokenStream.Reset(); @@ -54,12 +50,16 @@ private static IEnumerable getProcessingRomajies(st tokenStream.ClearAttributes(); tokenStream.IncrementToken(); + // Get result and offset + var charTermAttribute = tokenStream.GetAttribute(); + var offsetAttribute = tokenStream.GetAttribute(); + // Get parsed result, result is Katakana. - string katakana = result.ToString(); + string katakana = charTermAttribute.ToString(); if (string.IsNullOrEmpty(katakana)) break; - string parentText = text[offsetAtt.StartOffset..offsetAtt.EndOffset]; + string parentText = text[offsetAttribute.StartOffset..offsetAttribute.EndOffset]; bool fromKanji = JpStringUtils.ToKatakana(katakana) != JpStringUtils.ToKatakana(parentText); // Convert to romaji. @@ -74,8 +74,8 @@ private static IEnumerable getProcessingRomajies(st RomajiTag = new RomajiTag { Text = romaji, - StartIndex = offsetAtt.StartOffset, - EndIndex = offsetAtt.EndOffset - 1, + StartIndex = offsetAttribute.StartOffset, + EndIndex = offsetAttribute.EndOffset - 1, }, }; } diff --git a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RubyTags/Ja/JaRubyTagGenerator.cs b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RubyTags/Ja/JaRubyTagGenerator.cs index a5a26ac65..49c54d143 100644 --- a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RubyTags/Ja/JaRubyTagGenerator.cs +++ b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/RubyTags/Ja/JaRubyTagGenerator.cs @@ -37,10 +37,6 @@ protected override RubyTag[] GenerateFromItem(Lyric item) private static IEnumerable getProcessingRubyTags(string text, TokenStream tokenStream, JaRubyTagGeneratorConfig config) { - // Get result and offset - var result = tokenStream.GetAttribute(); - var offsetAtt = tokenStream.GetAttribute(); - // Reset the stream and convert all result tokenStream.Reset(); @@ -50,8 +46,12 @@ private static IEnumerable getProcessingRubyTags(string text, TokenStre tokenStream.ClearAttributes(); tokenStream.IncrementToken(); + // Get result and offset + var charTermAttribute = tokenStream.GetAttribute(); + var offsetAttribute = tokenStream.GetAttribute(); + // Get parsed result, result is Katakana. - string katakana = result.ToString(); + string katakana = charTermAttribute.ToString(); if (string.IsNullOrEmpty(katakana)) break; @@ -61,7 +61,7 @@ private static IEnumerable getProcessingRubyTags(string text, TokenStre if (!config.EnableDuplicatedRuby.Value) { // Not add duplicated ruby if same as parent. - string parentText = text[offsetAtt.StartOffset..offsetAtt.EndOffset]; + string parentText = text[offsetAttribute.StartOffset..offsetAttribute.EndOffset]; if (parentText == katakana || parentText == hiragana) continue; } @@ -70,8 +70,8 @@ private static IEnumerable getProcessingRubyTags(string text, TokenStre yield return new RubyTag { Text = config.RubyAsKatakana.Value ? katakana : hiragana, - StartIndex = offsetAtt.StartOffset, - EndIndex = offsetAtt.EndOffset - 1, + StartIndex = offsetAttribute.StartOffset, + EndIndex = offsetAttribute.EndOffset - 1, }; } From 07eea622d9b42d78118caa75a734f18399ef8e44 Mon Sep 17 00:00:00 2001 From: andy840119 Date: Sun, 16 Jul 2023 21:46:41 +0800 Subject: [PATCH 3/8] Implement the base romaji generator. --- .../Lyrics/Romajies/RomajiGenerateResult.cs | 15 ++++++++++ .../Lyrics/Romajies/RomajiGenerator.cs | 29 +++++++++++++++++++ .../Lyrics/Romajies/RomajiGeneratorConfig.cs | 8 +++++ 3 files changed, 52 insertions(+) create mode 100644 osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGenerateResult.cs create mode 100644 osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGenerator.cs create mode 100644 osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGeneratorConfig.cs diff --git a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGenerateResult.cs b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGenerateResult.cs new file mode 100644 index 000000000..8b7435b0d --- /dev/null +++ b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGenerateResult.cs @@ -0,0 +1,15 @@ +// Copyright (c) andy840119 . Licensed under the GPL Licence. +// See the LICENCE file in the repository root for full licence text. + +using osu.Game.Rulesets.Karaoke.Objects; + +namespace osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies; + +public struct RomajiGenerateResult +{ + public TimeTag TimeTag { get; set; } + + public bool InitialRomaji { get; set; } + + public string? RomajiText { get; set; } +} diff --git a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGenerator.cs b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGenerator.cs new file mode 100644 index 000000000..744fc2d51 --- /dev/null +++ b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGenerator.cs @@ -0,0 +1,29 @@ +// Copyright (c) andy840119 . Licensed under the GPL Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Linq; +using osu.Framework.Graphics.Sprites; +using osu.Framework.Localisation; +using osu.Game.Rulesets.Karaoke.Objects; + +namespace osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies; + +public abstract class RomajiGenerator : LyricPropertyGenerator + where TConfig : RomajiGeneratorConfig, new() +{ + protected RomajiGenerator(TConfig config) + : base(config) + { + } + + protected override LocalisableString? GetInvalidMessageFromItem(Lyric item) + { + if (string.IsNullOrWhiteSpace(item.Text)) + return "Lyric should not be empty."; + + if (item.TimeTags.FirstOrDefault()?.Index != new TextIndex()) + return "Should have at least one index and that index should at the start of the lyric."; + + return null; + } +} diff --git a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGeneratorConfig.cs b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGeneratorConfig.cs new file mode 100644 index 000000000..a0d5ede2a --- /dev/null +++ b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGeneratorConfig.cs @@ -0,0 +1,8 @@ +// Copyright (c) andy840119 . Licensed under the GPL Licence. +// See the LICENCE file in the repository root for full licence text. + +namespace osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies; + +public abstract class RomajiGeneratorConfig : GeneratorConfig +{ +} From 0ab67389f55bebfebddf50263b1436958725340e Mon Sep 17 00:00:00 2001 From: andy840119 Date: Mon, 17 Jul 2023 23:27:00 +0800 Subject: [PATCH 4/8] Implement the helper for able to convert the string to the generate result. --- .../Romajis/RomajiGenerateResultHelper.cs | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/RomajiGenerateResultHelper.cs diff --git a/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/RomajiGenerateResultHelper.cs b/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/RomajiGenerateResultHelper.cs new file mode 100644 index 000000000..d02f4bba7 --- /dev/null +++ b/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/RomajiGenerateResultHelper.cs @@ -0,0 +1,51 @@ +// Copyright (c) andy840119 . Licensed under the GPL Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using System.Linq; +using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies; +using osu.Game.Rulesets.Karaoke.Objects; + +namespace osu.Game.Rulesets.Karaoke.Tests.Editor.Generator.Lyrics.Romajis; + +public class RomajiGenerateResultHelper +{ + /// + /// Convert the string format into the . + /// + /// + /// karaoke + /// ^karaoke + /// + /// Origin time-tag + /// Generate result string format + /// Romaji generate result. + public static RomajiGenerateResult ParseRomajiGenerateResult(TimeTag timeTag, string str) + { + bool initialRomaji = str.StartsWith("^", StringComparison.Ordinal); + + return new RomajiGenerateResult + { + TimeTag = timeTag, + InitialRomaji = initialRomaji, + RomajiText = str.Replace("^", ""), + }; + } + + public static RomajiGenerateResult[] ParseRomajiGenerateResults(IList timeTags, IList strings) + { + if (timeTags.Count != strings.Count) + throw new InvalidOperationException(); + + return parseRomajiGenerateResults(timeTags, strings).ToArray(); + + static IEnumerable parseRomajiGenerateResults(IList timeTags, IList strings) + { + for (int i = 0; i < timeTags.Count; i++) + { + yield return ParseRomajiGenerateResult(timeTags[i], strings[i]); + } + } + } +} From e5f83b20cd385c8c4b7adbb10ca2b08b652c65c0 Mon Sep 17 00:00:00 2001 From: andy840119 Date: Mon, 17 Jul 2023 23:29:02 +0800 Subject: [PATCH 5/8] Implement the base test case for the romaji generator. --- .../Lyrics/Romajis/BaseRomajiGeneratorTest.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/BaseRomajiGeneratorTest.cs diff --git a/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/BaseRomajiGeneratorTest.cs b/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/BaseRomajiGeneratorTest.cs new file mode 100644 index 000000000..cf96bd3a1 --- /dev/null +++ b/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/BaseRomajiGeneratorTest.cs @@ -0,0 +1,27 @@ +// Copyright (c) andy840119 . Licensed under the GPL Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Linq; +using NUnit.Framework; +using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies; +using osu.Game.Rulesets.Karaoke.Objects; +using osu.Game.Rulesets.Karaoke.Tests.Asserts; + +namespace osu.Game.Rulesets.Karaoke.Tests.Editor.Generator.Lyrics.Romajis; + +public abstract class BaseRomajiGeneratorTest : BaseLyricGeneratorTest + where TRomajiGenerator : RomajiGenerator where TConfig : RomajiGeneratorConfig, new() +{ + protected void CheckGenerateResult(Lyric lyric, string[] expectedRubies, TConfig config) + { + var expected = RomajiGenerateResultHelper.ParseRomajiGenerateResults(lyric.TimeTags, expectedRubies); + CheckGenerateResult(lyric, expected, config); + } + + protected override void AssertEqual(RomajiGenerateResult[] expected, RomajiGenerateResult[] actual) + { + TimeTagAssert.ArePropertyEqual(expected.Select(x => x.TimeTag).ToArray(), actual.Select(x => x.TimeTag).ToArray()); + Assert.AreEqual(expected.Select(x => x.InitialRomaji), actual.Select(x => x.InitialRomaji)); + Assert.AreEqual(expected.Select(x => x.RomajiText), actual.Select(x => x.RomajiText)); + } +} From 4936faca56aa69079d4acdb375b10bac294f7f26 Mon Sep 17 00:00:00 2001 From: andy840119 Date: Sun, 16 Jul 2023 21:50:03 +0800 Subject: [PATCH 6/8] Implement the generator for able to generate the ja romaji tag. --- .../Romajis/Ja/JaRomajiGeneratorTest.cs | 111 +++++++++++++ .../Lyrics/Romajies/Ja/JaRomajiGenerator.cs | 149 ++++++++++++++++++ .../Romajies/Ja/JaRomajiGeneratorConfig.cs | 12 ++ 3 files changed, 272 insertions(+) create mode 100644 osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/Ja/JaRomajiGeneratorTest.cs create mode 100644 osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/Ja/JaRomajiGenerator.cs create mode 100644 osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/Ja/JaRomajiGeneratorConfig.cs diff --git a/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/Ja/JaRomajiGeneratorTest.cs b/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/Ja/JaRomajiGeneratorTest.cs new file mode 100644 index 000000000..6214c8895 --- /dev/null +++ b/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/Ja/JaRomajiGeneratorTest.cs @@ -0,0 +1,111 @@ +// Copyright (c) andy840119 . Licensed under the GPL Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies.Ja; +using osu.Game.Rulesets.Karaoke.Objects; +using osu.Game.Rulesets.Karaoke.Tests.Helper; + +namespace osu.Game.Rulesets.Karaoke.Tests.Editor.Generator.Lyrics.Romajis.Ja; + +public class JaRomajiGeneratorTest : BaseRomajiGeneratorTest +{ + [TestCase("花火大会", new[] { "[0,start]", "[3,end]" }, true)] + [TestCase("花火大会", new[] { "[0,start]" }, true)] + [TestCase("花火大会", new[] { "[3,end]" }, false)] // not able to generate the has no start time-tag. + [TestCase("花火大会", new string[] { }, false)] // not able to generate the romaji if has no time-tag. + [TestCase("", new string[] { }, false)] // not able to generate the romaji if lyric is empty. + [TestCase(" ", new string[] { }, false)] + [TestCase(null, new string[] { }, false)] + public void TestCanGenerate(string text, string[] timeTagStrings, bool canGenerate) + { + var config = GeneratorEmptyConfig(); + + var timeTags = TestCaseTagHelper.ParseTimeTags(timeTagStrings); + var lyric = new Lyric + { + Text = text, + TimeTags = timeTags, + }; + + CheckCanGenerate(lyric, canGenerate, config); + } + + // the generated result is not perfect, but it's OK for now. + [TestCase("はなび", new[] { "[0,start]" }, new[] { "^hana bi" })] + [TestCase("花火大会", new[] { "[0,start]", "[3,end]" }, new[] { "^hanabi taikai", "" })] + [TestCase("花火大会", new[] { "[0,start]", "[2,start]", "[3,end]" }, new[] { "^hanabi", "taikai", "" })] + [TestCase("枯れた世界に輝く", + new[] { "[0,start]", "[1,start]", "[2,start]", "[3,start]", "[4,start]", "[5,start]", "[6,start]", "[6,start]", "[6,start]", "[7,start]", "[7,end]" }, + new[] { "^kare", "", "ta", "sekai", "", "ni", "kagayaku", "", "", "", "" })] + public void TestGenerate(string text, string[] timeTagStrings, string[] expectedRomajies) + { + var config = GeneratorEmptyConfig(); + + var timeTags = TestCaseTagHelper.ParseTimeTags(timeTagStrings); + var lyric = new Lyric + { + Text = text, + TimeTags = timeTags, + }; + + CheckGenerateResult(lyric, expectedRomajies, config); + } + + [TestCase("はなび", new[] { "[0,start]" }, new[] { "^HANA BI" })] + [TestCase("花火大会", new[] { "[0,start]", "[2,start]", "[3,end]" }, new[] { "^HANABI", "TAIKAI", "" })] + public void TestGenerateWithUppercase(string text, string[] timeTagStrings, string[] expectedRomajies) + { + var config = GeneratorEmptyConfig(x => x.Uppercase.Value = true); + + var timeTags = TestCaseTagHelper.ParseTimeTags(timeTagStrings); + var lyric = new Lyric + { + Text = text, + TimeTags = timeTags, + }; + + CheckGenerateResult(lyric, expectedRomajies, config); + } + + [TestCase("花", new[] { "[0,start]", "[0,end]" }, new[] { "[0]:hana" }, new[] { "^hana", "" })] + [TestCase("花火", new[] { "[0,start]", "[1,end]" }, new[] { "[0]:hana", "[1]:bi" }, new[] { "^hana bi", "" })] + [TestCase("花火", new[] { "[0,start]", "[1,start]", "[1,end]" }, new[] { "[0]:hana", "[1]:bi" }, new[] { "^hana", "bi", "" })] + [TestCase("花火", new[] { "[0,start]", "[0,start]", "[1,start]", "[1,end]" }, new[] { "[0]:hana", "[1]:bi" }, new[] { "^hana", "", "bi", "" })] + [TestCase("はなび", new[] { "[0,start]", "[1,start]", "[2,start]", "[2,end]" }, new[] { "[0]:hana", "[2]:bi" }, new[] { "^hana", "", "bi", "" })] + public void TestConvertToRomajiGenerateResult(string text, string[] timeTagStrings, string[] romajiParams, string[] expectedResults) + { + var timeTags = TestCaseTagHelper.ParseTimeTags(timeTagStrings); + var romajis = parseRomajiGenerateResults(romajiParams); + + var expected = RomajiGenerateResultHelper.ParseRomajiGenerateResults(timeTags, expectedResults); + var actual = JaRomajiGenerator.Convert(timeTags, romajis).ToArray(); + + AssertEqual(expected, actual); + } + + /// + /// Process test case time tag string format into + /// + /// + /// + /// + /// Time tag string format + /// Time tag object + private static JaRomajiGenerator.RomajiGeneratorParameter parseRomajiGenerateResult(string str) + { + // because format is same as the text-tag testing format, so just use this helper. + var romajiTag = TestCaseTagHelper.ParseRomajiTag(str); + return new JaRomajiGenerator.RomajiGeneratorParameter + { + StartIndex = romajiTag.StartIndex, + EndIndex = romajiTag.EndIndex, + RomajiText = romajiTag.Text, + }; + } + + private static JaRomajiGenerator.RomajiGeneratorParameter[] parseRomajiGenerateResults(IEnumerable strings) + => strings.Select(parseRomajiGenerateResult).ToArray(); +} diff --git a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/Ja/JaRomajiGenerator.cs b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/Ja/JaRomajiGenerator.cs new file mode 100644 index 000000000..abce21b86 --- /dev/null +++ b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/Ja/JaRomajiGenerator.cs @@ -0,0 +1,149 @@ +// Copyright (c) andy840119 . Licensed under the GPL Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Ja; +using Lucene.Net.Analysis.TokenAttributes; +using osu.Framework.Graphics.Sprites; +using osu.Game.Rulesets.Karaoke.Extensions; +using osu.Game.Rulesets.Karaoke.Objects; +using osu.Game.Rulesets.Karaoke.Utils; + +namespace osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies.Ja; + +public class JaRomajiGenerator : RomajiGenerator +{ + private readonly Analyzer analyzer; + + public JaRomajiGenerator(JaRomajiGeneratorConfig config) + : base(config) + { + analyzer = Analyzer.NewAnonymous((fieldName, reader) => + { + Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizerMode.SEARCH); + return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, false)); + }); + } + + protected override RomajiGenerateResult[] GenerateFromItem(Lyric item) + { + // Tokenize the text + string text = item.Text; + var tokenStream = analyzer.GetTokenStream("dummy", new StringReader(text)); + + // get the processing tags. + var processingRomajies = getProcessingRomajies(text, tokenStream, Config).ToArray(); + + // then, trying to mapping them with the time-tags. + return Convert(item.TimeTags, processingRomajies).ToArray(); + } + + private static IEnumerable getProcessingRomajies(string text, TokenStream tokenStream, JaRomajiGeneratorConfig config) + { + // Reset the stream and convert all result + tokenStream.Reset(); + + while (true) + { + // Read next token + tokenStream.ClearAttributes(); + tokenStream.IncrementToken(); + + // Get result and offset + var charTermAttribute = tokenStream.GetAttribute(); + var offsetAttribute = tokenStream.GetAttribute(); + + // Get parsed result, result is Katakana. + string katakana = charTermAttribute.ToString(); + if (string.IsNullOrEmpty(katakana)) + break; + + string parentText = text[offsetAttribute.StartOffset..offsetAttribute.EndOffset]; + bool fromKanji = JpStringUtils.ToKatakana(katakana) != JpStringUtils.ToKatakana(parentText); + + // Convert to romaji. + string romaji = JpStringUtils.ToRomaji(katakana); + if (config.Uppercase.Value) + romaji = romaji.ToUpper(); + + // Make tag + yield return new RomajiGeneratorParameter + { + FromKanji = fromKanji, + StartIndex = offsetAttribute.StartOffset, + EndIndex = offsetAttribute.EndOffset - 1, + RomajiText = romaji, + }; + } + + // Dispose + tokenStream.End(); + tokenStream.Dispose(); + } + + internal static IEnumerable Convert(IList timeTags, IList romajis) + { + var group = createGroup(timeTags, romajis); + return group.Select((x, i) => + { + string romajiText = string.Join(" ", x.Value.Select(r => r.RomajiText)); + + return new RomajiGenerateResult + { + TimeTag = x.Key, + InitialRomaji = i == 0, // todo: use better to mark the initial romaji. + RomajiText = romajiText, + }; + }); + + static IDictionary> createGroup(IList timeTags, IList romajis) + { + var dictionary = timeTags.ToDictionary(x => x, v => new List()); + + int processedIndex = 0; + + foreach (var (timeTag, list) in dictionary) + { + while (processedIndex < romajis.Count && isTimeTagInRange(timeTags, timeTag, romajis[processedIndex])) + { + list.Add(romajis[processedIndex]); + processedIndex++; + } + } + + if (processedIndex < romajis.Count - 1) + throw new InvalidOperationException("Still have romajies that haven't process"); + + return dictionary; + } + + static bool isTimeTagInRange(IEnumerable timeTags, TimeTag currentTimeTag, RomajiGeneratorParameter parameter) + { + if (currentTimeTag.Index.State == TextIndex.IndexState.End) + return false; + + int romajiIndex = parameter.StartIndex; + + var nextTimeTag = timeTags.GetNextMatch(currentTimeTag, x => x.Index > currentTimeTag.Index && x.Index.State == TextIndex.IndexState.Start); + if (nextTimeTag == null) + return romajiIndex >= currentTimeTag.Index.Index; + + return romajiIndex >= currentTimeTag.Index.Index && romajiIndex < nextTimeTag.Index.Index; + } + } + + internal class RomajiGeneratorParameter + { + public bool FromKanji { get; set; } + + public int StartIndex { get; set; } + + public int EndIndex { get; set; } + + public string RomajiText { get; set; } = string.Empty; + } +} diff --git a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/Ja/JaRomajiGeneratorConfig.cs b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/Ja/JaRomajiGeneratorConfig.cs new file mode 100644 index 000000000..5f67b92eb --- /dev/null +++ b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/Ja/JaRomajiGeneratorConfig.cs @@ -0,0 +1,12 @@ +// Copyright (c) andy840119 . Licensed under the GPL Licence. +// See the LICENCE file in the repository root for full licence text. + +using osu.Framework.Bindables; + +namespace osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies.Ja; + +public class JaRomajiGeneratorConfig : RomajiGeneratorConfig +{ + [ConfigSource("Uppercase", "Export romaji with uppercase.")] + public Bindable Uppercase { get; } = new BindableBool(); +} From 5eebc778a2de753d2924e514be8aaaedf2c2e087 Mon Sep 17 00:00:00 2001 From: andy840119 Date: Sun, 23 Jul 2023 15:00:34 +0800 Subject: [PATCH 7/8] Add config for the generator. --- .../KaraokeRulesetEditGeneratorConfigManager.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/osu.Game.Rulesets.Karaoke/Configuration/KaraokeRulesetEditGeneratorConfigManager.cs b/osu.Game.Rulesets.Karaoke/Configuration/KaraokeRulesetEditGeneratorConfigManager.cs index ef23eb1af..ab682af21 100644 --- a/osu.Game.Rulesets.Karaoke/Configuration/KaraokeRulesetEditGeneratorConfigManager.cs +++ b/osu.Game.Rulesets.Karaoke/Configuration/KaraokeRulesetEditGeneratorConfigManager.cs @@ -12,6 +12,7 @@ using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Language; using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Notes; using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.ReferenceLyric; +using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies.Ja; using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.RomajiTags.Ja; using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.RubyTags.Ja; using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.TimeTags.Ja; @@ -54,6 +55,9 @@ protected override void InitialiseDefaults() // Time tag generator SetDefault(); SetDefault(); + + // Romaji generator + SetDefault(); } protected void SetDefault() where T : GeneratorConfig, new() @@ -81,6 +85,7 @@ protected static KaraokeRulesetEditGeneratorSetting GetSettingByType() = Type t when t == typeof(JaRubyTagGeneratorConfig) => KaraokeRulesetEditGeneratorSetting.JaRubyTagGeneratorConfig, Type t when t == typeof(JaTimeTagGeneratorConfig) => KaraokeRulesetEditGeneratorSetting.JaTimeTagGeneratorConfig, Type t when t == typeof(ZhTimeTagGeneratorConfig) => KaraokeRulesetEditGeneratorSetting.ZhTimeTagGeneratorConfig, + Type t when t == typeof(JaRomajiGeneratorConfig) => KaraokeRulesetEditGeneratorSetting.JaRomajiGeneratorConfig, _ => throw new NotSupportedException(), }; @@ -134,4 +139,7 @@ public enum KaraokeRulesetEditGeneratorSetting // Time tag generator JaTimeTagGeneratorConfig, ZhTimeTagGeneratorConfig, + + // Romaji generator. + JaRomajiGeneratorConfig, } From 2468709727d40582c06dad94f503ecdbb42c7d0b Mon Sep 17 00:00:00 2001 From: andy840119 Date: Sun, 16 Jul 2023 21:51:34 +0800 Subject: [PATCH 8/8] Implement the selector for able to select the romaji generator by culture info. --- .../Romajis/RomajiGeneratorSelectorTest.cs | 63 +++++++++++++++++++ .../Romajies/RomajiGeneratorSelector.cs | 18 ++++++ 2 files changed, 81 insertions(+) create mode 100644 osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/RomajiGeneratorSelectorTest.cs create mode 100644 osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGeneratorSelector.cs diff --git a/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/RomajiGeneratorSelectorTest.cs b/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/RomajiGeneratorSelectorTest.cs new file mode 100644 index 000000000..7b1a981ca --- /dev/null +++ b/osu.Game.Rulesets.Karaoke.Tests/Editor/Generator/Lyrics/Romajis/RomajiGeneratorSelectorTest.cs @@ -0,0 +1,63 @@ +// Copyright (c) andy840119 . Licensed under the GPL Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Globalization; +using System.Linq; +using NUnit.Framework; +using osu.Framework.Graphics.Sprites; +using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies; +using osu.Game.Rulesets.Karaoke.Objects; +using osu.Game.Rulesets.Karaoke.Tests.Asserts; +using osu.Game.Rulesets.Karaoke.Tests.Helper; + +namespace osu.Game.Rulesets.Karaoke.Tests.Editor.Generator.Lyrics.Romajis; + +public class RomajiTagGeneratorSelectorTest : BaseLyricGeneratorSelectorTest +{ + [TestCase(17, "花火大会", true)] + [TestCase(17, "我是中文", true)] // only change the language code to decide should be able to generate or not. + [TestCase(17, "", false)] // will not able to generate the romaji if lyric is empty. + [TestCase(17, " ", false)] + [TestCase(17, null, false)] + [TestCase(1028, "はなび", false)] // Should not be able to generate if language is not supported. + public void TestCanGenerate(int lcid, string text, bool canGenerate) + { + var selector = CreateSelector(); + var lyric = new Lyric + { + Language = new CultureInfo(lcid), + Text = text, + TimeTags = new[] + { + new TimeTag(new TextIndex()), + }, + }; + + CheckCanGenerate(lyric, canGenerate, selector); + } + + [TestCase(17, "はなび", new[] { "[0,start]" }, new[] { "^hana bi" })] // Japanese + [TestCase(1041, "花火大会", new[] { "[0,start]", "[3,end]" }, new[] { "^hanabi taikai", "" })] // Japanese + public void TestGenerate(int lcid, string text, string[] timeTagStrings, string[] expectedRomajies) + { + var selector = CreateSelector(); + + var timeTags = TestCaseTagHelper.ParseTimeTags(timeTagStrings); + var lyric = new Lyric + { + Language = new CultureInfo(lcid), + Text = text, + TimeTags = timeTags, + }; + + var expected = RomajiGenerateResultHelper.ParseRomajiGenerateResults(timeTags, expectedRomajies); + CheckGenerateResult(lyric, expected, selector); + } + + protected override void AssertEqual(RomajiGenerateResult[] expected, RomajiGenerateResult[] actual) + { + TimeTagAssert.ArePropertyEqual(expected.Select(x => x.TimeTag).ToArray(), actual.Select(x => x.TimeTag).ToArray()); + Assert.AreEqual(expected.Select(x => x.InitialRomaji), actual.Select(x => x.InitialRomaji)); + Assert.AreEqual(expected.Select(x => x.RomajiText), actual.Select(x => x.RomajiText)); + } +} diff --git a/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGeneratorSelector.cs b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGeneratorSelector.cs new file mode 100644 index 000000000..9215854d3 --- /dev/null +++ b/osu.Game.Rulesets.Karaoke/Edit/Generator/Lyrics/Romajies/RomajiGeneratorSelector.cs @@ -0,0 +1,18 @@ +// Copyright (c) andy840119 . Licensed under the GPL Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Globalization; +using osu.Game.Rulesets.Karaoke.Configuration; +using osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies.Ja; + +namespace osu.Game.Rulesets.Karaoke.Edit.Generator.Lyrics.Romajies; + +public class RomajiGeneratorSelector : LyricGeneratorSelector +{ + public RomajiGeneratorSelector(KaraokeRulesetEditGeneratorConfigManager generatorConfigManager) + : base(generatorConfigManager) + { + RegisterGenerator(new CultureInfo(17)); + RegisterGenerator(new CultureInfo(1041)); + } +}