Skip to content

Commit

Permalink
Added expression control
Browse files Browse the repository at this point in the history
TODO implement project default expression setting
TODO implement Undo & Redo in expression Edit
  • Loading branch information
EX3exp committed Oct 14, 2024
1 parent 5e6ff07 commit 8f7230d
Show file tree
Hide file tree
Showing 16 changed files with 329 additions and 22 deletions.
7 changes: 4 additions & 3 deletions Mirivoice/Assets/Lang/en-US.axaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@
<system:String x:Key="singlelinebox.placeholder">(Line is empty)</system:String>

<system:String x:Key="edits.phoneme">Phonemes</system:String>
<!--
<system:String x:Key="edits.expression">Expressions</system:String>
<system:String x:Key="edits.expression.speed">Speed</system:String>
<system:String x:Key="edits.expression.speedVITS2">Speed</system:String>
<system:String x:Key="edits.expression.noise1VITS2">Noise Scale</system:String>
<system:String x:Key="edits.expression.noise2VITS2">Noise Weight</system:String>
<!--
<system:String x:Key="edits.pitch">Pitch</system:String>
-->
<!-- Menu -->
Expand Down
8 changes: 5 additions & 3 deletions Mirivoice/Assets/Lang/ko-KR.axaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@
<system:String x:Key="singlelinebox.placeholder">(대사가 비어 있어요)</system:String>

<system:String x:Key="edits.phoneme">음소</system:String>
<!--
<system:String x:Key="edits.expression">표현</system:String>
<system:String x:Key="edits.expression.speed">속도</system:String>

<system:String x:Key="edits.expression">표현</system:String>
<system:String x:Key="edits.expression.speedVITS2">속도</system:String>
<system:String x:Key="edits.expression.noise1VITS2">노이즈 크기</system:String>
<system:String x:Key="edits.expression.noise2VITS2">노이즈 영향력</system:String>
<!--
<system:String x:Key="edits.pitch">피치</system:String>
-->

Expand Down
13 changes: 13 additions & 0 deletions Mirivoice/Commands/DelLineBoxReceiver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ public class DelLineBoxReceiver : MReceiver


private SingleLineEditorView lastEditor;

public DelLineBoxReceiver(MainViewModel mainViewModel, LineBoxView l)
{
this.l = l;
Expand All @@ -45,6 +46,12 @@ public override void DoAction()
v.CurrentSingleLineEditor = null;
lastResults = new ObservableCollection<MResult>(l.MResultsCollection);
v.MResultsCollection.Clear();

if (v.CurrentEditIndex == 1)
{
v.CurrentEdit = null;
v.OnPropertyChanged(nameof(v.CurrentEdit));
}
}

RefreshLineNos(control);
Expand All @@ -64,6 +71,12 @@ public override void UndoAction()
l.MResultsCollection = lastResults;
v.MResultsCollection = lastResults;
v.OnPropertyChanged(nameof(v.MResultsCollection));
if (v.CurrentEditIndex == 1)
{
v.CurrentEdit = l.ExpressionEditor;
v.OnPropertyChanged(nameof(v.CurrentEdit));
}

}
}

Expand Down
2 changes: 1 addition & 1 deletion Mirivoice/Engines/TTS/BaseEngine.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ protected string ReadTxtFile(string txtPath)
}
}

public virtual void Inference(string ipaText, string cacheFilePath, int spkid)
public virtual void Inference(string ipaText, string cacheFilePath, int spkid, MExpressionsWrapper expression)
{

}
Expand Down
8 changes: 4 additions & 4 deletions Mirivoice/Engines/TTS/EngineVITS2.cs
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ private Tensor<long> Intersperse(Tensor<long> sequence, long interspersedValue)
return interspersed;
}

public override void Inference(string ipaText, string cacheFilePath, int spkid)
public override void Inference(string ipaText, string cacheFilePath, int spkid, MExpressionsWrapper expression)
{
if (!init)
{
Expand Down Expand Up @@ -160,9 +160,9 @@ public override void Inference(string ipaText, string cacheFilePath, int spkid)

// scales array
Tensor<float> scalesTensor = new DenseTensor<float>(3);
scalesTensor[0] = 0.667f; // noise_scale
scalesTensor[1] = 1.0f; // length_scale. If increased, the speed of the speech will be faster
scalesTensor[2] = 0.8f; // noise_scale_w
scalesTensor[0] = expression.VITS2Noise1; // Noise1 / noise_scale
scalesTensor[1] = expression.VITS2Speed; // Speed / length_scale. If increased, the speed of the speech will be faster
scalesTensor[2] = expression.VITS2Noise2; // Noise2 / noise_scale_w


var inputs = new List<NamedOnnxValue>
Expand Down
32 changes: 32 additions & 0 deletions Mirivoice/Mirivoice.Core/Format/MExpressionsWrapper.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
using Avalonia.Threading;
using Mirivoice.Engines;
using Mirivoice.ViewModels;
using Mirivoice.Views;
using R3;
using Serilog;
using System;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using VYaml.Annotations;
using VYaml.Serialization;
using Mirivoice.Mirivoice.Core.Format;
using Mirivoice.Mirivoice.Plugins.Builtin.Phonemizers;

namespace Mirivoice.Mirivoice.Core.Format
{
[YamlObject]
public partial class MExpressionsWrapper
{
// For VITS2
public float VITS2Speed { get; set; } = 1.0f;
public float VITS2Noise1 { get; set; } = 0.667f;
public float VITS2Noise2 { get; set; } = 0.8f;

public MExpressionsWrapper()
{

}
}
}
4 changes: 3 additions & 1 deletion Mirivoice/Mirivoice.Core/Format/Mrp.cs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public partial class MLinePrototype
public int voicerStyleId { get; set; }
public string IPAText { get; set; }
public string cacheName { get; set; }
public MExpressionsWrapper Exp { get; set; } = new MExpressionsWrapper();

[YamlConstructor]
public MLinePrototype()
Expand All @@ -75,7 +76,7 @@ public MLinePrototype(LineBoxView l)
{
cacheName = l.CurrentCacheName;
}

this.Exp = l.Exp;
}
}

Expand Down Expand Up @@ -127,6 +128,7 @@ public async Task Load(MainViewModel v)
foreach (var m in mLines)
{
m.PhonemeEdit = BasePhonemizer.SetUpProsody(null, m.PhonemeEdit.ToList(), false);
m.Exp = new MExpressionsWrapper();
}
}

Expand Down
3 changes: 2 additions & 1 deletion Mirivoice/Mirivoice.Core/Format/Voicer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ public void Inference(string ipaText, string cacheFilePath, LineBoxView l, int s
Log.Error("Engine is not initialized.");
return;
}
Engine.Inference(ipaText, cacheFilePath, sid);
Log.Debug($"speed: {l.Exp.VITS2Speed}, noise1: {l.Exp.VITS2Noise1}, noise2: {l.Exp.VITS2Noise2}");
Engine.Inference(ipaText, cacheFilePath, sid, l.Exp);
if (l != null)
{
l.IsCacheIsVaild = true;
Expand Down
3 changes: 3 additions & 0 deletions Mirivoice/Mirivoice.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@
<Compile Update="Views\LineBoxView.axaml.cs">
<DependentUpon>LineBoxView.axaml</DependentUpon>
</Compile>
<Compile Update="Views\ExpressionEditViewVITS2.axaml.cs">
<DependentUpon>ExpressionEditViewVITS2.axaml</DependentUpon>
</Compile>
<Compile Update="Views\PhonemeEditView.axaml.cs">
<SubType>Code</SubType>
<DependentUpon>PhonemeEditView.axaml</DependentUpon>
Expand Down
105 changes: 105 additions & 0 deletions Mirivoice/ViewModels/ExpressionEditViewModelVITS2.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
using Avalonia.Controls;
using Avalonia.Input;
using Avalonia.Layout;
using Mirivoice.Engines;
using Mirivoice.Mirivoice.Core.Format;
using Mirivoice.Views;
using ReactiveUI;
using System;

namespace Mirivoice.ViewModels
{
public class ExpressionEditViewModelVITS2: ViewModelBase
{
public StackPanel CurrentExpression { get; set; }
private readonly LineBoxView l;
public int MaxSpeed { get; set; } = 100;
public int MinSpeed { get; set; } = 0;

public int MaxNoise1 { get; set; } = 100;
public int MinNoise1 { get; set; } = 0;

public int MaxNoise2 { get; set; } = 100;
public int MinNoise2 { get; set; } = 0;

private int _vits2Speed;
public int VITS2Speed
{
get => _vits2Speed;
set
{
this.RaiseAndSetIfChanged(ref _vits2Speed, value);
l.Exp.VITS2Speed = ScaleValue(100 - value, 0.5f, 1.5f);
l.IsCacheIsVaild = false;
OnPropertyChanged(nameof(VITS2Speed));
OnPropertyChanged(nameof(l.Exp.VITS2Speed));
}
}

private int _vits2Noise1;
public int VITS2Noise1
{
get => _vits2Noise1;
set
{
this.RaiseAndSetIfChanged(ref _vits2Noise1, value);
l.Exp.VITS2Noise1 = ScaleValue(value, -0.3335f, 1.6675f);
l.IsCacheIsVaild = false;
OnPropertyChanged(nameof(VITS2Noise1));
OnPropertyChanged(nameof(l.Exp.VITS2Noise1));
}
}

private int _vits2Noise2;
public int VITS2Noise2
{
get => _vits2Noise2;
set
{
this.RaiseAndSetIfChanged(ref _vits2Noise2, value);

l.Exp.VITS2Noise2 = ScaleValue(value, -0.4f, 2f);
l.IsCacheIsVaild = false;
OnPropertyChanged(nameof(VITS2Noise2));
OnPropertyChanged(nameof(l.Exp.VITS2Noise2));
}
}
public ExpressionEditViewModelVITS2(LineBoxView l)
{
this.l = l;

VITS2Speed = 100 - GetSliderValue(l.Exp.VITS2Speed, 0.5f, 1.5f);
VITS2Noise1 = GetSliderValue(l.Exp.VITS2Noise1, -0.3335f, 1.6675f);
VITS2Noise2 = GetSliderValue(l.Exp.VITS2Noise2, -0.4f, 2f);

}



public static float ScaleValue(int input, float minValue, float maxValue)
{

return minValue + input / 100f * (maxValue - minValue);

}

public static int GetSliderValue(float value, float minValue, float maxValue)
{
return (int)((value - minValue) / (maxValue - minValue) * 100f);
}
public void ClrVITS2Speed()
{
VITS2Speed = 50;
}

public void ClrVITS2Noise1()
{
VITS2Noise1 = 50;
}

public void ClrVITS2Noise2()
{
VITS2Noise2 = 50;
}
}
}
10 changes: 8 additions & 2 deletions Mirivoice/ViewModels/LineBoxViewModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using Avalonia.Media;
using Avalonia.Media.Imaging;
using Avalonia.Platform;
using Mirivoice.Engines;
using Mirivoice.Mirivoice.Core;
using Mirivoice.Mirivoice.Core.Editor;
using Mirivoice.Mirivoice.Core.Format;
Expand Down Expand Up @@ -141,7 +142,7 @@ public ImageBrush Icon
}

public BasePhonemizer phonemizer;

string lastType;
public override void OnVoicerChanged(Voicer voicer)
{
//Log.Debug($"OnVoicerChanged: {voicer.NickAndStyle}");
Expand All @@ -151,7 +152,12 @@ public override void OnVoicerChanged(Voicer voicer)
{
phonemizer = GetPhonemizer(voicerSelector.CurrentVoicer.Info.LangCode);
}

if (lastType is not null && lastType != voicer.Info.Type)
{
Log.Debug("Type Changed: {0}", voicer.Info.Type);
l.ResetExpEditor(voicer.Info.Type);
}

l.IsCacheIsVaild = false;

LangCode = voicer.Info.LangCode.ToUpper().Substring(0, 2);
Expand Down
25 changes: 24 additions & 1 deletion Mirivoice/ViewModels/MainViewModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,8 @@ public bool StopButtonEnabled
private int _valueProgressbar;

private int _currentEditIndex;


public int CurrentEditIndex
{
get => _currentEditIndex;
Expand All @@ -272,7 +274,28 @@ public int CurrentEditIndex
switch (_currentEditIndex)
{
case 0:
CurrentEdit = new PhonemeEditView();
if (CurrentLineBox is not null)
{
CurrentEdit = new PhonemeEditView();

}
else
{
CurrentEdit = null;
}

this.OnPropertyChanged(nameof(CurrentEdit));
break;
case 1:
if (CurrentLineBox is not null)
{
CurrentEdit = CurrentLineBox.ExpressionEditor;

}
else
{
CurrentEdit = null;
}
this.OnPropertyChanged(nameof(CurrentEdit));
break;
default:
Expand Down
37 changes: 37 additions & 0 deletions Mirivoice/Views/ExpressionEditViewVITS2.axaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:vm="using:Mirivoice.ViewModels"
xmlns:views="clr-namespace:Mirivoice.Views"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
x:Class="Mirivoice.ExpressionEditViewVITS2">

<Border BorderBrush="DarkSlateGray" BorderThickness="1" Background="Gray">
<ScrollViewer AllowAutoHide="False" VerticalScrollBarVisibility="Disabled" HorizontalScrollBarVisibility="Auto">
<StackPanel Background="White">
<StackPanel Orientation="Horizontal">
<TextBlock Text="{DynamicResource edits.expression.speedVITS2} " MaxLines="2" TextWrapping="Wrap" TextElement.FontWeight="Bold" Foreground="DarkSlateGray" TextAlignment="Center" Width="50" FontSize="14" Margin="13 15 15 15" />

<NumericUpDown Value="{Binding VITS2Speed}" Minimum="{Binding MinSpeed}" Maximum="{Binding MaxSpeed}" HorizontalContentAlignment="Center" TextElement.FontSize="15" ParsingNumberStyle="Integer" Increment="1" Margin="13 5 15 5" />
<Slider PointerPressed="ClearVITS2Speed" Value="{Binding VITS2Speed}" Minimum="{Binding MinSpeed}" Width="300" Maximum="{Binding MaxSpeed}" SmallChange="1" LargeChange="10" Margin="13 5 15 5" />

</StackPanel>
<StackPanel Orientation="Horizontal">
<TextBlock Text="{DynamicResource edits.expression.noise1VITS2} " MaxLines="2" TextWrapping="Wrap" TextElement.FontWeight="Bold" Foreground="DarkSlateGray" TextAlignment="Center" Width="50" FontSize="14" Margin="13 15 15 15" />

<NumericUpDown Value="{Binding VITS2Noise1}" Minimum="{Binding MinNoise1}" Maximum="{Binding MaxNoise1}" HorizontalContentAlignment="Center" TextElement.FontSize="15" ParsingNumberStyle="Integer" Increment="1" Margin="13 5 15 5" />
<Slider PointerPressed="ClearVITS2Noise1" Value="{Binding VITS2Noise1}" Minimum="{Binding MinNoise1}" Width="300" Maximum="{Binding MaxNoise1}" SmallChange="1" LargeChange="10" Margin="13 5 15 5" />

</StackPanel>
<StackPanel Orientation="Horizontal">
<TextBlock Text="{DynamicResource edits.expression.noise2VITS2} " MaxLines="2" TextWrapping="Wrap" TextElement.FontWeight="Bold" Foreground="DarkSlateGray" TextAlignment="Center" Width="50" FontSize="14" Margin="13 15 15 15" />

<NumericUpDown Value="{Binding VITS2Noise2}" Minimum="{Binding MinNoise2}" Maximum="{Binding MaxNoise2}" HorizontalContentAlignment="Center" TextElement.FontSize="15" ParsingNumberStyle="Integer" Increment="1" Margin="13 5 15 5" />
<Slider PointerPressed="ClearVITS2Noise2" Value="{Binding VITS2Noise2}" Minimum="{Binding MinNoise2}" Width="300" Maximum="{Binding MaxNoise2}" SmallChange="1" LargeChange="10" Margin="13 5 15 5" />

</StackPanel>
</StackPanel>
</ScrollViewer>
</Border>
</UserControl>
Loading

0 comments on commit 8f7230d

Please sign in to comment.