Skip to content

Commit

Permalink
Treat statistics output as golden data for the test to check against
Browse files Browse the repository at this point in the history
  • Loading branch information
@dsharlet
dsharlet committed Dec 15, 2024
1 parent 2e5b885 commit 1fceed8
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 22 deletions.
37 changes: 20 additions & 17 deletions Tests/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,22 @@ class Program
static async Task<int> Main(string[] args)
{
var rootCommand = new RootCommand().WithCommand("test", "Run tests", c => c
.WithArgument<string>("pattern", "Glob pattern for files to test")
.WithArgument<string>("pattern", "Glob pattern for files to test", new ArgumentArity(1, 100))
.WithOption<bool>(new[] { "--plot" }, "Plot results")
.WithOption<bool>(new[] { "--stats" }, "Write statistics")
.WithOption(new[] { "--samples" }, () => 4800, "Samples")
.WithHandler(CommandHandler.Create<string, bool, bool, int, int, int, int>(Test)))
.WithOption<bool>(new[] { "--updateGolden" }, "Write statistics")
.WithHandler(CommandHandler.Create<string[], bool, int, int, int, int, bool>(Test)))
.WithCommand("benchmark", "Run benchmarks", c => c
.WithArgument<string>("pattern", "Glob pattern for files to benchmark")
.WithHandler(CommandHandler.Create<string, int, int, int>(Benchmark)))
.WithArgument<string>("pattern", "Glob pattern for files to benchmark", new ArgumentArity(1, 100))
.WithHandler(CommandHandler.Create<string[], int, int, int>(Benchmark)))
.WithGlobalOption(new Option<int>("--sampleRate", () => 48000, "Sample Rate"))
.WithGlobalOption(new Option<int>("--oversample", () => 8, "Oversample"))
.WithGlobalOption(new Option<int>("--iterations", () => 8, "Iterations"));

return await rootCommand.InvokeAsync(args);
}

public static void Test(string pattern, bool plot, bool stats, int sampleRate, int samples, int oversample, int iterations)
public static void Test(string[] pattern, bool plot, int sampleRate, int samples, int oversample, int iterations, bool updateGolden)
{
var log = new ConsoleLog() { Verbosity = MessageType.Info };
var tester = new Test();
Expand All @@ -42,14 +42,11 @@ public static void Test(string pattern, bool plot, bool stats, int sampleRate, i
{
tester.PlotAll(circuit.Name, outputs);
}
if (stats)
{
tester.WriteStatistics(circuit.Name, outputs);
}
tester.CheckStatistics(circuit.Name, outputs, updateGolden);
}
}

public static void Benchmark(string pattern, int sampleRate, int oversample, int iterations)
public static void Benchmark(string[] pattern, int sampleRate, int oversample, int iterations)
{
var log = new ConsoleLog() { Verbosity = MessageType.Error };
var tester = new Test();
Expand All @@ -68,13 +65,19 @@ public static void Benchmark(string pattern, int sampleRate, int oversample, int
}
}

private static IEnumerable<Circuit.Circuit> GetCircuits(string glob, ILog log) => Globber.Glob(glob).Select(filename =>
private static IEnumerable<Circuit.Circuit> GetCircuits(string[] globs, ILog log)
{
log.WriteLine(MessageType.Info, filename);
var circuit = Schematic.Load(filename, log).Build();
circuit.Name = Path.GetFileNameWithoutExtension(filename);
return circuit;
});
foreach (string glob in globs)
{
foreach (string filename in Globber.Glob(glob))
{
log.WriteLine(MessageType.Info, filename);
var circuit = Schematic.Load(filename, log).Build();
circuit.Name = Path.GetFileNameWithoutExtension(filename);
yield return circuit;
}
}
}

// Generate a function with the first N harmonics of f0.
private static double Harmonics(double t, double A, double f0, int N)
Expand Down
9 changes: 7 additions & 2 deletions Tests/Properties/launchSettings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,17 @@
},
"Examples": {
"commandName": "Project",
"commandLineArgs": "test Examples\\*.schx --sampleRate 44100 --plot --stats",
"commandLineArgs": "test Examples\\*.schx --sampleRate 44100 --plot",
"workingDirectory": "$(ProjectDir)"
},
"Circuits": {
"commandName": "Project",
"commandLineArgs": "test Circuits\\*.schx --sampleRate 44100 --plot --stats",
"commandLineArgs": "test Circuits\\*.schx --sampleRate 44100 --plot",
"workingDirectory": "$(ProjectDir)"
},
"Update Golden Stats": {
"commandName": "Project",
"commandLineArgs": "test Circuits\\*.schx Examples\\*.schx --sampleRate 44100 --plot --updateGolden",
"workingDirectory": "$(ProjectDir)"
}
}
Expand Down
29 changes: 26 additions & 3 deletions Tests/Test.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,21 @@ private static Dictionary<Expression, double[]> ComputeStatistics(Dictionary<Exp
return stats;
}

public void WriteStatistics(string Title, Dictionary<Expression, List<double>> Outputs)
private string StatsToString(string Title, Dictionary<Expression, List<double>> Outputs)
{
Dictionary<Expression, double[]> stats = ComputeStatistics(Outputs);

string cols = "{0}, {1:G5}, {2:G5}, {3:G5}";

StringBuilder sb = new StringBuilder();
sb.AppendLine(string.Format(cols, "var", "mean", "min", "max"));
foreach (var i in stats)
sb.AppendLine(string.Format(cols, i.Key, i.Value[0], i.Value[1], i.Value[2]));

return sb.ToString();
}

public void CheckStatistics(string Title, Dictionary<Expression, List<double>> Outputs, bool Update)
{
Dictionary<Expression, double[]> stats = ComputeStatistics(Outputs);

Expand All @@ -201,8 +215,17 @@ public void WriteStatistics(string Title, Dictionary<Expression, List<double>> O
sb.AppendLine(string.Format(cols, i.Key, i.Value[0], i.Value[1], i.Value[2]));

string path = "Stats\\" + Title + ".csv";
System.IO.Directory.CreateDirectory("Stats");
File.WriteAllText(path, sb.ToString());
if (Update)
{
System.IO.Directory.CreateDirectory("Stats");
File.WriteAllText(path, sb.ToString());
}
else
{
string golden = File.ReadAllText(path);
if (golden != sb.ToString())
throw new Exception(String.Format("Statistics mismatch in {0}", Title));
}
}
}
}

0 comments on commit 1fceed8

Please sign in to comment.