From 711e06c8ba612220b713c42bc025bb5b7a055eb4 Mon Sep 17 00:00:00 2001 From: towsey Date: Wed, 4 Mar 2020 10:03:50 +1000 Subject: [PATCH 01/12] Update SpectrogramSettings.cs Change default window function from Hamming to Hanning. Also add documentation to some of the class properties. --- .../SpectrogramSettings.cs | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramSettings.cs b/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramSettings.cs index 7d2ce4252..820e0d524 100644 --- a/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramSettings.cs +++ b/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramSettings.cs @@ -5,7 +5,7 @@ namespace AudioAnalysisTools.StandardSpectrograms { using System; - using DSP; + using AudioAnalysisTools.DSP; using TowseyLibrary; public class SpectrogramSettings @@ -16,6 +16,9 @@ public class SpectrogramSettings /// public string SourceFileName { get; set; } + /// + /// Gets or sets the window or frame size. 512 is usually a suitable choice for recordings of the environment. + /// public int WindowSize { get; set; } = 512; public double WindowOverlap { get; set; } = 0.0; @@ -26,8 +29,18 @@ public class SpectrogramSettings /// public int WindowStep { get; set; } = 512; - public string WindowFunction { get; set; } = WindowFunctions.HAMMING.ToString(); + /// + /// Gets or sets the default FFT Window function to the Hanning window. + /// THe Hanning window was made default in March 2020 because it was found to produce better spectrograms + /// in cases where the recording is resampled up or down. + /// + public string WindowFunction { get; set; } = WindowFunctions.HANNING.ToString(); + /// + /// Gets or sets the smoothing window. + /// Following the FFT, each spectrum is smoothed with a moving average filter to reduce its variance. + /// We do the minimum smoothing in order to retain spectral definition. + /// public int SmoothingWindow { get; set; } = 3; public bool DoMelScale { get; set; } = false; @@ -49,6 +62,10 @@ public class SpectrogramAttributes public double MaxAmplitude { get; set; } + /// + /// Gets or sets the maximum frequency that can be represented given the signals sampling rate. + /// The Nyquist is half the SR. + /// public int NyquistFrequency { get; set; } public TimeSpan Duration { get; set; } @@ -56,7 +73,7 @@ public class SpectrogramAttributes public int FrameCount { get; set; } /// - /// Gets or sets duration of full frame or window in seconds + /// Gets or sets duration of full frame or window in seconds. /// public TimeSpan FrameDuration { get; set; } @@ -64,9 +81,15 @@ public class SpectrogramAttributes public double FBinWidth { get; set; } - //this.FBinWidth = this.NyquistFrequency / (double) this.FreqBinCount; + /// + /// Gets or sets the real value difference between two adjacent values of the 16, 24 bit signed integer, + /// used to represent the signal amplitude in the range -1 to +1. + /// public double Epsilon { get; set; } + /// + /// Gets or sets the signal power added by using the chosen FFT window function. + /// public double WindowPower { get; set; } /// From 02947b7745f81c21735c6ea5fe499baf7ec76544 Mon Sep 17 00:00:00 2001 From: towsey Date: Thu, 5 Mar 2020 20:09:37 +1000 Subject: [PATCH 02/12] Write one unit test for overlay of hit scores Issue #300 Write one unit test for overlay of sonogram with matrix of score/hits. Remove duplicated method RecognizerBase.WriteSpectrumIndicesFiles() Rewrite method for drawing of event rectangles. --- .../Recognizers/Base/RecognizerBase.cs | 29 ------- src/AudioAnalysisTools/AcousticEvent.cs | 3 +- .../StandardSpectrograms/Image_MultiTrack.cs | 81 ++++++++++++++++--- .../StandardSpectrograms/SpectrogramTools.cs | 2 - .../StandardSpectrograms/SonogramTests.cs | 42 ++++++++++ 5 files changed, 112 insertions(+), 45 deletions(-) diff --git a/src/AnalysisPrograms/Recognizers/Base/RecognizerBase.cs b/src/AnalysisPrograms/Recognizers/Base/RecognizerBase.cs index 8e068092a..9f2306d53 100644 --- a/src/AnalysisPrograms/Recognizers/Base/RecognizerBase.cs +++ b/src/AnalysisPrograms/Recognizers/Base/RecognizerBase.cs @@ -336,35 +336,6 @@ protected virtual Image DrawSonogram( double eventThreshold) { var image = SpectrogramTools.GetSonogramPlusCharts(sonogram, predictedEvents, scores, hits); - - //const bool doHighlightSubband = false; - //const bool add1KHzLines = true; - //var image = new Image_MultiTrack(sonogram.GetImage(doHighlightSubband, add1KHzLines, doMelScale: false)); - //image.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration, sonogram.FramesPerSecond)); - //image.AddTrack(ImageTrack.GetSegmentationTrack(sonogram)); - - //if (scores != null) - //{ - // foreach (var plot in scores) - // { - // image.AddTrack(ImageTrack.GetNamedScoreTrack(plot.data, 0.0, 1.0, plot.threshold, plot.title)); - // } - //} - - //if (hits != null) - //{ - // image.OverlayRedTransparency(hits); - //} - - //if (predictedEvents != null && predictedEvents.Count > 0) - //{ - // image.AddEvents( - // predictedEvents, - // sonogram.NyquistFrequency, - // sonogram.Configuration.FreqBinCount, - // sonogram.FramesPerSecond); - //} - return image; } diff --git a/src/AudioAnalysisTools/AcousticEvent.cs b/src/AudioAnalysisTools/AcousticEvent.cs index 4b188aa29..e2e726814 100644 --- a/src/AudioAnalysisTools/AcousticEvent.cs +++ b/src/AudioAnalysisTools/AcousticEvent.cs @@ -417,6 +417,7 @@ public void DrawEvent(Image imageToReturn, double framesPerSecond, double if (duration >= 0.0 && framesPerSecond >= 0.0) { t1 = (int)Math.Round(this.TimeStart * framesPerSecond); + t2 = (int)Math.Round(this.TimeEnd * framesPerSecond); } else if (this.Oblong != null) @@ -426,7 +427,7 @@ public void DrawEvent(Image imageToReturn, double framesPerSecond, double t2 = this.Oblong.RowBottom; } - imageToReturn.Mutate(g => g.DrawRectangle(borderPen, t1, y1, t2, y2)); + imageToReturn.Mutate(g => g.DrawRectangle(borderPen, t1, y1, t2 - t1 + 1, y2 - y1 + 1)); if (this.HitElements != null) { diff --git a/src/AudioAnalysisTools/StandardSpectrograms/Image_MultiTrack.cs b/src/AudioAnalysisTools/StandardSpectrograms/Image_MultiTrack.cs index 1e36e0e25..f3d3bfd7e 100644 --- a/src/AudioAnalysisTools/StandardSpectrograms/Image_MultiTrack.cs +++ b/src/AudioAnalysisTools/StandardSpectrograms/Image_MultiTrack.cs @@ -353,33 +353,42 @@ public Image OverlayRedTransparency(Image bmp) } /// - /// superimposes a matrix of scores on top of a sonogram. - /// TODO: WARNING: THIS METHOD IS YET TO BE DEBUGGED SINCE TRANSITION TO SIX-LABOURS, FEB 2020. + /// It is assumed that the spectrogram image is grey scale. + /// NOTE: The score matrix must consist of reals in [0.0, 1.0]. + /// NOTE: The image and the score matrix must have the same number of rows and columns. + /// In case of a spectrogram, it is assumed that the rows are frequency bins and the columns are individual spectra. /// + /// the spectrogram image. + /// the matrix of scores or hits. public static Image OverlayScoresAsRedTransparency(Image bmp, double[,] hits) { Image newBmp = (Image)bmp.Clone(); int rows = hits.GetLength(0); int cols = hits.GetLength(1); - int imageHt = bmp.Height - 1; //subtract 1 because indices start at zero - //traverse columns - skip DC column - for (int c = 1; c < cols; c++) + if (rows != bmp.Height || cols != bmp.Width) { - for (int r = 0; r < rows; r++) + LoggedConsole.WriteErrorLine("ERROR: Image and hits matrix do not have the same dimensions."); + return bmp; + } + + for (int r = 0; r < rows; r++) + { + for (int c = 0; c < cols; c++) { - if (hits[r, c] == 0.0) + if (hits[r, c] <= 0.0) { continue; } - var pixel = bmp[r, imageHt - c]; - if (pixel.R == 255) + if (hits[r, c] > 1.0) { - continue; // white + hits[r, c] = 1.0; } - newBmp[r, imageHt - c] = Color.FromRgb(255, pixel.G, pixel.B); + var pixel = bmp[r, c]; + byte value = (byte)Math.Floor(hits[r, c] * 255); + newBmp[r, c] = Color.FromRgb(value, pixel.G, pixel.B); } } @@ -387,8 +396,54 @@ public static Image OverlayScoresAsRedTransparency(Image bmp, doub } /// - /// superimposes a matrix of scores on top of a sonogram. USES RAINBOW PALLETTE. - /// ASSUME MATRIX NORMALIZED IN [0,1]. + /// Overlays a matrix of scores on an image, typically a spectrogram image. + /// It is assumed that the spectrogram image is grey scale. + /// NOTE: The score matrix must consist of integers from 0 to 255. + /// NOTE: The image and the score matrix must have the same number of rows and columns. + /// In case of a spectrogram, it is assumed that the rows are frequency bins and the columns are individual spectra. + /// + /// the spectrogram image. + /// the matrix of scores or hits. + /// The new image with overlay of scores as red transparency. + public static Image OverlayScoresAsRedTransparency(Image bmp, int[,] hits) + { + Image newBmp = (Image)bmp.Clone(); + int rows = hits.GetLength(0); + int cols = hits.GetLength(1); + + if (rows != bmp.Height || cols != bmp.Width) + { + LoggedConsole.WriteErrorLine("ERROR: Image and hits matrix do not have the same dimensions."); + return bmp; + } + + for (int r = 0; r < rows; r++) + { + for (int c = 0; c < cols; c++) + { + if (hits[r, c] <= 0) + { + continue; + } + + if (hits[r, c] > 255) + { + hits[r, c] = 255; + } + + var pixel = bmp[c, r]; + newBmp[c, r] = Color.FromRgb((byte)hits[r, c], pixel.G, pixel.B); + } + } + + return newBmp; + } + + /// + /// WARNING: THis method is yet to be debugged and tested following move to SixLabors drawing libraries. + /// superimposes a matrix of scores on top of a sonogram. + /// USES A PALLETTE of ten RAINBOW colours. + /// ASSUME MATRIX is NORMALIZED IN [0,1]. /// public void OverlayRainbowTransparency(IImageProcessingContext g, Image bmp) { diff --git a/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramTools.cs b/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramTools.cs index 8d940389a..337b4a351 100644 --- a/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramTools.cs +++ b/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramTools.cs @@ -125,8 +125,6 @@ public static Image GetSonogramPlusCharts( if (hits != null) { spectrogram = Image_MultiTrack.OverlayScoresAsRedTransparency(spectrogram, hits); - //OverlayRedTransparency(bmp, this.SuperimposedRedTransparency); - //this.SonogramImage = this.OverlayRedTransparency((Image)this.SonogramImage); } int pixelWidth = spectrogram.Width; diff --git a/tests/Acoustics.Test/AudioAnalysisTools/StandardSpectrograms/SonogramTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/StandardSpectrograms/SonogramTests.cs index 2986aab4d..9681809b2 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/StandardSpectrograms/SonogramTests.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/StandardSpectrograms/SonogramTests.cs @@ -17,6 +17,7 @@ namespace Acoustics.Test.AudioAnalysisTools.StandardSpectrograms using global::TowseyLibrary; using Microsoft.VisualStudio.TestTools.UnitTesting; using SixLabors.ImageSharp; + using SixLabors.ImageSharp.PixelFormats; using Path = System.IO.Path; /// @@ -219,5 +220,46 @@ public void TestAnnotatedSonogramWithPlots() Assert.AreEqual(1621, image.Width); Assert.AreEqual(656, image.Height); } + + [TestMethod] + public void TestSonogramHitsOverlay() + { + int width = 100; + int height = 50; + + // make a pretend sonogram image + var pretendSonogram = new Image(width, height); + + // make a pretend hits matrix with crossed diagonals + var hitsMatrix = new int[height, width]; + for (int i = 0; i < width; i++) + { + int intensity = (int)Math.Floor(i / (double)width * 255); + hitsMatrix[i / 2, i] = intensity; + hitsMatrix[i / 2, width - i - 1] = intensity; + } + + // now add in hits to the spectrogram image. + if (hitsMatrix != null) + { + pretendSonogram = Image_MultiTrack.OverlayScoresAsRedTransparency(pretendSonogram, hitsMatrix); + } + + //pretendSonogram.Save("C:\\temp\\image.png"); + var pixel = new Argb32(252, 0, 0); + var expectedColor = new Color(pixel); + var actualColor = pretendSonogram[0, height - 1]; + Assert.AreEqual(expectedColor, actualColor); + + pixel = new Argb32(127, 0, 0); + expectedColor = new Color(pixel); + actualColor = pretendSonogram[width / 2, height / 2]; + Assert.AreEqual(expectedColor, actualColor); + + pixel = new Argb32(0, 0, 0); + expectedColor = new Color(pixel); + actualColor = pretendSonogram[0, 0]; + Assert.AreEqual(expectedColor, actualColor); + } } } From b3cf451cd1841ace34e285622c5ab1c3901f72ba Mon Sep 17 00:00:00 2001 From: towsey Date: Sat, 7 Mar 2020 07:58:25 +1000 Subject: [PATCH 03/12] Unit tests for drawing events on sonograms Issue #300 Unit tests for drawing events on sonograms Make unit test for drawing a matrix of scores over a spectrogram. Shift unit test EventStatisticsCalculateTests to a new namespace of tests for acoustic events. Create new class of tests for acoustic events. --- .../EventStatisticsCalculateTests.cs | 11 +- .../AcousticEvents/EventTests.cs | 157 ++++++++++++++++++ .../StandardSpectrograms/SonogramTests.cs | 22 +-- 3 files changed, 174 insertions(+), 16 deletions(-) rename tests/Acoustics.Test/AudioAnalysisTools/{EventStatistics => AcousticEvents}/EventStatisticsCalculateTests.cs (95%) create mode 100644 tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs diff --git a/tests/Acoustics.Test/AudioAnalysisTools/EventStatistics/EventStatisticsCalculateTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventStatisticsCalculateTests.cs similarity index 95% rename from tests/Acoustics.Test/AudioAnalysisTools/EventStatistics/EventStatisticsCalculateTests.cs rename to tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventStatisticsCalculateTests.cs index f43c73762..eb912ab9b 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/EventStatistics/EventStatisticsCalculateTests.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventStatisticsCalculateTests.cs @@ -2,7 +2,7 @@ // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // -namespace Acoustics.Test.AudioAnalysisTools.EventStatistics +namespace Acoustics.Test.AudioAnalysisTools.AcousticEvents { using System; using Acoustics.Shared; @@ -16,11 +16,10 @@ namespace Acoustics.Test.AudioAnalysisTools.EventStatistics [TestClass] public class EventStatisticsCalculateTests { - [TestMethod] public void TestCalculateEventStatistics() { - int sampleRate = 22050; + var sampleRate = 22050; double duration = 28; int[] harmonics1 = { 500 }; int[] harmonics2 = { 500, 1000, 2000, 4000, 8000 }; @@ -37,8 +36,8 @@ public void TestCalculateEventStatistics() var start = TimeSpan.FromSeconds(28) + segmentOffset; var end = TimeSpan.FromSeconds(32) + segmentOffset; - double lowFreq = 1500.0; - double topFreq = 8500.0; + var lowFreq = 1500.0; + var topFreq = 8500.0; var statsConfig = new EventStatisticsConfiguration() { @@ -46,7 +45,7 @@ public void TestCalculateEventStatistics() FrameStep = 512, }; - EventStatistics stats = + var stats = EventStatisticsCalculate.AnalyzeAudioEvent( recording, (start, end).AsRange(), diff --git a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs new file mode 100644 index 000000000..18cd37773 --- /dev/null +++ b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs @@ -0,0 +1,157 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace Acoustics.Test.AudioAnalysisTools.EventStatistics +{ + using System; + using System.Collections.Generic; + using Acoustics.Shared; + using Acoustics.Tools.Wav; + using global::AudioAnalysisTools; + using global::AudioAnalysisTools.DSP; + using global::AudioAnalysisTools.EventStatistics; + using global::AudioAnalysisTools.WavTools; + using global::TowseyLibrary; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using SixLabors.ImageSharp; + using SixLabors.ImageSharp.PixelFormats; + + [TestClass] + public class EventTests + { + [TestMethod] + public void TestEventMerging() + { + // make a list of events + var events = new List(); + + var segmentStartOffset = TimeSpan.Zero; + double maxPossibleScore = 10.0; + var startTime1 = 1.0; + var duration1 = 5.0; + var minHz1 = 1000; + var maxHz1 = 8000; + var event1 = new AcousticEvent(segmentStartOffset, startTime1, duration1, minHz1, maxHz1) + { + Name = "Event1", + Score = 1.0, + ScoreNormalised = 1.0 / maxPossibleScore, + }; + + events.Add(event1); + + var startTime2 = 4.5; + var duration2 = 2.0; + var minHz2 = 1500; + var maxHz2 = 6000; + var event2 = new AcousticEvent(segmentStartOffset, startTime2, duration2, minHz2, maxHz2) + { + Name = "Event2", + Score = 5.0, + ScoreNormalised = 5.0 / maxPossibleScore, + }; + events.Add(event2); + + var startTime3 = 7.0; + var duration3 = 2.0; + var minHz3 = 1000; + var maxHz3 = 8000; + var event3 = new AcousticEvent(segmentStartOffset, startTime3, duration3, minHz3, maxHz3) + { + Name = "Event3", + Score = 9.0, + ScoreNormalised = 9.0 / maxPossibleScore, + }; + events.Add(event3); + + // combine adjacent acoustic events + events = AcousticEvent.CombineOverlappingEvents(events, segmentStartOffset); + + Assert.AreEqual(2, events.Count); + Assert.AreEqual(1.0, events[0].EventStartSeconds, 1E-4); + Assert.AreEqual(6.5, events[0].EventEndSeconds, 1E-4); + Assert.AreEqual(7.0, events[1].EventStartSeconds, 1E-4); + Assert.AreEqual(9.0, events[1].EventEndSeconds, 1E-4); + + Assert.AreEqual(1000, events[0].LowFrequencyHertz); + Assert.AreEqual(8000, events[0].HighFrequencyHertz); + Assert.AreEqual(1000, events[1].LowFrequencyHertz); + Assert.AreEqual(8000, events[1].HighFrequencyHertz); + + Assert.AreEqual(5.0, events[0].Score, 1E-4); + Assert.AreEqual(9.0, events[1].Score, 1E-4); + Assert.AreEqual(0.5, events[0].ScoreNormalised, 1E-4); + Assert.AreEqual(0.9, events[1].ScoreNormalised, 1E-4); + } + + [TestMethod] + public void TestSonogramWithEventsOverlay() + { + // make a substitute sonogram image + int width = 100; + int height = 256; + var substituteSonogram = new Image(width, height); + + //substituteSonogram.Mutate(x => x.Pad(width, height, Color.Gray)); + // image.Mutate(x => x.Resize(image.Width / 2, image.Height / 2)); + + // make a list of events + var framesPerSecond = 10.0; + var freqBinWidth = 43.0664; + var segmentStartOffset = TimeSpan.Zero; + var minHz = 1000; + var maxHz = 8000; + double maxPossibleScore = 10.0; + + var events = new List(); + var startTime1 = 1.0; + var duration1 = 5.0; + var event1 = new AcousticEvent(segmentStartOffset, startTime1, duration1, minHz, maxHz) + { + Score = 10.0, + Name = "Event1", + ScoreNormalised = 10.0 / maxPossibleScore, + }; + + events.Add(event1); + var startTime2 = 7.0; + var duration2 = 2.0; + var event2 = new AcousticEvent(segmentStartOffset, startTime2, duration2, minHz, maxHz) + { + Score = 1.0, + Name = "Event2", + ScoreNormalised = 1.0 / maxPossibleScore, + }; + events.Add(event2); + + // now add events into the spectrogram image. + // set colour for the events + foreach (AcousticEvent ev in events) + { + ev.BorderColour = AcousticEvent.DefaultBorderColor; + ev.ScoreColour = AcousticEvent.DefaultScoreColor; + ev.DrawEvent(substituteSonogram, framesPerSecond, freqBinWidth, height); + } + + //substituteSonogram.Save("C:\\temp\\image.png"); + var redPixel1 = new Argb32(110, 10, 30); + var expectedRed1 = new Color(redPixel1); + var redPixel2 = new Argb32(124, 11, 34); + var expectedRed2 = new Color(redPixel2); + var greenPixel = new Argb32(55, 133, 15); + var expectedGreen = new Color(greenPixel); + + //var actualColor = substituteSonogram[0, height - 1]; + Assert.AreEqual(expectedRed1, substituteSonogram[61, 119]); + Assert.AreEqual(expectedRed1, substituteSonogram[70, 122]); + Assert.AreEqual(expectedRed1, substituteSonogram[91, 181]); + Assert.AreEqual(expectedRed2, substituteSonogram[36, 233]); + Assert.AreEqual(expectedRed1, substituteSonogram[56, 69]); + + //actualColor = substituteSonogram[9, 72]; + Assert.AreEqual(expectedGreen, substituteSonogram[9, 72]); + Assert.AreEqual(expectedGreen, substituteSonogram[69, 217]); + } + } +} diff --git a/tests/Acoustics.Test/AudioAnalysisTools/StandardSpectrograms/SonogramTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/StandardSpectrograms/SonogramTests.cs index 9681809b2..b35ee9f63 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/StandardSpectrograms/SonogramTests.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/StandardSpectrograms/SonogramTests.cs @@ -18,6 +18,7 @@ namespace Acoustics.Test.AudioAnalysisTools.StandardSpectrograms using Microsoft.VisualStudio.TestTools.UnitTesting; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; + using SixLabors.ImageSharp.Processing; using Path = System.IO.Path; /// @@ -91,7 +92,7 @@ public void Cleanup() /// /// METHOD TO CHECK that averaging of decibel values is working. /// var array = new[] { 96.0, 100.0, 90.0, 97.0 }; - /// The return value should = 96.98816759 dB + /// The return value should = 96.98816759 dB. /// [TestMethod] public void TestAverageOfDecibelValues() @@ -225,18 +226,19 @@ public void TestAnnotatedSonogramWithPlots() public void TestSonogramHitsOverlay() { int width = 100; - int height = 50; + int height = 256; - // make a pretend sonogram image + // make a substitute sonogram image var pretendSonogram = new Image(width, height); - // make a pretend hits matrix with crossed diagonals + // make a hits matrix with crossed diagonals var hitsMatrix = new int[height, width]; - for (int i = 0; i < width; i++) + for (int i = 0; i < height; i++) { - int intensity = (int)Math.Floor(i / (double)width * 255); - hitsMatrix[i / 2, i] = intensity; - hitsMatrix[i / 2, width - i - 1] = intensity; + int col = (int)Math.Floor(width * i / (double)height); + int intensity = col; + hitsMatrix[i, col] = i; + hitsMatrix[i, width - col - 1] = i; } // now add in hits to the spectrogram image. @@ -246,12 +248,12 @@ public void TestSonogramHitsOverlay() } //pretendSonogram.Save("C:\\temp\\image.png"); - var pixel = new Argb32(252, 0, 0); + var pixel = new Argb32(255, 0, 0); var expectedColor = new Color(pixel); var actualColor = pretendSonogram[0, height - 1]; Assert.AreEqual(expectedColor, actualColor); - pixel = new Argb32(127, 0, 0); + pixel = new Argb32(128, 0, 0); expectedColor = new Color(pixel); actualColor = pretendSonogram[width / 2, height / 2]; Assert.AreEqual(expectedColor, actualColor); From 3d1b26349bc8f7314ccf950d700707355c2ca1c8 Mon Sep 17 00:00:00 2001 From: towsey Date: Sat, 7 Mar 2020 09:52:13 +1000 Subject: [PATCH 04/12] Fixing bugs in merging of acoustic events Issue #300 The main difficulty here is confusing terminology of temporal fileds and properties. Changed some of these to be more explicit. There appears to remain one more bug - the number of acoustic events in the ae.csv file does not match number that appears in the spectrogram images?? --- src/AnalysisBase/ResultBases/EventBase.cs | 17 +- .../Recognizers/Base/WhistleParameters.cs | 37 +--- .../Recognizers/GenericRecognizer.cs | 2 +- src/AudioAnalysisTools/AcousticEvent.cs | 191 +++++++++--------- 4 files changed, 114 insertions(+), 133 deletions(-) diff --git a/src/AnalysisBase/ResultBases/EventBase.cs b/src/AnalysisBase/ResultBases/EventBase.cs index ea37af352..42f4fd6d2 100644 --- a/src/AnalysisBase/ResultBases/EventBase.cs +++ b/src/AnalysisBase/ResultBases/EventBase.cs @@ -1,4 +1,4 @@ -// -------------------------------------------------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // @@ -12,14 +12,14 @@ namespace AnalysisBase.ResultBases using System; /// - /// The base class for all Event style results + /// The base class for all Event style results. /// public abstract class EventBase : ResultBase { private double eventStartSeconds; /// - /// Gets or sets the time the current audio segment is offset from the start of the file/recording. + /// Gets or sets the time (in seconds) from start of the file/recording to start of the current audio segment. /// /// /// will always be greater than or equal to . @@ -36,7 +36,7 @@ public abstract class EventBase : ResultBase /// /// /// 2017-09: This field USED to be offset relative to the current segment. - /// 2017-09: This field is NOW equivalent to + /// 2017-09: This field is NOW equivalent to . /// public virtual double EventStartSeconds { @@ -60,10 +60,13 @@ public virtual double EventStartSeconds /// public virtual double? LowFrequencyHertz { get; protected set; } - protected void SetEventStartRelative(TimeSpan segmentStart, double eventStartSegmentRelative) + /// + /// Sets both the Segment start and the Event start wrt to recording. + /// + protected void SetSegmentAndEventStartsWrtRecording(TimeSpan segmentStartWrtRecording, double eventStartWrtSegment) { - this.SegmentStartSeconds = segmentStart.TotalSeconds; - this.EventStartSeconds = this.SegmentStartSeconds + eventStartSegmentRelative; + this.SegmentStartSeconds = segmentStartWrtRecording.TotalSeconds; + this.EventStartSeconds = this.SegmentStartSeconds + eventStartWrtSegment; } } } diff --git a/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs b/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs index 071b49772..f1007d972 100644 --- a/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs +++ b/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs @@ -30,7 +30,7 @@ public static (List, double[]) GetWhistles( double decibelThreshold, double minDuration, double maxDuration, - TimeSpan segmentStartOffset) + TimeSpan segmentStartWrtRecording) { var sonogramData = sonogram.Data; int frameCount = sonogramData.GetLength(0); @@ -39,10 +39,6 @@ public static (List, double[]) GetWhistles( double binWidth = nyquist / (double)binCount; int minBin = (int)Math.Round(minHz / binWidth); int maxBin = (int)Math.Round(maxHz / binWidth); - //int binCountInBand = maxBin - minBin + 1; - - // buffer zone around whistle is four bins wide. - int N = 4; // list of accumulated acoustic events var events = new List(); @@ -54,7 +50,8 @@ public static (List, double[]) GetWhistles( // set up an intensity array for the frequency bin. double[] intensity = new double[frameCount]; - if (minBin < N) + // buffer zone around whistle is four bins wide. + if (minBin < 4) { // for all time frames in this frequency bin for (int t = 0; t < frameCount; t++) @@ -96,7 +93,7 @@ public static (List, double[]) GetWhistles( decibelThreshold, minDuration, maxDuration, - segmentStartOffset); + segmentStartWrtRecording); // add to conbined intensity array for (int t = 0; t < frameCount; t++) @@ -110,33 +107,9 @@ public static (List, double[]) GetWhistles( } //end for all freq bins // combine adjacent acoustic events - events = AcousticEvent.CombineOverlappingEvents(events); + events = AcousticEvent.CombineOverlappingEvents(events, segmentStartWrtRecording); return (events, combinedIntensityArray); } - - /* - /// - /// Calculates the average intensity in a freq band having min and max freq, - /// AND then subtracts average intensity in the side/buffer bands, below and above. - /// THis method adds dB log values incorrectly but it is faster than doing many log conversions. - /// This method is used to find acoustic events and is accurate enough for the purpose. - /// - public static double[] CalculateFreqBandAvIntensityMinusBufferIntensity(double[,] sonogramData, int minHz, int maxHz, int nyquist) - { - var bandIntensity = SNR.CalculateFreqBandAvIntensity(sonogramData, minHz, maxHz, nyquist); - var bottomSideBandIntensity = SNR.CalculateFreqBandAvIntensity(sonogramData, minHz - bottomHzBuffer, minHz, nyquist); - var topSideBandIntensity = SNR.CalculateFreqBandAvIntensity(sonogramData, maxHz, maxHz + topHzBuffer, nyquist); - - int frameCount = sonogramData.GetLength(0); - double[] netIntensity = new double[frameCount]; - for (int i = 0; i < frameCount; i++) - { - netIntensity[i] = bandIntensity[i] - bottomSideBandIntensity[i] - topSideBandIntensity[i]; - } - - return netIntensity; - } - */ } } diff --git a/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs b/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs index 76c451136..d5c77f897 100644 --- a/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs +++ b/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs @@ -334,7 +334,7 @@ static void SaveDebugSpectrogram(RecognizerResults results, Config genericConfig var image3 = SpectrogramTools.GetSonogramPlusCharts(results.Sonogram, results.Events, results.Plots, null); //image3.Save(Path.Combine(outputDirectory.FullName, baseName + ".profile.png")); - image3.Save(Path.Combine("C:\\temp", baseName + ".profile.png")); + //image3.Save(Path.Combine("C:\\temp", baseName + ".profile.png")); //sonogram.GetImageFullyAnnotated("test").Save("C:\\temp\\test.png"); } diff --git a/src/AudioAnalysisTools/AcousticEvent.cs b/src/AudioAnalysisTools/AcousticEvent.cs index e2e726814..f13354e13 100644 --- a/src/AudioAnalysisTools/AcousticEvent.cs +++ b/src/AudioAnalysisTools/AcousticEvent.cs @@ -83,30 +83,39 @@ public AcousticEventClassMap() public double TimeStart { get; private set; } /// - /// Gets or sets units = seconds - /// Time offset from start of current segment to end of the event. + /// Gets the time offset (in seconds) from start of current segment to end of the event. /// Written into the csv file under column "EventEndSeconds" /// This field is NOT in EventBase. EventBase only requires TimeStart because it is designed to also accomodate points. /// /// - /// Note: converted to private setter so we can control how this is set. Recommend using - /// after event instantiation to modify bounds. + /// Note: converted to private setter so we can control how this is set. + /// Recommend using after event instantiation to modify bounds. /// public double TimeEnd { get; private set; } + /// + /// Gets the end time of an event WRT the recording/file start. + /// public double EventEndSeconds => this.TimeEnd + this.SegmentStartSeconds; + /// + /// Gets the start time of an event WRT the recording/file start. + /// public override double EventStartSeconds => this.TimeStart + this.SegmentStartSeconds; + /// + /// Set the start and end times of an event with respect to the segment start time + /// AND also calls method to set event start time with respect the recording/file start. + /// public void SetEventPositionRelative( - TimeSpan segmentStartOffset, - double eventStartSegment, - double eventEndSegment) + TimeSpan segmentStartWrtRecording, + double eventStartWrtSegment, + double eventEndWrtSegment) { - this.TimeStart = eventStartSegment; - this.TimeEnd = eventEndSegment; + this.TimeStart = eventStartWrtSegment; + this.TimeEnd = eventEndWrtSegment; - this.SetEventStartRelative(segmentStartOffset, eventStartSegment); + this.SetSegmentAndEventStartsWrtRecording(segmentStartWrtRecording, eventStartWrtSegment); } /// @@ -126,30 +135,39 @@ public void SetEventPositionRelative( } } - /// Gets or sets units = Hertz + /// Gets or sets units = Hertz. public double HighFrequencyHertz { get; set; } + /// + /// Gets the bandwidth of an acoustic event. + /// public double Bandwidth => this.HighFrequencyHertz - this.LowFrequencyHertz + 1; public bool IsMelscale { get; set; } + /// + /// Gets or sets the bounds of an event with respect to the segment start + /// BUT in terms of the frame count (from segment start) and frequency bin (from zero Hertz). + /// This is no longer the preferred way to operate with acoustic event bounds. + /// Better to use real units (seconds and Hertz) and provide the acoustic event with scale information. + /// public Oblong Oblong { get; set; } - /// Gets or sets required for conversions to & from MEL scale AND for drawing event on spectrum + /// Gets or sets required for conversions to & from MEL scale AND for drawing event on spectrum. public int FreqBinCount { get; set; } /// - /// Gets required for freq-binID conversions + /// Gets required for freq-binID conversions. /// public double FreqBinWidth { get; private set; } - /// Gets frame duration in seconds + /// Gets frame duration in seconds. public double FrameDuration { get; private set; } - /// Gets or sets time between frame starts in seconds. Inverse of FramesPerSecond + /// Gets or sets time between frame starts in seconds. Inverse of FramesPerSecond. public double FrameOffset { get; set; } - /// Gets or sets number of frame starts per second. Inverse of the frame offset + /// Gets or sets number of frame starts per second. Inverse of the frame offset. public double FramesPerSecond { get; set; } //PROPERTIES OF THE EVENTS i.e. Name, SCORE ETC @@ -162,7 +180,7 @@ public void SetEventPositionRelative( /// Gets or sets average score through the event. public string ScoreComment { get; set; } - /// Gets or sets score normalised in range [0,1]. NOTE: Max is set = to five times user supplied threshold + /// Gets or sets score normalised in range [0,1]. NOTE: Max is set = to five times user supplied threshold. public double ScoreNormalised { get; set; } /// Gets max Possible Score: set = to 5x user supplied threshold. An arbitrary value used for score normalisation. @@ -174,8 +192,10 @@ public void SetEventPositionRelative( public string Score2Name { get; set; } - /// Gets or sets second score if required - public double Score2 { get; set; } // e.g. for Birgits recognisers + /// Gets or sets second score if required. + /// Was used for Birgits recognisers but should now be used only for debug purposes. + /// + public double Score2 { get; set; } /// /// Gets or sets a list of points that can be used to identifies features in spectrogram relative to the Event. @@ -184,7 +204,11 @@ public void SetEventPositionRelative( /// public List Points { get; set; } - public double Periodicity { get; set; } // for events which have an oscillating acoustic energy - used for frog calls + /// + /// Gets or sets the periodicity of acoustic energy in an event. + /// Use for events which have an oscillating acoustic energy - e.g. for frog calls. + /// + public double Periodicity { get; set; } public double DominantFreq { get; set; } // the dominant freq in the event - used for frog calls @@ -192,8 +216,8 @@ public void SetEventPositionRelative( // double I1Var; //, // double I2MeandB; // mean intensity of pixels in the event after Wiener filter, prior to noise subtraction // double I2Var; //, - private double I3Mean; // mean intensity of pixels in the event AFTER noise reduciton - USED FOR CLUSTERING - private double I3Var; // variance of intensity of pixels in the event. + private double i3Mean; // mean intensity of pixels in the event AFTER noise reduciton - USED FOR CLUSTERING + private double i3Var; // variance of intensity of pixels in the event. // following are no longer needed. Delete eventually. /* @@ -213,10 +237,10 @@ public void SetEventPositionRelative( /// Gets or sets a value indicating whether use this if want to filter or tag some members of a list for some purpose. public bool Tag { get; set; } - /// Gets or sets assigned value when reading in a list of user identified events. Indicates a user assigned assessment of event intensity + /// Gets or sets assigned value when reading in a list of user identified events. Indicates a user assigned assessment of event intensity. public int Intensity { get; set; } - /// Gets or sets assigned value when reading in a list of user identified events. Indicates a user assigned assessment of event quality + /// Gets or sets assigned value when reading in a list of user identified events. Indicates a user assigned assessment of event quality. public int Quality { get; set; } public Color BorderColour { get; set; } @@ -231,11 +255,20 @@ public AcousticEvent() this.IsMelscale = false; } - public AcousticEvent(TimeSpan segmentStartOffset, double startTime, double eventDuration, double minFreq, double maxFreq) + /// + /// Initializes a new instance of the class. + /// This constructor requires the minimum information to establish the temporal and frequency bounds of an acoustic event. + /// + /// The start of the current segment with respect to start of recording/file. + /// event start with respect to start of segment. + /// event end with respect to start of segment. + /// Lower frequency bound of event. + /// Upper frequency bound of event. + public AcousticEvent(TimeSpan segmentStartWrtRecording, double startTimeWrtSegment, double eventDuration, double minFreq, double maxFreq) : this() { - this.SetEventPositionRelative(segmentStartOffset, startTime, startTime + eventDuration); - + var endTimeWrtSegment = startTimeWrtSegment + eventDuration; + this.SetEventPositionRelative(segmentStartWrtRecording, startTimeWrtSegment, endTimeWrtSegment); this.LowFrequencyHertz = minFreq; this.HighFrequencyHertz = maxFreq; @@ -246,6 +279,8 @@ public AcousticEvent(TimeSpan segmentStartOffset, double startTime, double event /// /// Initializes a new instance of the class. /// This constructor currently works ONLY for linear Hertz scale events. + /// It requires the event bounds to provided (using Oblong) in terms of time frame and frequency bin counts. + /// Scale information must also be provided to convert bounds into real values (seconds, Hertz). /// /// An oblong initialized with bin and frame numbers marking location of the event. /// to set the freq scale. @@ -503,12 +538,6 @@ public bool Overlaps(AcousticEvent ae) /// public static double EventFractionalOverlap(AcousticEvent event1, AcousticEvent event2) { - //if (event1.EndTime < event2.StartTime) return 0.0; - //if (event2.EndTime < event1.StartTime) return 0.0; - //if (event1.MaxFreq < event2.MinFreq) return 0.0; - //if (event2.MaxFreq < event1.MinFreq) return 0.0; - //at this point the two events do overlap - int timeOverlap = Oblong.RowOverlap(event1.Oblong, event2.Oblong); if (timeOverlap == 0) { @@ -558,6 +587,19 @@ public static bool EventsOverlap(AcousticEvent event1, AcousticEvent event2) timeOverlap = true; } + // now check possibility that event2 is inside event1. + //check if event 2 starts within event 1 + if (event2.EventStartSeconds >= event1.EventStartSeconds && event2.EventStartSeconds <= event1.EventEndSeconds) + { + timeOverlap = true; + } + + // check if event 2 ends within event 1 + if (event2.EventEndSeconds >= event1.EventStartSeconds && event2.EventEndSeconds <= event1.EventEndSeconds) + { + timeOverlap = true; + } + //check if event 1 freq band overlaps event 2 freq band if (event1.HighFrequencyHertz >= event1.LowFrequencyHertz && event1.HighFrequencyHertz <= event2.HighFrequencyHertz) { @@ -579,7 +621,7 @@ public static bool EventsOverlap(AcousticEvent event1, AcousticEvent event2) /// Freq dimension = bins = matrix columns. Origin is top left - as per matrix in the sonogram class. /// Time dimension = frames = matrix rows. /// - public static List CombineOverlappingEvents(List events) + public static List CombineOverlappingEvents(List events, TimeSpan segmentStartWrtRecording) { if (events.Count < 2) { @@ -592,7 +634,7 @@ public static List CombineOverlappingEvents(List e { if (EventsOverlap(events[i], events[j])) { - events[j] = AcousticEvent.MergeTwoEvents(events[i], events[j]); + events[j] = AcousticEvent.MergeTwoEvents(events[i], events[j], segmentStartWrtRecording); events.RemoveAt(i); break; } @@ -602,14 +644,16 @@ public static List CombineOverlappingEvents(List e return events; } - public static AcousticEvent MergeTwoEvents(AcousticEvent e1, AcousticEvent e2) + public static AcousticEvent MergeTwoEvents(AcousticEvent e1, AcousticEvent e2, TimeSpan segmentStartWrtRecording) { - //e1.EventEndSeconds = Math.Max(e1.EventEndSeconds, e2.EventEndSeconds); - e1.EventStartSeconds = Math.Min(e1.EventStartSeconds, e2.EventStartSeconds); + //segmentStartOffset = TimeSpan.Zero; + var minTime = Math.Min(e1.TimeStart, e2.TimeStart); + var maxTime = Math.Max(e1.TimeEnd, e2.TimeEnd); + e1.SetEventPositionRelative(segmentStartWrtRecording, minTime, maxTime); e1.LowFrequencyHertz = Math.Min(e1.LowFrequencyHertz, e2.LowFrequencyHertz); e1.HighFrequencyHertz = Math.Max(e1.HighFrequencyHertz, e2.HighFrequencyHertz); - - //e1.ResultMinute = (int)e1.ResultStartSeconds.Floor(); + e1.Score = Math.Max(e1.Score, e2.Score); + e1.ScoreNormalised = Math.Max(e1.ScoreNormalised, e2.ScoreNormalised); e1.ResultStartSeconds = e1.EventStartSeconds; return e1; } @@ -667,8 +711,8 @@ public static void Time2RowIDs(double startTime, double duration, double frameOf public void SetNetIntensityAfterNoiseReduction(double mean, double var) { - this.I3Mean = mean; - this.I3Var = var; + this.i3Mean = mean; + this.i3Var = var; } /// @@ -1248,9 +1292,9 @@ public static List GetEventsAroundMaxima( /// The method uses the passed scoreThreshold in order to calculate a normalised score. /// Max possible score := threshold * 5. /// normalised score := score / maxPossibleScore. - /// Some analysis techniques (e.g. OD) have their own methods for extracting events from score arrays. + /// Some analysis techniques (e.g. Oscillation Detection) have their own methods for extracting events from score arrays. /// - /// the array of scores + /// the array of scores. /// lower freq bound of the acoustic event. /// upper freq bound of the acoustic event. /// the time scale required by AcousticEvent class. @@ -1258,7 +1302,7 @@ public static List GetEventsAroundMaxima( /// threshold. /// duration of event must exceed this to count as an event. /// duration of event must be less than this to count as an event. - /// offset. + /// offset. /// a list of acoustic events. public static List ConvertScoreArray2Events( double[] scores, @@ -1269,14 +1313,14 @@ public static List ConvertScoreArray2Events( double scoreThreshold, double minDuration, double maxDuration, - TimeSpan segmentStartOffset) + TimeSpan segmentStartWrtRecording) { int count = scores.Length; var events = new List(); double maxPossibleScore = 5 * scoreThreshold; // used to calculate a normalised score between 0 - 1.0 bool isHit = false; double frameOffset = 1 / framesPerSec; // frame offset in fractions of second - double startTime = 0.0; + double startTimeWrtSegment = 0.0; // units = seconds int startFrame = 0; // pass over all frames @@ -1286,7 +1330,7 @@ public static List ConvertScoreArray2Events( { //start of an event isHit = true; - startTime = i * frameOffset; + startTimeWrtSegment = i * frameOffset; startFrame = i; } else // check for the end of an event @@ -1295,7 +1339,7 @@ public static List ConvertScoreArray2Events( // this is end of an event, so initialise it isHit = false; double endTime = i * frameOffset; - double duration = endTime - startTime; + double duration = endTime - startTimeWrtSegment; // if (duration < minDuration) continue; //skip events with duration shorter than threshold if (duration < minDuration || duration > maxDuration) @@ -1312,14 +1356,8 @@ public static List ConvertScoreArray2Events( av /= i - startFrame + 1; - //NOTE av cannot be < threhsold because event started and ended based on threhsold. - // Therefore remove the following condition on 04/02/2020 - //if (av < scoreThreshold) - //{ - // continue; //skip events whose score is < the threshold - //} - - AcousticEvent ev = new AcousticEvent(segmentStartOffset, startTime, duration, minHz, maxHz); + // Initialize the event. + AcousticEvent ev = new AcousticEvent(segmentStartWrtRecording, startTimeWrtSegment, duration, minHz, maxHz); ev.SetTimeAndFreqScales(framesPerSec, freqBinWidth); ev.Score = av; @@ -1366,12 +1404,12 @@ public static double[] ExtractScoreArrayFromEvents(List events, i double windowOffset = events[0].FrameOffset; double frameRate = 1 / windowOffset; //frames per second - //int count = events.Count; - foreach ( AcousticEvent ae in events) + foreach (AcousticEvent ae in events) { if (!ae.Name.Equals(nameOfTargetEvent)) { - continue; //skip irrelevant events + //skip irrelevant events + continue; } int startFrame = (int)(ae.TimeStart * frameRate); @@ -1388,39 +1426,6 @@ public static double[] ExtractScoreArrayFromEvents(List events, i //############################################################################################################################################## - /// - /// TODO: THis should be deprecated! - /// This method is used to do unit test on lists of events. - /// First developed for frog recognizers - October 2016. - /// - public static void TestToCompareEvents(string fileName, DirectoryInfo opDir, string testName, List events) - { - var testDir = new DirectoryInfo(opDir + $"\\UnitTest_{testName}"); - var benchmarkDir = new DirectoryInfo(testDir + "\\ExpectedOutput"); - if (!benchmarkDir.Exists) - { - benchmarkDir.Create(); - } - - var benchmarkFilePath = Path.Combine(benchmarkDir.FullName, fileName + ".TestEvents.csv"); - var eventsFilePath = Path.Combine(testDir.FullName, fileName + ".Events.csv"); - var eventsFile = new FileInfo(eventsFilePath); - Csv.WriteToCsv(eventsFile, events); - - LoggedConsole.WriteLine($"# EVENTS TEST: Comparing List of {testName} events with those in benchmark file:"); - var benchmarkFile = new FileInfo(benchmarkFilePath); - if (!benchmarkFile.Exists) - { - LoggedConsole.WriteWarnLine(" A file of test/benchmark events does not exist. Writing output as future events-test file"); - Csv.WriteToCsv(benchmarkFile, events); - } - else - { - // compare the test events with benchmark - TestTools.FileEqualityTest("Compare acoustic events.", eventsFile, benchmarkFile); - } - } - /// /// Although not currently used, this method and following methods could be useful in future for clustering of events. /// From 072742f4ff07ca5c4c8cc79dae6c4694912f6122 Mon Sep 17 00:00:00 2001 From: towsey Date: Sat, 7 Mar 2020 09:53:14 +1000 Subject: [PATCH 05/12] Remove a call to one of my do-it-yourself test methods. Issue #300 remove DIY test method. --- src/AnalysisPrograms/Recognizers/LewiniaPectoralis.cs | 6 ++++-- src/AnalysisPrograms/Recognizers/LitoriaBicolor.cs | 6 ++++-- src/AnalysisPrograms/Recognizers/LitoriaWatjulumensis.cs | 8 +++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/AnalysisPrograms/Recognizers/LewiniaPectoralis.cs b/src/AnalysisPrograms/Recognizers/LewiniaPectoralis.cs index ca8dee5d9..b994cccce 100644 --- a/src/AnalysisPrograms/Recognizers/LewiniaPectoralis.cs +++ b/src/AnalysisPrograms/Recognizers/LewiniaPectoralis.cs @@ -1,4 +1,4 @@ -// -------------------------------------------------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // @@ -193,13 +193,15 @@ public override RecognizerResults Recognize( prunedEvents.Add(ae); } - // do a recognizer TEST. + /* + // do a recognizer TEST. DELETE ONE DAY! if (false) { var testDir = new DirectoryInfo(outputDirectory.Parent.Parent.FullName); TestTools.RecognizerScoresTest(recording.BaseName, testDir, recognizerConfig.AnalysisName, scoreArray); AcousticEvent.TestToCompareEvents(recording.BaseName, testDir, recognizerConfig.AnalysisName, predictedEvents); } + */ // increase very low scores for (int j = 0; j < scoreArray.Length; j++) diff --git a/src/AnalysisPrograms/Recognizers/LitoriaBicolor.cs b/src/AnalysisPrograms/Recognizers/LitoriaBicolor.cs index a0a6109f7..2d94aa632 100644 --- a/src/AnalysisPrograms/Recognizers/LitoriaBicolor.cs +++ b/src/AnalysisPrograms/Recognizers/LitoriaBicolor.cs @@ -152,13 +152,15 @@ public override RecognizerResults Recognize(AudioRecording recording, Config con ae.SegmentDurationSeconds = recordingDuration.TotalSeconds; } - // do a RECOGNIZER TEST. + /* + // do a RECOGNIZER TEST. DELETE ONE DAY! if (false) { var testDir = new DirectoryInfo(outputDirectory.Parent.Parent.FullName); TestTools.RecognizerScoresTest(recording.BaseName, testDir, recognizerConfig.AnalysisName, scoreArray); AcousticEvent.TestToCompareEvents(recording.BaseName, testDir, recognizerConfig.AnalysisName, predictedEvents); } + */ var plot = new Plot(this.DisplayName, scoreArray, recognizerConfig.EventThreshold); return new RecognizerResults() @@ -171,7 +173,7 @@ public override RecognizerResults Recognize(AudioRecording recording, Config con } /// - /// ################ THE KEY ANALYSIS METHOD + /// THE KEY ANALYSIS METHOD. /// /// /// diff --git a/src/AnalysisPrograms/Recognizers/LitoriaWatjulumensis.cs b/src/AnalysisPrograms/Recognizers/LitoriaWatjulumensis.cs index a8741adb3..3f1c2ee47 100644 --- a/src/AnalysisPrograms/Recognizers/LitoriaWatjulumensis.cs +++ b/src/AnalysisPrograms/Recognizers/LitoriaWatjulumensis.cs @@ -157,13 +157,15 @@ public override RecognizerResults Recognize(AudioRecording recording, Config con ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds; } - // do a recognizer TEST. + /* + // do a recognizer TEST. DELETE ONE DAY! if (false) { var testDir = new DirectoryInfo(outputDirectory.Parent.Parent.FullName); TestTools.RecognizerScoresTest(recording.BaseName, testDir, recognizerConfig.AnalysisName, scoreArray); AcousticEvent.TestToCompareEvents(recording.BaseName, testDir, recognizerConfig.AnalysisName, predictedEvents); } + */ var plot = new Plot(this.DisplayName, scoreArray, recognizerConfig.EventThreshold); return new RecognizerResults() @@ -175,8 +177,8 @@ public override RecognizerResults Recognize(AudioRecording recording, Config con }; } - /// - /// ################ THE KEY ANALYSIS METHOD for TRILLS + /// + /// ################ THE KEY ANALYSIS METHOD for TRILLS /// /// See Anthony's ExempliGratia.Recognize() method in order to see how to use methods for config profiles. /// From fa15eef38c0f52056babaec99a069e0cd9ded4f0 Mon Sep 17 00:00:00 2001 From: towsey Date: Sat, 7 Mar 2020 19:31:22 +1000 Subject: [PATCH 06/12] Update AcousticEvent.cs Issue #300 Final debug of merge events. It now works correctly. --- src/AudioAnalysisTools/AcousticEvent.cs | 141 ++++++++++++++++-------- 1 file changed, 95 insertions(+), 46 deletions(-) diff --git a/src/AudioAnalysisTools/AcousticEvent.cs b/src/AudioAnalysisTools/AcousticEvent.cs index f13354e13..ebd3e7124 100644 --- a/src/AudioAnalysisTools/AcousticEvent.cs +++ b/src/AudioAnalysisTools/AcousticEvent.cs @@ -527,7 +527,7 @@ public AcousticEvent OverlapsEventInList(List events) /// public bool Overlaps(AcousticEvent ae) { - return EventsOverlap(this, ae); + return EventsOverlapInTime(this, ae); } /// @@ -565,54 +565,74 @@ public static double EventFractionalOverlap(AcousticEvent event1, AcousticEvent } /// - /// Determines if two events overlap in time or frequency or both. + /// Determines if two events overlap in frequency. /// /// event one. /// event two. /// true if events overlap. - public static bool EventsOverlap(AcousticEvent event1, AcousticEvent event2) + public static bool EventsOverlapInFrequency(AcousticEvent event1, AcousticEvent event2) { - var timeOverlap = false; - var freqOverlap = false; + //check if event 1 freq band overlaps event 2 freq band + if (event1.HighFrequencyHertz >= event2.LowFrequencyHertz && event1.HighFrequencyHertz <= event2.HighFrequencyHertz) + { + return true; + } + // check if event 1 freq band overlaps event 2 freq band + if (event1.LowFrequencyHertz >= event2.LowFrequencyHertz && event1.LowFrequencyHertz <= event2.HighFrequencyHertz) + { + return true; + } + + //check if event 2 freq band overlaps event 1 freq band + if (event2.HighFrequencyHertz >= event1.LowFrequencyHertz && event2.HighFrequencyHertz <= event1.HighFrequencyHertz) + { + return true; + } + + // check if event 2 freq band overlaps event 1 freq band + if (event2.LowFrequencyHertz >= event1.LowFrequencyHertz && event2.LowFrequencyHertz <= event1.HighFrequencyHertz) + { + return true; + } + + return false; + } + + /// + /// Determines if two events overlap in time. + /// + /// event one. + /// event two. + /// true if events overlap. + public static bool EventsOverlapInTime(AcousticEvent event1, AcousticEvent event2) + { //check if event 1 starts within event 2 if (event1.EventStartSeconds >= event2.EventStartSeconds && event1.EventStartSeconds <= event2.EventEndSeconds) { - timeOverlap = true; + return true; } // check if event 1 ends within event 2 if (event1.EventEndSeconds >= event2.EventStartSeconds && event1.EventEndSeconds <= event2.EventEndSeconds) { - timeOverlap = true; + return true; } // now check possibility that event2 is inside event1. //check if event 2 starts within event 1 if (event2.EventStartSeconds >= event1.EventStartSeconds && event2.EventStartSeconds <= event1.EventEndSeconds) { - timeOverlap = true; + return true; } // check if event 2 ends within event 1 if (event2.EventEndSeconds >= event1.EventStartSeconds && event2.EventEndSeconds <= event1.EventEndSeconds) { - timeOverlap = true; + return true; } - //check if event 1 freq band overlaps event 2 freq band - if (event1.HighFrequencyHertz >= event1.LowFrequencyHertz && event1.HighFrequencyHertz <= event2.HighFrequencyHertz) - { - freqOverlap = true; - } - - // check if event 1 freq band overlaps event 2 freq band - if (event1.LowFrequencyHertz >= event2.LowFrequencyHertz && event1.LowFrequencyHertz <= event2.HighFrequencyHertz) - { - freqOverlap = true; - } - - return timeOverlap && freqOverlap; + return false; } /// @@ -632,7 +652,7 @@ public static List CombineOverlappingEvents(List e { for (int j = i - 1; j >= 0; j--) { - if (EventsOverlap(events[i], events[j])) + if (EventsOverlapInTime(events[i], events[j]) && EventsOverlapInFrequency(events[i], events[j])) { events[j] = AcousticEvent.MergeTwoEvents(events[i], events[j], segmentStartWrtRecording); events.RemoveAt(i); @@ -662,16 +682,16 @@ public static AcousticEvent MergeTwoEvents(AcousticEvent e1, AcousticEvent e2, T //METHODS TO CONVERT BETWEEN FREQ BIN AND HERZ OR MELS /// - /// converts frequency bounds of an event to left and right columns of object in sonogram matrix - /// NOTE: binCount is required only if freq is in Mel scale + /// converts frequency bounds of an event to left and right columns of object in sonogram matrix. + /// NOTE: binCount is required only if freq is in Mel scale. /// - /// mel scale - /// lower freq bound - /// upper freq bound - /// Nyquist freq in Herz - /// frequency scale - /// return bin index for lower freq bound - /// return bin index for upper freq bound + /// mel scale. + /// lower freq bound. + /// upper freq bound. + /// Nyquist freq in Herz. + /// frequency scale. + /// return bin index for lower freq bound. + /// return bin index for upper freq bound. public static void Freq2BinIDs(bool doMelscale, int minFreq, int maxFreq, int nyquist, double binWidth, out int leftCol, out int rightCol) { if (doMelscale) @@ -839,8 +859,15 @@ public static Tuple, double, double, double, double[]> GetSe return tuple; } - public static Tuple, double, double, double, double[]> GetSegmentationEvents(SpectrogramStandard sonogram, TimeSpan segmentStartOffset, - int minHz, int maxHz, double smoothWindow, double thresholdSD, double minDuration, double maxDuration) + public static Tuple, double, double, double, double[]> GetSegmentationEvents( + SpectrogramStandard sonogram, + TimeSpan segmentStartOffset, + int minHz, + int maxHz, + double smoothWindow, + double thresholdSD, + double minDuration, + double maxDuration) { int nyquist = sonogram.SampleRate / 2; var tuple = SNR.SubbandIntensity_NoiseReduced(sonogram.Data, minHz, maxHz, nyquist, smoothWindow, sonogram.FramesPerSecond); @@ -964,7 +991,7 @@ public static void CalculateAccuracy(List results, List results, List results, List - public static void CalculateAccuracyOnOneRecording(List results, List labels, out int tp, out int fp, out int fn, - out double precision, out double recall, out double accuracy, out string resultsText) + public static void CalculateAccuracyOnOneRecording( + List results, + List labels, + out int tp, + out int fp, + out int fn, + out double precision, + out double recall, + out double accuracy, + out string resultsText) { //init values tp = 0; @@ -1087,7 +1127,12 @@ public static void CalculateAccuracyOnOneRecording(List results, fn++; line = string.Format( "False NEGATIVE: {0,4} {5,15} {1,6:f1} ...{2,6:f1} intensity={3} quality={4}", - count, ae.TimeStart, ae.TimeEnd, ae.Intensity, ae.Quality, ae.Name); + count, + ae.TimeStart, + ae.TimeEnd, + ae.Intensity, + ae.Quality, + ae.Name); sb.Append(line + "\t" + ae.FileName + "\n"); } } @@ -1113,7 +1158,7 @@ public static void CalculateAccuracyOnOneRecording(List results, accuracy = (precision + recall) / 2; resultsText = sb.ToString(); - } //end method + } //############################################################################################################################################## // THE NEXT THREE METHODS CONVERT BETWEEN SCORE ARRAYS AND ACOUSTIC EVENTS @@ -1138,17 +1183,20 @@ public static List ConvertIntensityArray2Events( double startTime = 0.0; int startFrame = 0; - for (int i = 0; i < count; i++) //pass over all frames + //pass over all frames + for (int i = 0; i < count; i++) { - if (isHit == false && values[i] > scoreThreshold) //start of an event + //start of an event + if (isHit == false && values[i] > scoreThreshold) { isHit = true; startTime = i * frameOffset; startFrame = i; } else //check for the end of an event - if (isHit && values[i] <= scoreThreshold) //this is end of an event, so initialise it + if (isHit && values[i] <= scoreThreshold) { + //this is end of an event, so initialise it isHit = false; double endTime = i * frameOffset; double duration = endTime - startTime; @@ -1159,9 +1207,10 @@ public static List ConvertIntensityArray2Events( continue; //skip events with duration shorter than threshold } - AcousticEvent ev = new AcousticEvent(segmentStartOffset, startTime, duration, minHz, maxHz); - - ev.Name = "Acoustic Segment"; //default name + AcousticEvent ev = new AcousticEvent(segmentStartOffset, startTime, duration, minHz, maxHz) + { + Name = "Acoustic Segment", //default name + }; ev.SetTimeAndFreqScales(framesPerSec, freqBinWidth); //obtain average intensity score. From ecf3ad07370454605881fe34266725c0e3f54d4f Mon Sep 17 00:00:00 2001 From: towsey Date: Tue, 10 Mar 2020 10:20:22 +1000 Subject: [PATCH 07/12] Update AcousticEvent.cs Issue #300 Removed unneccesary methods and added documentation. Grouped methods according to functionality. --- src/AudioAnalysisTools/AcousticEvent.cs | 438 +++++++++++------------- 1 file changed, 202 insertions(+), 236 deletions(-) diff --git a/src/AudioAnalysisTools/AcousticEvent.cs b/src/AudioAnalysisTools/AcousticEvent.cs index ebd3e7124..fb22fb711 100644 --- a/src/AudioAnalysisTools/AcousticEvent.cs +++ b/src/AudioAnalysisTools/AcousticEvent.cs @@ -11,10 +11,8 @@ namespace AudioAnalysisTools { using System; using System.Collections.Generic; - using System.IO; using System.Linq; using System.Text; - using Acoustics.Shared; using Acoustics.Shared.Contracts; using Acoustics.Shared.Csv; using Acoustics.Shared.ImageSharp; @@ -26,7 +24,6 @@ namespace AudioAnalysisTools using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; using TowseyLibrary; - using Path = System.IO.Path; public class AcousticEvent : EventBase { @@ -40,7 +37,7 @@ public sealed class AcousticEventClassMap : ClassMap { nameof(TimeStart), nameof(TimeEnd), nameof(Bandwidth), nameof(IsMelscale), nameof(FrameOffset), - nameof(FramesPerSecond), nameof(Name2), nameof(ScoreComment), + nameof(FramesPerSecond), nameof(ScoreNormalised), nameof(Score_MaxPossible), nameof(Score_MaxInEvent), nameof(Score_TimeOfMaxInEvent), nameof(Score2Name), nameof(Score2), nameof(Periodicity), nameof(DominantFreq), @@ -71,6 +68,13 @@ public AcousticEventClassMap() } } + // double I1MeandB; //mean intensity of pixels in the event prior to noise subtraction + // double I1Var; //, + // double I2MeandB; // mean intensity of pixels in the event after Wiener filter, prior to noise subtraction + // double I2Var; //, + // private double i3Mean; // mean intensity of pixels in the event AFTER noise reduciton - USED FOR CLUSTERING + // private double i3Var; // variance of intensity of pixels in the event. + /// /// Gets the time offset from start of current segment to start of event in seconds. /// NOTE: AcousticEvents do not have a notion of time offset wrt start of recording ; - only to start of current recording segment. @@ -103,21 +107,6 @@ public AcousticEventClassMap() /// public override double EventStartSeconds => this.TimeStart + this.SegmentStartSeconds; - /// - /// Set the start and end times of an event with respect to the segment start time - /// AND also calls method to set event start time with respect the recording/file start. - /// - public void SetEventPositionRelative( - TimeSpan segmentStartWrtRecording, - double eventStartWrtSegment, - double eventEndWrtSegment) - { - this.TimeStart = eventStartWrtSegment; - this.TimeEnd = eventEndWrtSegment; - - this.SetSegmentAndEventStartsWrtRecording(segmentStartWrtRecording, eventStartWrtSegment); - } - /// /// Gets or sets units = Hertz. /// Proxied to EventBase.MinHz. @@ -175,11 +164,6 @@ public void SetEventPositionRelative( public string Name { get; set; } - public string Name2 { get; set; } - - /// Gets or sets average score through the event. - public string ScoreComment { get; set; } - /// Gets or sets score normalised in range [0,1]. NOTE: Max is set = to five times user supplied threshold. public double ScoreNormalised { get; set; } @@ -202,7 +186,7 @@ public void SetEventPositionRelative( /// i.e. Points can be outside of events and can have negative values. /// Point location is relative to the top left corner of the event. /// - public List Points { get; set; } + //public List Points { get; set; } /// /// Gets or sets the periodicity of acoustic energy in an event. @@ -212,29 +196,10 @@ public void SetEventPositionRelative( public double DominantFreq { get; set; } // the dominant freq in the event - used for frog calls - // double I1MeandB; //mean intensity of pixels in the event prior to noise subtraction - // double I1Var; //, - // double I2MeandB; // mean intensity of pixels in the event after Wiener filter, prior to noise subtraction - // double I2Var; //, - private double i3Mean; // mean intensity of pixels in the event AFTER noise reduciton - USED FOR CLUSTERING - private double i3Var; // variance of intensity of pixels in the event. - - // following are no longer needed. Delete eventually. - /* - //KIWI SCORES - public double kiwi_durationScore; - public double kiwi_hitScore; - public double kiwi_snrScore; - public double kiwi_sdPeakScore; - public double kiwi_intensityScore; - public double kiwi_gridScore; - public double kiwi_chirpScore; - public double kiwi_bandWidthScore; - public double kiwi_deltaPeriodScore; - public double kiwi_comboScore; - */ - - /// Gets or sets a value indicating whether use this if want to filter or tag some members of a list for some purpose. + /// + /// Gets or sets a value that can be used to filter or tag some members of a list of acoustic events. + /// Was used for constructing data sets. + /// public bool Tag { get; set; } /// Gets or sets assigned value when reading in a list of user identified events. Indicates a user assigned assessment of event intensity. @@ -247,6 +212,11 @@ public void SetEventPositionRelative( public Color ScoreColour { get; set; } + /// + /// Initializes a new instance of the class. + /// Sets some default colors for drawing an event on a spectrogram. + /// THis is the first of three constructors. + /// public AcousticEvent() { this.BorderColour = DefaultBorderColor; @@ -323,22 +293,44 @@ public AcousticEvent(TimeSpan segmentStartOffset, Oblong o, int nyquistFrequency /// public string Profile { get; set; } - public void DoMelScale(bool doMelscale, int freqBinCount) + //public void DoMelScale(bool doMelscale, int freqBinCount) + //{ + // this.IsMelscale = doMelscale; + // this.FreqBinCount = freqBinCount; + //} + + /// + /// Set the start and end times of an event with respect to the segment start time + /// AND also calls method to set event start time with respect the recording/file start. + /// + public void SetEventPositionRelative( + TimeSpan segmentStartWrtRecording, + double eventStartWrtSegment, + double eventEndWrtSegment) { - this.IsMelscale = doMelscale; - this.FreqBinCount = freqBinCount; + this.TimeStart = eventStartWrtSegment; + this.TimeEnd = eventEndWrtSegment; + + this.SetSegmentAndEventStartsWrtRecording(segmentStartWrtRecording, eventStartWrtSegment); } + /// + /// THe only call to this method is from a no-longer used recogniser. + /// Could be deleted. + /// It sets the time and frequency scales for an event given the sr, and window size. + /// public void SetTimeAndFreqScales(int samplingRate, int windowSize, int windowOffset) { - CalculateTimeScale(samplingRate, windowSize, windowOffset, out var frameDuration, out var frameOffset, out var framesPerSecond); - this.FrameDuration = frameDuration; //frame duration in seconds - this.FrameOffset = frameOffset; //frame offset in seconds - this.FramesPerSecond = framesPerSecond; //inverse of the frame offset + //set the frame duration and offset in seconds + this.FrameDuration = windowSize / (double)samplingRate; + this.FrameOffset = windowOffset / (double)samplingRate; + this.FramesPerSecond = 1 / this.FrameOffset; + + //set the Freq Scale. Required for freq-binID conversions + this.FreqBinWidth = samplingRate / (double)windowSize; - CalculateFreqScale(samplingRate, windowSize, out var binCount, out var binWidth); - this.FreqBinCount = binCount; //required for conversions to & from MEL scale - this.FreqBinWidth = binWidth; //required for freq-binID conversions + //required for conversions to & from MEL scale + this.FreqBinCount = windowSize / 2; if (this.Oblong == null) { @@ -361,9 +353,7 @@ public void SetTimeAndFreqScales(double frameOffset, double frameDuration, doubl { this.FramesPerSecond = 1 / frameOffset; //inverse of the frame offset this.FrameDuration = frameDuration; //frame duration in seconds - this.FrameOffset = frameOffset; //frame duration in seconds - - //this.FreqBinCount = binCount; //required for conversions to & from MEL scale + this.FrameOffset = frameOffset; //frame duration in seconds this.FreqBinWidth = freqBinWidth; //required for freq-binID conversions if (this.Oblong == null) @@ -372,6 +362,34 @@ public void SetTimeAndFreqScales(double frameOffset, double frameDuration, doubl } } + /// + /// Converts frequency bounds of an event to left and right columns of object in sonogram matrix. + /// + /// mel scale. + /// lower freq bound. + /// upper freq bound. + /// Nyquist freq in Herz. + /// frequency scale. + /// return bin index for lower freq bound. + /// return bin index for upper freq bound. + public static void Freq2BinIDs(bool doMelscale, int minFreq, int maxFreq, int nyquist, double binWidth, out int leftCol, out int rightCol) + { + if (doMelscale) + { + int binCount = (int)(nyquist / binWidth) + 1; + double maxMel = MFCCStuff.Mel(nyquist); + int melRange = (int)(maxMel - 0 + 1); + double binsPerMel = binCount / (double)melRange; + leftCol = (int)Math.Round(MFCCStuff.Mel(minFreq) * binsPerMel); + rightCol = (int)Math.Round(MFCCStuff.Mel(maxFreq) * binsPerMel); + } + else + { + leftCol = (int)Math.Round(minFreq / binWidth); + rightCol = (int)Math.Round(maxFreq / binWidth); + } + } + /// /// Calculates the matrix/image indices of the acoustic event, when given the time/freq scales. /// This method called only by previous method:- Acousticevent.SetTimeAndFreqScales(). @@ -380,8 +398,9 @@ public void SetTimeAndFreqScales(double frameOffset, double frameDuration, doubl /// public static Oblong ConvertEvent2Oblong(AcousticEvent ae) { - // Translate time dimension = frames = matrix rows. - Time2RowIDs(ae.TimeStart, ae.EventDurationSeconds, ae.FrameOffset, out var topRow, out var bottomRow); + // Translate time dimension (seconds) to frames to matrix rows. + var topRow = (int)Math.Round(ae.TimeStart / ae.FrameOffset); + var bottomRow = (int)Math.Round((ae.TimeStart + ae.EventDurationSeconds) / ae.FrameOffset); //Translate freq dimension = freq bins = matrix columns. Freq2BinIDs(ae.IsMelscale, (int)ae.LowFrequencyHertz, (int)ae.HighFrequencyHertz, ae.FreqBinCount, ae.FreqBinWidth, out var leftCol, out var rightCol); @@ -412,20 +431,20 @@ public void SetScores(double score, double min, double max) } } - public string WriteProperties() - { - return " min-max=" + this.LowFrequencyHertz + "-" + this.HighFrequencyHertz + ", " + this.Oblong.ColumnLeft + "-" + this.Oblong.ColumnRight; - } + //public string WriteProperties() + //{ + // return " min-max=" + this.LowFrequencyHertz + "-" + this.HighFrequencyHertz + ", " + this.Oblong.ColumnLeft + "-" + this.Oblong.ColumnRight; + //} /// /// Draws an event on the image. Uses the fields already set on the audio event to determine correct placement. /// Fields requireed to be set include: `FramesPerSecond`, `FreqBinWidth`. /// - public void DrawEvent(Image sonogram) - where T : unmanaged, IPixel - { - this.DrawEvent(sonogram, this.FramesPerSecond, this.FreqBinWidth, sonogram.Height); - } + //public void DrawEvent(Image sonogram) + // where T : unmanaged, IPixel + //{ + // this.DrawEvent(sonogram, this.FramesPerSecond, this.FreqBinWidth, sonogram.Height); + //} /// /// Draws an event on the image. Allows for custom specification of variables. @@ -482,6 +501,7 @@ public void DrawEvent(Image imageToReturn, double framesPerSecond, double }); } + /* /// /// Passed point is relative to top-left corner of the Acoustic Event. /// Oblong needs to be set for this method to work. @@ -504,65 +524,10 @@ public void DrawPoint(Image bmp, Point point, Color colour) bmp[col, row] = colour; } + */ - /// - /// Returns the first event in the passed list which overlaps with this one IN THE SAME RECORDING. - /// If no event overlaps return null. - /// - public AcousticEvent OverlapsEventInList(List events) - { - foreach (AcousticEvent ae in events) - { - if (this.FileName.Equals(ae.FileName) && this.Overlaps(ae)) - { - return ae; - } - } - - return null; - } - - /// - /// Returns true/false if this event time-overlaps the passed event. - /// - public bool Overlaps(AcousticEvent ae) - { - return EventsOverlapInTime(this, ae); - } - - /// - /// Returns the fractional overlap of two events. - /// Translate time/freq dimensions to coordinates in a matrix. - /// Freq dimension = bins = matrix columns. Origin is top left - as per matrix in the sonogram class. - /// Time dimension = frames = matrix rows. - /// - public static double EventFractionalOverlap(AcousticEvent event1, AcousticEvent event2) - { - int timeOverlap = Oblong.RowOverlap(event1.Oblong, event2.Oblong); - if (timeOverlap == 0) - { - return 0.0; - } - - int hzOverlap = Oblong.ColumnOverlap(event1.Oblong, event2.Oblong); - if (hzOverlap == 0) - { - return 0.0; - } - - int overlapArea = timeOverlap * hzOverlap; - double fractionalOverlap1 = overlapArea / (double)event1.Oblong.Area(); - double fractionalOverlap2 = overlapArea / (double)event2.Oblong.Area(); - - if (fractionalOverlap1 > fractionalOverlap2) - { - return fractionalOverlap1; - } - else - { - return fractionalOverlap2; - } - } + //################################################################################################################# + //FOLLOWING METHODS DEAL WITH THE OVERLAP OF EVENTS /// /// Determines if two events overlap in frequency. @@ -678,115 +643,65 @@ public static AcousticEvent MergeTwoEvents(AcousticEvent e1, AcousticEvent e2, T return e1; } - //################################################################################################################# - //METHODS TO CONVERT BETWEEN FREQ BIN AND HERZ OR MELS - /// - /// converts frequency bounds of an event to left and right columns of object in sonogram matrix. - /// NOTE: binCount is required only if freq is in Mel scale. + /// Returns the first event in the passed list which overlaps with this one IN THE SAME RECORDING. + /// If no event overlaps return null. /// - /// mel scale. - /// lower freq bound. - /// upper freq bound. - /// Nyquist freq in Herz. - /// frequency scale. - /// return bin index for lower freq bound. - /// return bin index for upper freq bound. - public static void Freq2BinIDs(bool doMelscale, int minFreq, int maxFreq, int nyquist, double binWidth, out int leftCol, out int rightCol) + public AcousticEvent OverlapsEventInList(List events) { - if (doMelscale) - { - Freq2MelsBinIDs(minFreq, maxFreq, binWidth, nyquist, out leftCol, out rightCol); - } - else + foreach (AcousticEvent ae in events) { - Freq2HerzBinIDs(minFreq, maxFreq, binWidth, out leftCol, out rightCol); + if (this.FileName.Equals(ae.FileName) && EventsOverlapInTime(this, ae)) + { + return ae; + } } - } - - public static void Freq2HerzBinIDs(int minFreq, int maxFreq, double binWidth, out int leftCol, out int rightCol) - { - leftCol = (int)Math.Round(minFreq / binWidth); - rightCol = (int)Math.Round(maxFreq / binWidth); - } - public static void Freq2MelsBinIDs(int minFreq, int maxFreq, double binWidth, int nyquistFrequency, out int leftCol, out int rightCol) - { - int binCount = (int)(nyquistFrequency / binWidth) + 1; - double maxMel = MFCCStuff.Mel(nyquistFrequency); - int melRange = (int)(maxMel - 0 + 1); - double binsPerMel = binCount / (double)melRange; - leftCol = (int)Math.Round(MFCCStuff.Mel(minFreq) * binsPerMel); - rightCol = (int)Math.Round(MFCCStuff.Mel(maxFreq) * binsPerMel); - } - - //################################################################################################################# - //METHODS TO CONVERT BETWEEN TIME BIN AND SECONDS - - public static void Time2RowIDs(double startTime, double duration, double frameOffset, out int topRow, out int bottomRow) - { - topRow = (int)Math.Round(startTime / frameOffset); - bottomRow = (int)Math.Round((startTime + duration) / frameOffset); - } - - public void SetNetIntensityAfterNoiseReduction(double mean, double var) - { - this.i3Mean = mean; - this.i3Var = var; + return null; } + /* /// - /// returns the frame duration and offset duration in seconds. + /// This method not currently called but is POTENTIALLY USEFUL. + /// Returns the fractional overlap of two events. + /// Translate time/freq dimensions to coordinates in a matrix. + /// Freq dimension = bins = matrix columns. Origin is top left - as per matrix in the sonogram class. + /// Time dimension = frames = matrix rows. /// - /// signal samples per second. - /// number of signal samples in one window or frame. - /// number of signal samples between start of one frame and start of next frame. - /// units = seconds. - /// units = second. - /// number of frames in one second. - public static void CalculateTimeScale(int samplingRate, int windowSize, int windowOffset, out double frameDuration, out double frameOffset, out double framesPerSecond) + public static double EventFractionalOverlap(AcousticEvent event1, AcousticEvent event2) { - frameDuration = windowSize / (double)samplingRate; - frameOffset = windowOffset / (double)samplingRate; - framesPerSecond = 1 / frameOffset; - } + int timeOverlap = Oblong.RowOverlap(event1.Oblong, event2.Oblong); + if (timeOverlap == 0) + { + return 0.0; + } - /// - /// return Nyquist / binCount. - /// - public static void CalculateFreqScale(int samplingRate, int windowSize, out int binCount, out double binWidth) - { - binCount = windowSize / 2; - binWidth = samplingRate / (double)windowSize; - } + int hzOverlap = Oblong.ColumnOverlap(event1.Oblong, event2.Oblong); + if (hzOverlap == 0) + { + return 0.0; + } - public static void WriteEvents(List eventList, ref StringBuilder sb) - { - if (eventList.Count == 0) + int overlapArea = timeOverlap * hzOverlap; + double fractionalOverlap1 = overlapArea / (double)event1.Oblong.Area(); + double fractionalOverlap2 = overlapArea / (double)event2.Oblong.Area(); + + if (fractionalOverlap1 > fractionalOverlap2) { - string line = - $"# Event Name\t{"Start",8:f3}\t{"End",6:f3}\t{"MinF"}\t{"MaxF"}\t{"Score1":f2}\t{"Score2":f1}\t{"SourceFile"}"; - sb.AppendLine(line); - line = $"{"NoEvent"}\t{0.000,8:f3}\t{0.000,8:f3}\t{"N/A"}\t{"N/A"}\t{0.000:f2}\t{0.000:f1}\t{"N/A"}"; - sb.AppendLine(line); + return fractionalOverlap1; } else { - AcousticEvent ae1 = eventList[0]; - string line = - $"# Event Name\t{"Start",8:f3}\t{"End",6:f3}\t{"MinF"}\t{"MaxF"}\t{"Score":f2}\t{ae1.Score2Name:f1}\t{"SourceFile"}"; - sb.AppendLine(line); - foreach (AcousticEvent ae in eventList) - { - line = - $"{ae.Name}\t{ae.TimeStart,8:f3}\t{ae.TimeEnd,8:f3}\t{ae.LowFrequencyHertz}\t{ae.HighFrequencyHertz}\t{ae.Score:f2}\t{ae.Score2:f1}\t{ae.FileName}"; - sb.AppendLine(line); - } + return fractionalOverlap2; } } + */ + + //################################################################################################################# + //METHOD TO WRITE ACOUSTIC EVETNS TO FILE /// - /// used to write lists of acousitc event data to an excell spread sheet. + /// Writes acousitc event data to an excel spread sheet. /// public static StringBuilder WriteEvents(List eventList, string str) { @@ -819,6 +734,36 @@ public static StringBuilder WriteEvents(List eventList, string st return sb; } + /* + public static void WriteEvents(List eventList, ref StringBuilder sb) + { + if (eventList.Count == 0) + { + string line = + $"# Event Name\t{"Start",8:f3}\t{"End",6:f3}\t{"MinF"}\t{"MaxF"}\t{"Score1":f2}\t{"Score2":f1}\t{"SourceFile"}"; + sb.AppendLine(line); + line = $"{"NoEvent"}\t{0.000,8:f3}\t{0.000,8:f3}\t{"N/A"}\t{"N/A"}\t{0.000:f2}\t{0.000:f1}\t{"N/A"}"; + sb.AppendLine(line); + } + else + { + AcousticEvent ae1 = eventList[0]; + string line = + $"# Event Name\t{"Start",8:f3}\t{"End",6:f3}\t{"MinF"}\t{"MaxF"}\t{"Score":f2}\t{ae1.Score2Name:f1}\t{"SourceFile"}"; + sb.AppendLine(line); + foreach (AcousticEvent ae in eventList) + { + line = + $"{ae.Name}\t{ae.TimeStart,8:f3}\t{ae.TimeEnd,8:f3}\t{ae.LowFrequencyHertz}\t{ae.HighFrequencyHertz}\t{ae.Score:f2}\t{ae.Score2:f1}\t{ae.FileName}"; + sb.AppendLine(line); + } + } + } + */ + + //################################################################################################################# + //METHODS FOR SEGMENTATION OF A FREQ BAND BASED ON ACOUSTIC ENERGY + /// /// Segments or not depending value of boolean doSegmentation. /// @@ -859,6 +804,19 @@ public static Tuple, double, double, double, double[]> GetSe return tuple; } + /// + /// Segments the acoustic energy in the passed frequency band and returns as list of acoustic events. + /// Noise reduction is done first. + /// + /// the full spectrogram. + /// Start of current segment wrt recording start. + /// Bottom of the required frequency band. + /// Top of the required frequency band. + /// To smooth the amplitude array. + /// Determines the threshold for an acoustic event. + /// Minimum duration of an acceptable acoustic event. + /// Maximum duration of an acceptable acoustic event. + /// a list of acoustic events. public static Tuple, double, double, double, double[]> GetSegmentationEvents( SpectrogramStandard sonogram, TimeSpan segmentStartOffset, @@ -872,10 +830,11 @@ public static Tuple, double, double, double, double[]> GetSe int nyquist = sonogram.SampleRate / 2; var tuple = SNR.SubbandIntensity_NoiseReduced(sonogram.Data, minHz, maxHz, nyquist, smoothWindow, sonogram.FramesPerSecond); double[] intensity = tuple.Item1; //noise reduced intensity array - double Q = tuple.Item2; //baseline dB in the original scale + double baselineDb = tuple.Item2; //baseline dB in the original scale double oneSD = tuple.Item3; //1 SD in dB around the baseline double dBThreshold = thresholdSD * oneSD; + // get list of acoustic events var segmentEvents = ConvertIntensityArray2Events( intensity, segmentStartOffset, @@ -889,13 +848,16 @@ public static Tuple, double, double, double, double[]> GetSe foreach (AcousticEvent ev in segmentEvents) { ev.FileName = sonogram.Configuration.SourceFName; - - //ev.Name = callName; } - return Tuple.Create(segmentEvents, Q, oneSD, dBThreshold, intensity); + return Tuple.Create(segmentEvents, baselineDb, oneSD, dBThreshold, intensity); } + //############################################################################################################################################## + // THE NEXT FOUR METHODS ARE NOT CURRENTLY CALLED. + // THEY WERE USED FOR COLLECTING EVENTS INTO DATA SETS for Machine Learning purposes. (Kiwi publications) + // MAY BE USEFUL IN FUTURE + /// /// returns all the events in a list that occur in the recording with passed file name. /// @@ -911,7 +873,7 @@ public static List GetEventsInFile(List eventList, } return events; - } // end method GetEventsInFile(List eventList, string fileName) + } public static List GetTaggedEventsInFile(List labeledEvents, string filename) { @@ -1057,7 +1019,7 @@ public static void CalculateAccuracy(List results, List /// Given two lists of AcousticEvents, one being labelled events and the other being predicted events, @@ -1161,9 +1123,7 @@ public static void CalculateAccuracyOnOneRecording( } //############################################################################################################################################## -// THE NEXT THREE METHODS CONVERT BETWEEN SCORE ARRAYS AND ACOUSTIC EVENTS -// THE NEXT TWO METHOD CONVERT AN ARRAY OF SCORE (USUALLY INTENSITY VALUES IN A SUB-BAND) TO ACOUSTIC EVENTS. -// THE THIRD METHOD PRODUCES A SCORE ARRAY GIVEN A LIST OF EVENTS. +// THE NEXT THREE METHODS CONVERT AN ARRAY OF SCORE VALUES (USUALLY INTENSITY VALUES IN A SUB-BAND) TO ACOUSTIC EVENTS. public static List ConvertIntensityArray2Events( double[] values, @@ -1229,7 +1189,7 @@ public static List ConvertIntensityArray2Events( } /// - /// Given a time series of acoustic amplitude (typically in decibels), this method finds events that match the passed constraints. + /// Given a time series of acoustic amplitude (typically in decibels), finds events that match the passed constraints. /// /// an array of amplitude values, typically decibel values. /// not sure what this is about!. @@ -1368,7 +1328,7 @@ public static List ConvertScoreArray2Events( var events = new List(); double maxPossibleScore = 5 * scoreThreshold; // used to calculate a normalised score between 0 - 1.0 bool isHit = false; - double frameOffset = 1 / framesPerSec; // frame offset in fractions of second + double frameOffset = 1 / framesPerSec; double startTimeWrtSegment = 0.0; // units = seconds int startFrame = 0; @@ -1393,7 +1353,8 @@ public static List ConvertScoreArray2Events( // if (duration < minDuration) continue; //skip events with duration shorter than threshold if (duration < minDuration || duration > maxDuration) { - continue; //skip events with duration shorter than threshold + //skip events with duration shorter than threshold + continue; } // obtain an average score for the duration of the potential event. @@ -1407,10 +1368,11 @@ public static List ConvertScoreArray2Events( // Initialize the event. AcousticEvent ev = new AcousticEvent(segmentStartWrtRecording, startTimeWrtSegment, duration, minHz, maxHz); - ev.SetTimeAndFreqScales(framesPerSec, freqBinWidth); ev.Score = av; - ev.ScoreNormalised = ev.Score / maxPossibleScore; // normalised to the user supplied threshold + + // normalised to the user supplied threshold + ev.ScoreNormalised = ev.Score / maxPossibleScore; if (ev.ScoreNormalised > 1.0) { ev.ScoreNormalised = 1.0; @@ -1432,14 +1394,17 @@ public static List ConvertScoreArray2Events( events.Add(ev); } - } //end of pass over all frames + } return events; - } //end method ConvertScoreArray2Events() + } /// - /// Extracts an array of scores from a list of events. - /// The events are required to have the passed name. + /// FOR POSSIBLE DELETION! + /// THis method called only once from a frog recogniser class that is no longer used> LitoriaCaerulea:RecognizerBase. + /// THis method is potentially useful but can be deleted. + /// Attempts to reconstruct an array of scores from a list of acoustic events. + /// The events are required to have the passed name (a filter). /// The events are assumed to contain sufficient info about frame rate in order to populate the array. /// public static double[] ExtractScoreArrayFromEvents(List events, int arraySize, string nameOfTargetEvent) @@ -1474,6 +1439,7 @@ public static double[] ExtractScoreArrayFromEvents(List events, i } //############################################################################################################################################## + // METHODS to CLUSTER acoustic events /// /// Although not currently used, this method and following methods could be useful in future for clustering of events. From bca4013f2173d01fcd935b2e20e8d0a4964de6ec Mon Sep 17 00:00:00 2001 From: towsey Date: Tue, 10 Mar 2020 17:11:11 +1000 Subject: [PATCH 08/12] Changes as requested by Anthony Issue #300 Most of the changes are to terminology of temporal variables. Also to signatures of test methods. --- src/AnalysisBase/ResultBases/EventBase.cs | 6 +- .../Recognizers/Base/WhistleParameters.cs | 6 +- src/AudioAnalysisTools/AcousticEvent.cs | 112 ++++-------------- .../StandardSpectrograms/SpectrogramTools.cs | 4 +- .../AcousticEvents/EventTests.cs | 39 +----- 5 files changed, 35 insertions(+), 132 deletions(-) diff --git a/src/AnalysisBase/ResultBases/EventBase.cs b/src/AnalysisBase/ResultBases/EventBase.cs index 42f4fd6d2..411c76c21 100644 --- a/src/AnalysisBase/ResultBases/EventBase.cs +++ b/src/AnalysisBase/ResultBases/EventBase.cs @@ -63,10 +63,10 @@ public virtual double EventStartSeconds /// /// Sets both the Segment start and the Event start wrt to recording. /// - protected void SetSegmentAndEventStartsWrtRecording(TimeSpan segmentStartWrtRecording, double eventStartWrtSegment) + protected void SetEventStartRelative(TimeSpan segmentStart, double eventStartSegmentRelative) { - this.SegmentStartSeconds = segmentStartWrtRecording.TotalSeconds; - this.EventStartSeconds = this.SegmentStartSeconds + eventStartWrtSegment; + this.SegmentStartSeconds = segmentStart.TotalSeconds; + this.EventStartSeconds = this.SegmentStartSeconds + eventStartSegmentRelative; } } } diff --git a/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs b/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs index f1007d972..77231d622 100644 --- a/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs +++ b/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs @@ -30,7 +30,7 @@ public static (List, double[]) GetWhistles( double decibelThreshold, double minDuration, double maxDuration, - TimeSpan segmentStartWrtRecording) + TimeSpan segmentStartOffset) { var sonogramData = sonogram.Data; int frameCount = sonogramData.GetLength(0); @@ -93,7 +93,7 @@ public static (List, double[]) GetWhistles( decibelThreshold, minDuration, maxDuration, - segmentStartWrtRecording); + segmentStartOffset); // add to conbined intensity array for (int t = 0; t < frameCount; t++) @@ -107,7 +107,7 @@ public static (List, double[]) GetWhistles( } //end for all freq bins // combine adjacent acoustic events - events = AcousticEvent.CombineOverlappingEvents(events, segmentStartWrtRecording); + events = AcousticEvent.CombineOverlappingEvents(events, segmentStartOffset); return (events, combinedIntensityArray); } diff --git a/src/AudioAnalysisTools/AcousticEvent.cs b/src/AudioAnalysisTools/AcousticEvent.cs index fb22fb711..f1c59667e 100644 --- a/src/AudioAnalysisTools/AcousticEvent.cs +++ b/src/AudioAnalysisTools/AcousticEvent.cs @@ -68,13 +68,6 @@ public AcousticEventClassMap() } } - // double I1MeandB; //mean intensity of pixels in the event prior to noise subtraction - // double I1Var; //, - // double I2MeandB; // mean intensity of pixels in the event after Wiener filter, prior to noise subtraction - // double I2Var; //, - // private double i3Mean; // mean intensity of pixels in the event AFTER noise reduciton - USED FOR CLUSTERING - // private double i3Var; // variance of intensity of pixels in the event. - /// /// Gets the time offset from start of current segment to start of event in seconds. /// NOTE: AcousticEvents do not have a notion of time offset wrt start of recording ; - only to start of current recording segment. @@ -177,7 +170,6 @@ public AcousticEventClassMap() public string Score2Name { get; set; } /// Gets or sets second score if required. - /// Was used for Birgits recognisers but should now be used only for debug purposes. /// public double Score2 { get; set; } @@ -234,11 +226,11 @@ public AcousticEvent() /// event end with respect to start of segment. /// Lower frequency bound of event. /// Upper frequency bound of event. - public AcousticEvent(TimeSpan segmentStartWrtRecording, double startTimeWrtSegment, double eventDuration, double minFreq, double maxFreq) + public AcousticEvent(TimeSpan segmentStartOffset, double eventStartSegmentRelative, double eventDuration, double minFreq, double maxFreq) : this() { - var endTimeWrtSegment = startTimeWrtSegment + eventDuration; - this.SetEventPositionRelative(segmentStartWrtRecording, startTimeWrtSegment, endTimeWrtSegment); + var eventEndSegmentRelative = eventStartSegmentRelative + eventDuration; + this.SetEventPositionRelative(segmentStartOffset, eventStartSegmentRelative, eventEndSegmentRelative); this.LowFrequencyHertz = minFreq; this.HighFrequencyHertz = maxFreq; @@ -304,14 +296,14 @@ public AcousticEvent(TimeSpan segmentStartOffset, Oblong o, int nyquistFrequency /// AND also calls method to set event start time with respect the recording/file start. /// public void SetEventPositionRelative( - TimeSpan segmentStartWrtRecording, - double eventStartWrtSegment, - double eventEndWrtSegment) + TimeSpan segmentStartOffset, + double eventStartSegmentRelative, + double eventEndSegmentRelative) { - this.TimeStart = eventStartWrtSegment; - this.TimeEnd = eventEndWrtSegment; + this.TimeStart = eventStartSegmentRelative; + this.TimeEnd = eventEndSegmentRelative; - this.SetSegmentAndEventStartsWrtRecording(segmentStartWrtRecording, eventStartWrtSegment); + this.SetEventStartRelative(segmentStartOffset, eventStartSegmentRelative); } /// @@ -363,7 +355,10 @@ public void SetTimeAndFreqScales(double frameOffset, double frameDuration, doubl } /// - /// Converts frequency bounds of an event to left and right columns of object in sonogram matrix. + /// Converts the Hertz (frequency) bounds of an event to the frequency bin number. + /// The frequency bin is an index into the columns of the spectrogram data matrix. + /// Since the spectrogram data matrix is oriented with the origin at top left, + /// the low frequency bin will have a lower column index than the high freq bin. /// /// mel scale. /// lower freq bound. @@ -372,7 +367,7 @@ public void SetTimeAndFreqScales(double frameOffset, double frameDuration, doubl /// frequency scale. /// return bin index for lower freq bound. /// return bin index for upper freq bound. - public static void Freq2BinIDs(bool doMelscale, int minFreq, int maxFreq, int nyquist, double binWidth, out int leftCol, out int rightCol) + public static void ConvertHertzToFrequencyBin(bool doMelscale, int minFreq, int maxFreq, int nyquist, double binWidth, out int leftCol, out int rightCol) { if (doMelscale) { @@ -403,7 +398,7 @@ public static Oblong ConvertEvent2Oblong(AcousticEvent ae) var bottomRow = (int)Math.Round((ae.TimeStart + ae.EventDurationSeconds) / ae.FrameOffset); //Translate freq dimension = freq bins = matrix columns. - Freq2BinIDs(ae.IsMelscale, (int)ae.LowFrequencyHertz, (int)ae.HighFrequencyHertz, ae.FreqBinCount, ae.FreqBinWidth, out var leftCol, out var rightCol); + ConvertHertzToFrequencyBin(ae.IsMelscale, (int)ae.LowFrequencyHertz, (int)ae.HighFrequencyHertz, ae.FreqBinCount, ae.FreqBinWidth, out var leftCol, out var rightCol); return new Oblong(topRow, leftCol, bottomRow, rightCol); } @@ -431,21 +426,6 @@ public void SetScores(double score, double min, double max) } } - //public string WriteProperties() - //{ - // return " min-max=" + this.LowFrequencyHertz + "-" + this.HighFrequencyHertz + ", " + this.Oblong.ColumnLeft + "-" + this.Oblong.ColumnRight; - //} - - /// - /// Draws an event on the image. Uses the fields already set on the audio event to determine correct placement. - /// Fields requireed to be set include: `FramesPerSecond`, `FreqBinWidth`. - /// - //public void DrawEvent(Image sonogram) - // where T : unmanaged, IPixel - //{ - // this.DrawEvent(sonogram, this.FramesPerSecond, this.FreqBinWidth, sonogram.Height); - //} - /// /// Draws an event on the image. Allows for custom specification of variables. /// Drawing the event requires a time scale and a frequency scale. Hence the additional arguments. @@ -481,6 +461,8 @@ public void DrawEvent(Image imageToReturn, double framesPerSecond, double t2 = this.Oblong.RowBottom; } + // rectangle width = t2 - t1 + 1 + // rectangle height = y2 - y1 + 1 imageToReturn.Mutate(g => g.DrawRectangle(borderPen, t1, y1, t2 - t1 + 1, y2 - y1 + 1)); if (this.HitElements != null) @@ -501,31 +483,6 @@ public void DrawEvent(Image imageToReturn, double framesPerSecond, double }); } - /* - /// - /// Passed point is relative to top-left corner of the Acoustic Event. - /// Oblong needs to be set for this method to work. - /// - public void DrawPoint(Image bmp, Point point, Color colour) - { - if (bmp == null) - { - return; - } - - int maxFreqBin = (int)Math.Round(this.HighFrequencyHertz / this.FreqBinWidth); - int row = bmp.Height - maxFreqBin - 1 + point.Y; - int t1 = (int)Math.Round(this.TimeStart * this.FramesPerSecond); // temporal start of event - int col = t1 + point.X; - if (row >= bmp.Height) - { - row = bmp.Height - 1; - } - - bmp[col, row] = colour; - } - */ - //################################################################################################################# //FOLLOWING METHODS DEAL WITH THE OVERLAP OF EVENTS @@ -606,7 +563,7 @@ public static bool EventsOverlapInTime(AcousticEvent event1, AcousticEvent event /// Freq dimension = bins = matrix columns. Origin is top left - as per matrix in the sonogram class. /// Time dimension = frames = matrix rows. /// - public static List CombineOverlappingEvents(List events, TimeSpan segmentStartWrtRecording) + public static List CombineOverlappingEvents(List events, TimeSpan segmentStartOffset) { if (events.Count < 2) { @@ -619,7 +576,7 @@ public static List CombineOverlappingEvents(List e { if (EventsOverlapInTime(events[i], events[j]) && EventsOverlapInFrequency(events[i], events[j])) { - events[j] = AcousticEvent.MergeTwoEvents(events[i], events[j], segmentStartWrtRecording); + events[j] = AcousticEvent.MergeTwoEvents(events[i], events[j], segmentStartOffset); events.RemoveAt(i); break; } @@ -629,12 +586,12 @@ public static List CombineOverlappingEvents(List e return events; } - public static AcousticEvent MergeTwoEvents(AcousticEvent e1, AcousticEvent e2, TimeSpan segmentStartWrtRecording) + public static AcousticEvent MergeTwoEvents(AcousticEvent e1, AcousticEvent e2, TimeSpan segmentStartOffset) { //segmentStartOffset = TimeSpan.Zero; var minTime = Math.Min(e1.TimeStart, e2.TimeStart); var maxTime = Math.Max(e1.TimeEnd, e2.TimeEnd); - e1.SetEventPositionRelative(segmentStartWrtRecording, minTime, maxTime); + e1.SetEventPositionRelative(segmentStartOffset, minTime, maxTime); e1.LowFrequencyHertz = Math.Min(e1.LowFrequencyHertz, e2.LowFrequencyHertz); e1.HighFrequencyHertz = Math.Max(e1.HighFrequencyHertz, e2.HighFrequencyHertz); e1.Score = Math.Max(e1.Score, e2.Score); @@ -734,33 +691,6 @@ public static StringBuilder WriteEvents(List eventList, string st return sb; } - /* - public static void WriteEvents(List eventList, ref StringBuilder sb) - { - if (eventList.Count == 0) - { - string line = - $"# Event Name\t{"Start",8:f3}\t{"End",6:f3}\t{"MinF"}\t{"MaxF"}\t{"Score1":f2}\t{"Score2":f1}\t{"SourceFile"}"; - sb.AppendLine(line); - line = $"{"NoEvent"}\t{0.000,8:f3}\t{0.000,8:f3}\t{"N/A"}\t{"N/A"}\t{0.000:f2}\t{0.000:f1}\t{"N/A"}"; - sb.AppendLine(line); - } - else - { - AcousticEvent ae1 = eventList[0]; - string line = - $"# Event Name\t{"Start",8:f3}\t{"End",6:f3}\t{"MinF"}\t{"MaxF"}\t{"Score":f2}\t{ae1.Score2Name:f1}\t{"SourceFile"}"; - sb.AppendLine(line); - foreach (AcousticEvent ae in eventList) - { - line = - $"{ae.Name}\t{ae.TimeStart,8:f3}\t{ae.TimeEnd,8:f3}\t{ae.LowFrequencyHertz}\t{ae.HighFrequencyHertz}\t{ae.Score:f2}\t{ae.Score2:f1}\t{ae.FileName}"; - sb.AppendLine(line); - } - } - } - */ - //################################################################################################################# //METHODS FOR SEGMENTATION OF A FREQ BAND BASED ON ACOUSTIC ENERGY diff --git a/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramTools.cs b/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramTools.cs index 337b4a351..b0cd931f4 100644 --- a/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramTools.cs +++ b/src/AudioAnalysisTools/StandardSpectrograms/SpectrogramTools.cs @@ -636,14 +636,14 @@ public static Tuple HistogramOfSpectralPeaks(double[,] spectrogram public static double[,] ExtractFreqSubband(double[,] m, int minHz, int maxHz, bool doMelscale, int binCount, double binWidth) { - AcousticEvent.Freq2BinIDs(doMelscale, minHz, maxHz, binCount, binWidth, out var c1, out var c2); + AcousticEvent.ConvertHertzToFrequencyBin(doMelscale, minHz, maxHz, binCount, binWidth, out var c1, out var c2); return DataTools.Submatrix(m, 0, c1, m.GetLength(0) - 1, c2); } public static double[] ExtractModalNoiseSubband(double[] modalNoise, int minHz, int maxHz, bool doMelScale, int nyquist, double binWidth) { //extract subband modal noise profile - AcousticEvent.Freq2BinIDs(doMelScale, minHz, maxHz, nyquist, binWidth, out var c1, out var c2); + AcousticEvent.ConvertHertzToFrequencyBin(doMelScale, minHz, maxHz, nyquist, binWidth, out var c1, out var c2); int subbandCount = c2 - c1 + 1; var subband = new double[subbandCount]; for (int i = 0; i < subbandCount; i++) diff --git a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs index 18cd37773..e57ddae04 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs @@ -6,13 +6,7 @@ namespace Acoustics.Test.AudioAnalysisTools.EventStatistics { using System; using System.Collections.Generic; - using Acoustics.Shared; - using Acoustics.Tools.Wav; using global::AudioAnalysisTools; - using global::AudioAnalysisTools.DSP; - using global::AudioAnalysisTools.EventStatistics; - using global::AudioAnalysisTools.WavTools; - using global::TowseyLibrary; using Microsoft.VisualStudio.TestTools.UnitTesting; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; @@ -26,13 +20,8 @@ public void TestEventMerging() // make a list of events var events = new List(); - var segmentStartOffset = TimeSpan.Zero; double maxPossibleScore = 10.0; - var startTime1 = 1.0; - var duration1 = 5.0; - var minHz1 = 1000; - var maxHz1 = 8000; - var event1 = new AcousticEvent(segmentStartOffset, startTime1, duration1, minHz1, maxHz1) + var event1 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 1.0, eventDuration: 5.0, minFreq: 1000, maxFreq: 8000) { Name = "Event1", Score = 1.0, @@ -41,11 +30,7 @@ public void TestEventMerging() events.Add(event1); - var startTime2 = 4.5; - var duration2 = 2.0; - var minHz2 = 1500; - var maxHz2 = 6000; - var event2 = new AcousticEvent(segmentStartOffset, startTime2, duration2, minHz2, maxHz2) + var event2 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 4.5, eventDuration: 2.0, minFreq: 1500, maxFreq: 6000) { Name = "Event2", Score = 5.0, @@ -53,11 +38,7 @@ public void TestEventMerging() }; events.Add(event2); - var startTime3 = 7.0; - var duration3 = 2.0; - var minHz3 = 1000; - var maxHz3 = 8000; - var event3 = new AcousticEvent(segmentStartOffset, startTime3, duration3, minHz3, maxHz3) + var event3 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 7.0, eventDuration: 2.0, minFreq: 1000, maxFreq: 8000) { Name = "Event3", Score = 9.0, @@ -66,7 +47,7 @@ public void TestEventMerging() events.Add(event3); // combine adjacent acoustic events - events = AcousticEvent.CombineOverlappingEvents(events, segmentStartOffset); + events = AcousticEvent.CombineOverlappingEvents(events: events, segmentStartOffset: TimeSpan.Zero); Assert.AreEqual(2, events.Count); Assert.AreEqual(1.0, events[0].EventStartSeconds, 1E-4); @@ -99,15 +80,10 @@ public void TestSonogramWithEventsOverlay() // make a list of events var framesPerSecond = 10.0; var freqBinWidth = 43.0664; - var segmentStartOffset = TimeSpan.Zero; - var minHz = 1000; - var maxHz = 8000; double maxPossibleScore = 10.0; var events = new List(); - var startTime1 = 1.0; - var duration1 = 5.0; - var event1 = new AcousticEvent(segmentStartOffset, startTime1, duration1, minHz, maxHz) + var event1 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 1.0, eventDuration: 5.0, minFreq: 1000, maxFreq: 8000) { Score = 10.0, Name = "Event1", @@ -115,9 +91,7 @@ public void TestSonogramWithEventsOverlay() }; events.Add(event1); - var startTime2 = 7.0; - var duration2 = 2.0; - var event2 = new AcousticEvent(segmentStartOffset, startTime2, duration2, minHz, maxHz) + var event2 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 7.0, eventDuration: 2.0, minFreq: 1000, maxFreq: 8000) { Score = 1.0, Name = "Event2", @@ -134,7 +108,6 @@ public void TestSonogramWithEventsOverlay() ev.DrawEvent(substituteSonogram, framesPerSecond, freqBinWidth, height); } - //substituteSonogram.Save("C:\\temp\\image.png"); var redPixel1 = new Argb32(110, 10, 30); var expectedRed1 = new Color(redPixel1); var redPixel2 = new Argb32(124, 11, 34); From 1d4e524ca2234fe97d7fe156e8abc5f1f24383a9 Mon Sep 17 00:00:00 2001 From: towsey Date: Fri, 20 Mar 2020 09:25:43 +1000 Subject: [PATCH 09/12] Fix anti-aliasing when drawing event Issue #300 Fix anti-aliasing problem and then fix unit tests for the drawing of events. --- src/AudioAnalysisTools/AcousticEvent.cs | 11 ++++---- .../AcousticEvents/EventTests.cs | 26 +++++++++---------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/AudioAnalysisTools/AcousticEvent.cs b/src/AudioAnalysisTools/AcousticEvent.cs index f1c59667e..02714912b 100644 --- a/src/AudioAnalysisTools/AcousticEvent.cs +++ b/src/AudioAnalysisTools/AcousticEvent.cs @@ -24,6 +24,7 @@ namespace AudioAnalysisTools using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; using TowseyLibrary; + using static Acoustics.Shared.ImageSharp.Drawing; public class AcousticEvent : EventBase { @@ -461,10 +462,7 @@ public void DrawEvent(Image imageToReturn, double framesPerSecond, double t2 = this.Oblong.RowBottom; } - // rectangle width = t2 - t1 + 1 - // rectangle height = y2 - y1 + 1 - imageToReturn.Mutate(g => g.DrawRectangle(borderPen, t1, y1, t2 - t1 + 1, y2 - y1 + 1)); - + imageToReturn.Mutate(g => g.NoAA().DrawRectangle(borderPen, t1, y1, t2, y2)); if (this.HitElements != null) { foreach (var hitElement in this.HitElements) @@ -478,8 +476,9 @@ public void DrawEvent(Image imageToReturn, double framesPerSecond, double int scoreHt = (int)Math.Round(eventHeight * this.ScoreNormalised); imageToReturn.Mutate(g => { - g.DrawLine(scorePen, t1, y2 - scoreHt, t1, y2); - g.DrawTextSafe(this.Name, Drawing.Tahoma6, Color.Black, new PointF(t1, y1 - 4)); + g.NoAA().DrawLine(scorePen, t1, y2 - scoreHt, t1, y2 + 1); + //g.DrawTextSafe(this.Name, Drawing.Tahoma6, Color.Black, new PointF(t1, y1 - 5)); + g.DrawTextSafe(this.Name, Drawing.Arial8, Color.Black, new PointF(t1, y1 - 5)); }); } diff --git a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs index e57ddae04..fcbad433f 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs @@ -108,23 +108,21 @@ public void TestSonogramWithEventsOverlay() ev.DrawEvent(substituteSonogram, framesPerSecond, freqBinWidth, height); } - var redPixel1 = new Argb32(110, 10, 30); - var expectedRed1 = new Color(redPixel1); - var redPixel2 = new Argb32(124, 11, 34); - var expectedRed2 = new Color(redPixel2); - var greenPixel = new Argb32(55, 133, 15); + substituteSonogram.Save("C:\\temp\\image.png"); + + var redPixel = new Argb32(220, 20, 60); + var expectedRed = new Color(redPixel); + var greenPixel = new Argb32(0, 255, 0); var expectedGreen = new Color(greenPixel); - //var actualColor = substituteSonogram[0, height - 1]; - Assert.AreEqual(expectedRed1, substituteSonogram[61, 119]); - Assert.AreEqual(expectedRed1, substituteSonogram[70, 122]); - Assert.AreEqual(expectedRed1, substituteSonogram[91, 181]); - Assert.AreEqual(expectedRed2, substituteSonogram[36, 233]); - Assert.AreEqual(expectedRed1, substituteSonogram[56, 69]); + Assert.AreEqual(expectedRed, substituteSonogram[60, 119]); + Assert.AreEqual(expectedRed, substituteSonogram[70, 122]); + Assert.AreEqual(expectedRed, substituteSonogram[90, 181]); + Assert.AreEqual(expectedRed, substituteSonogram[36, 232]); + Assert.AreEqual(expectedRed, substituteSonogram[56, 69]); - //actualColor = substituteSonogram[9, 72]; - Assert.AreEqual(expectedGreen, substituteSonogram[9, 72]); - Assert.AreEqual(expectedGreen, substituteSonogram[69, 217]); + Assert.AreEqual(expectedGreen, substituteSonogram[10, 72]); + Assert.AreEqual(expectedGreen, substituteSonogram[70, 217]); } } } From 04f9b044e4c0e281a6a681580bdd46c223be7037 Mon Sep 17 00:00:00 2001 From: towsey Date: Fri, 20 Mar 2020 14:35:41 +1000 Subject: [PATCH 10/12] Fixed test of drawing events Issue #300 Fixed test of drawing events on spectrograms. Test now uses the new image comparison facility. Set the event name to empty string in order to prevent test failing due to writing of poorly anti-aliased text. Returned font type to the original. Included new expected events image in Fixtures. --- src/AudioAnalysisTools/AcousticEvent.cs | 3 +- .../AcousticEvents/EventTests.cs | 35 ++++++++++--------- .../EventTests_SuperimposeEventsOnImage.png | 3 ++ 3 files changed, 23 insertions(+), 18 deletions(-) create mode 100644 tests/Fixtures/EventTests_SuperimposeEventsOnImage.png diff --git a/src/AudioAnalysisTools/AcousticEvent.cs b/src/AudioAnalysisTools/AcousticEvent.cs index 02714912b..8700effbd 100644 --- a/src/AudioAnalysisTools/AcousticEvent.cs +++ b/src/AudioAnalysisTools/AcousticEvent.cs @@ -477,8 +477,7 @@ public void DrawEvent(Image imageToReturn, double framesPerSecond, double imageToReturn.Mutate(g => { g.NoAA().DrawLine(scorePen, t1, y2 - scoreHt, t1, y2 + 1); - //g.DrawTextSafe(this.Name, Drawing.Tahoma6, Color.Black, new PointF(t1, y1 - 5)); - g.DrawTextSafe(this.Name, Drawing.Arial8, Color.Black, new PointF(t1, y1 - 5)); + g.DrawTextSafe(this.Name, Drawing.Tahoma6, Color.Black, new PointF(t1, y1 - 4)); }); } diff --git a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs index fcbad433f..a1be7e530 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs @@ -6,13 +6,14 @@ namespace Acoustics.Test.AudioAnalysisTools.EventStatistics { using System; using System.Collections.Generic; + using Acoustics.Test.TestHelpers; using global::AudioAnalysisTools; using Microsoft.VisualStudio.TestTools.UnitTesting; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; [TestClass] - public class EventTests + public class EventTests : GeneratedImageTest { [TestMethod] public void TestEventMerging() @@ -103,26 +104,28 @@ public void TestSonogramWithEventsOverlay() // set colour for the events foreach (AcousticEvent ev in events) { + // do not set an event name because text is not drawing anit-aliased at present time. + ev.Name = string.Empty; ev.BorderColour = AcousticEvent.DefaultBorderColor; ev.ScoreColour = AcousticEvent.DefaultScoreColor; ev.DrawEvent(substituteSonogram, framesPerSecond, freqBinWidth, height); } - substituteSonogram.Save("C:\\temp\\image.png"); - - var redPixel = new Argb32(220, 20, 60); - var expectedRed = new Color(redPixel); - var greenPixel = new Argb32(0, 255, 0); - var expectedGreen = new Color(greenPixel); - - Assert.AreEqual(expectedRed, substituteSonogram[60, 119]); - Assert.AreEqual(expectedRed, substituteSonogram[70, 122]); - Assert.AreEqual(expectedRed, substituteSonogram[90, 181]); - Assert.AreEqual(expectedRed, substituteSonogram[36, 232]); - Assert.AreEqual(expectedRed, substituteSonogram[56, 69]); - - Assert.AreEqual(expectedGreen, substituteSonogram[10, 72]); - Assert.AreEqual(expectedGreen, substituteSonogram[70, 217]); + substituteSonogram.Save("C:\\temp\\EventTests_SuperimposeEventsOnImage.png"); + + this.Actual = substituteSonogram; + /* + var pattern = @" +⬇150 +E100R50 +48×E100RE48R +E100R50 +"; + this.Expected = TestImage.Create(width: 100, height: 100, Color.Black, pattern); + */ + + this.Expected = Image.Load(PathHelper.ResolveAssetPath("EventTests_SuperimposeEventsOnImage.png")); + this.AssertImagesEqual(); } } } diff --git a/tests/Fixtures/EventTests_SuperimposeEventsOnImage.png b/tests/Fixtures/EventTests_SuperimposeEventsOnImage.png new file mode 100644 index 000000000..b45eba20e --- /dev/null +++ b/tests/Fixtures/EventTests_SuperimposeEventsOnImage.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fde5d7e3427a88eae8774bebb0b986b301edee6808d37bc41a3479d7320cc6e +size 699 From e8d593de715710f2d97fbb2e6dc844a7f1fca4c1 Mon Sep 17 00:00:00 2001 From: Anthony Truskinger Date: Fri, 20 Mar 2020 14:56:13 +1000 Subject: [PATCH 11/12] Cleaned up event test --- .../AcousticEvents/EventTests.cs | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs index a1be7e530..06bf11ae1 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs @@ -6,6 +6,7 @@ namespace Acoustics.Test.AudioAnalysisTools.EventStatistics { using System; using System.Collections.Generic; + using Acoustics.Shared.ImageSharp; using Acoustics.Test.TestHelpers; using global::AudioAnalysisTools; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -71,12 +72,7 @@ public void TestEventMerging() public void TestSonogramWithEventsOverlay() { // make a substitute sonogram image - int width = 100; - int height = 256; - var substituteSonogram = new Image(width, height); - - //substituteSonogram.Mutate(x => x.Pad(width, height, Color.Gray)); - // image.Mutate(x => x.Resize(image.Width / 2, image.Height / 2)); + var substituteSonogram = Drawing.NewImage(100, 256, Color.Black); // make a list of events var framesPerSecond = 10.0; @@ -101,29 +97,19 @@ public void TestSonogramWithEventsOverlay() events.Add(event2); // now add events into the spectrogram image. - // set colour for the events + // set color for the events foreach (AcousticEvent ev in events) { - // do not set an event name because text is not drawing anit-aliased at present time. + // because we are testing placement of box not text. ev.Name = string.Empty; ev.BorderColour = AcousticEvent.DefaultBorderColor; ev.ScoreColour = AcousticEvent.DefaultScoreColor; - ev.DrawEvent(substituteSonogram, framesPerSecond, freqBinWidth, height); + ev.DrawEvent(substituteSonogram, framesPerSecond, freqBinWidth, 256); } - substituteSonogram.Save("C:\\temp\\EventTests_SuperimposeEventsOnImage.png"); - this.Actual = substituteSonogram; - /* - var pattern = @" -⬇150 -E100R50 -48×E100RE48R -E100R50 -"; - this.Expected = TestImage.Create(width: 100, height: 100, Color.Black, pattern); - */ + // BUG: this asset is faulty. See https://github.com/QutEcoacoustics/audio-analysis/issues/300#issuecomment-601537263 this.Expected = Image.Load(PathHelper.ResolveAssetPath("EventTests_SuperimposeEventsOnImage.png")); this.AssertImagesEqual(); } From 7804814132b9c7fc64806ff6ad5eab23b4105dd1 Mon Sep 17 00:00:00 2001 From: Anthony Truskinger Date: Fri, 20 Mar 2020 15:34:33 +1000 Subject: [PATCH 12/12] Cleaned up unconventional code Adressing comments from #301. @towsey was unwilling to address all comments from code review. This commit actually addresses the feedback. Removes abbreviation `wrt` wherever used in recent changes and instead adds better documentation for parameters. Corrects mistaken comments and assumptions about TimeStart and TimeEnd on AcousticEvent and marks them as obsolete to prevent further misunderstanding. Move EventStatisticsCalculateTests.cs back to its correct folder and namespace. Also renames EventTests.cs to AcousticEventTests.cs (as per our convention) and moves the file to correct namespace and folder. --- src/AnalysisBase/ResultBases/EventBase.cs | 6 +- .../AnalyseLongRecording.cs | 3 +- .../EventStatistics/EventStatisticsEntry.cs | 2 +- src/AnalysisPrograms/OscillationRecogniser.cs | 10 +-- .../Recognizers/GenericRecognizer.cs | 11 +-- .../Recognizers/LewiniaPectoralis.cs | 10 --- .../Recognizers/LitoriaBicolor.cs | 10 --- .../Recognizers/LitoriaWatjulumensis.cs | 10 --- src/AudioAnalysisTools/AcousticEvent.cs | 84 +++++++------------ .../EventTests.cs => AcousticEventTests.cs} | 6 +- .../EventStatisticsCalculateTests.cs | 2 +- 11 files changed, 44 insertions(+), 110 deletions(-) rename tests/Acoustics.Test/AudioAnalysisTools/{AcousticEvents/EventTests.cs => AcousticEventTests.cs} (96%) rename tests/Acoustics.Test/AudioAnalysisTools/{AcousticEvents => EventStatistics}/EventStatisticsCalculateTests.cs (98%) diff --git a/src/AnalysisBase/ResultBases/EventBase.cs b/src/AnalysisBase/ResultBases/EventBase.cs index 411c76c21..80959cc82 100644 --- a/src/AnalysisBase/ResultBases/EventBase.cs +++ b/src/AnalysisBase/ResultBases/EventBase.cs @@ -61,7 +61,11 @@ public virtual double EventStartSeconds public virtual double? LowFrequencyHertz { get; protected set; } /// - /// Sets both the Segment start and the Event start wrt to recording. + /// Sets both the Segment start and the Event start. + /// is measured relative to the start of the recording. + /// is measured relative to the start of the segment. + /// This method sets both and which + /// are both measured relative to the start of the recording. /// protected void SetEventStartRelative(TimeSpan segmentStart, double eventStartSegmentRelative) { diff --git a/src/AnalysisPrograms/AnalyseLongRecordings/AnalyseLongRecording.cs b/src/AnalysisPrograms/AnalyseLongRecordings/AnalyseLongRecording.cs index 1cb00f4f5..9643f87e3 100644 --- a/src/AnalysisPrograms/AnalyseLongRecordings/AnalyseLongRecording.cs +++ b/src/AnalysisPrograms/AnalyseLongRecordings/AnalyseLongRecording.cs @@ -73,7 +73,8 @@ public static void Execute(Arguments arguments) { Log.Warn($"Config file {configFile.FullName} not found... attempting to resolve config file"); - // we use .ToString() here to get the original input string - Using fullname always produces an absolute path wrt to pwd... we don't want to prematurely make asusmptions: + // we use .ToString() here to get the original input string. + // Using fullname always produces an absolute path relative to pwd... we don't want to prematurely make assumptions: // e.g. We require a missing absolute path to fail... that wouldn't work with .Name // e.g. We require a relative path to try and resolve, using .FullName would fail the first absolute check inside ResolveConfigFile configFile = ConfigFile.Resolve(configFile.ToString(), Directory.GetCurrentDirectory().ToDirectoryInfo()); diff --git a/src/AnalysisPrograms/EventStatistics/EventStatisticsEntry.cs b/src/AnalysisPrograms/EventStatistics/EventStatisticsEntry.cs index eb5fab25b..bea3e007a 100644 --- a/src/AnalysisPrograms/EventStatistics/EventStatisticsEntry.cs +++ b/src/AnalysisPrograms/EventStatistics/EventStatisticsEntry.cs @@ -49,7 +49,7 @@ public static async Task ExecuteAsync(Arguments arguments) Log.Warn($"Config file {config.FullName} not found... attempting to resolve config file"); // we use the original input string - Using FileInfo fullname always produces an - // absolute path wrt to pwd... we don't want to prematurely make assumptions: + // absolute path relative to pwd... we don't want to prematurely make assumptions: // e.g. We require a missing absolute path to fail... that wouldn't work with .Name // e.g. We require a relative path to try and resolve, using .FullName would fail the first absolute // check inside ResolveConfigFile diff --git a/src/AnalysisPrograms/OscillationRecogniser.cs b/src/AnalysisPrograms/OscillationRecogniser.cs index 4cfb76af1..71e932483 100644 --- a/src/AnalysisPrograms/OscillationRecogniser.cs +++ b/src/AnalysisPrograms/OscillationRecogniser.cs @@ -12,6 +12,7 @@ namespace AnalysisPrograms using System.Linq; using System.Text; using System.Threading.Tasks; + using Acoustics.Shared.Csv; using AudioAnalysisTools; using AudioAnalysisTools.StandardSpectrograms; using AudioAnalysisTools.WavTools; @@ -126,15 +127,10 @@ public static void Execute(Arguments arguments) pcHIF = 100 * hifCount / sonogram.FrameCount; } - //write event count to results file. - double sigDuration = sonogram.Duration.TotalSeconds; + // write event count to results file. string fname = recordingFile.BaseName(); - int count = predictedEvents.Count; - //string str = String.Format("#RecordingName\tDuration(sec)\t#Ev\tCompT(ms)\t%hiFrames\n{0}\t{1}\t{2}\t{3}\t{4}\n", fname, sigDuration, count, analysisDuration.TotalMilliseconds, pcHIF); - string str = string.Format("{0}\t{1}\t{2}\t{3}\t{4}", fname, sigDuration, count, analysisDuration.TotalMilliseconds, pcHIF); - StringBuilder sb = AcousticEvent.WriteEvents(predictedEvents, str); - FileTools.WriteTextFile(opPath, sb.ToString()); + Csv.WriteToCsv(opPath.ToFileInfo(), predictedEvents); //draw images of sonograms string imagePath = outputDir + fname + ".png"; diff --git a/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs b/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs index d5c77f897..21aa9a855 100644 --- a/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs +++ b/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs @@ -325,18 +325,9 @@ private static Plot PreparePlot(double[] array, string title, double threshold) /// static void SaveDebugSpectrogram(RecognizerResults results, Config genericConfig, DirectoryInfo outputDirectory, string baseName) { - //var image1 = results.Sonogram.GetImage(false, true, false); - //image1.Save(Path.Combine("C:\\temp\\test1.profile.png")); - - //var image2 = results.Sonogram.GetImageFullyAnnotated("Test"); - //image2.Save(Path.Combine("C:\\temp\\test2.profile.png")); - var image3 = SpectrogramTools.GetSonogramPlusCharts(results.Sonogram, results.Events, results.Plots, null); - //image3.Save(Path.Combine(outputDirectory.FullName, baseName + ".profile.png")); - //image3.Save(Path.Combine("C:\\temp", baseName + ".profile.png")); - - //sonogram.GetImageFullyAnnotated("test").Save("C:\\temp\\test.png"); + image3.Save(Path.Combine(outputDirectory.FullName, baseName + ".profile.png")); } /// /> diff --git a/src/AnalysisPrograms/Recognizers/LewiniaPectoralis.cs b/src/AnalysisPrograms/Recognizers/LewiniaPectoralis.cs index b994cccce..499b1f654 100644 --- a/src/AnalysisPrograms/Recognizers/LewiniaPectoralis.cs +++ b/src/AnalysisPrograms/Recognizers/LewiniaPectoralis.cs @@ -193,16 +193,6 @@ public override RecognizerResults Recognize( prunedEvents.Add(ae); } - /* - // do a recognizer TEST. DELETE ONE DAY! - if (false) - { - var testDir = new DirectoryInfo(outputDirectory.Parent.Parent.FullName); - TestTools.RecognizerScoresTest(recording.BaseName, testDir, recognizerConfig.AnalysisName, scoreArray); - AcousticEvent.TestToCompareEvents(recording.BaseName, testDir, recognizerConfig.AnalysisName, predictedEvents); - } - */ - // increase very low scores for (int j = 0; j < scoreArray.Length; j++) { diff --git a/src/AnalysisPrograms/Recognizers/LitoriaBicolor.cs b/src/AnalysisPrograms/Recognizers/LitoriaBicolor.cs index 2d94aa632..cf14962cb 100644 --- a/src/AnalysisPrograms/Recognizers/LitoriaBicolor.cs +++ b/src/AnalysisPrograms/Recognizers/LitoriaBicolor.cs @@ -152,16 +152,6 @@ public override RecognizerResults Recognize(AudioRecording recording, Config con ae.SegmentDurationSeconds = recordingDuration.TotalSeconds; } - /* - // do a RECOGNIZER TEST. DELETE ONE DAY! - if (false) - { - var testDir = new DirectoryInfo(outputDirectory.Parent.Parent.FullName); - TestTools.RecognizerScoresTest(recording.BaseName, testDir, recognizerConfig.AnalysisName, scoreArray); - AcousticEvent.TestToCompareEvents(recording.BaseName, testDir, recognizerConfig.AnalysisName, predictedEvents); - } - */ - var plot = new Plot(this.DisplayName, scoreArray, recognizerConfig.EventThreshold); return new RecognizerResults() { diff --git a/src/AnalysisPrograms/Recognizers/LitoriaWatjulumensis.cs b/src/AnalysisPrograms/Recognizers/LitoriaWatjulumensis.cs index 3f1c2ee47..501aa406c 100644 --- a/src/AnalysisPrograms/Recognizers/LitoriaWatjulumensis.cs +++ b/src/AnalysisPrograms/Recognizers/LitoriaWatjulumensis.cs @@ -157,16 +157,6 @@ public override RecognizerResults Recognize(AudioRecording recording, Config con ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds; } - /* - // do a recognizer TEST. DELETE ONE DAY! - if (false) - { - var testDir = new DirectoryInfo(outputDirectory.Parent.Parent.FullName); - TestTools.RecognizerScoresTest(recording.BaseName, testDir, recognizerConfig.AnalysisName, scoreArray); - AcousticEvent.TestToCompareEvents(recording.BaseName, testDir, recognizerConfig.AnalysisName, predictedEvents); - } - */ - var plot = new Plot(this.DisplayName, scoreArray, recognizerConfig.EventThreshold); return new RecognizerResults() { diff --git a/src/AudioAnalysisTools/AcousticEvent.cs b/src/AudioAnalysisTools/AcousticEvent.cs index 8700effbd..b28e06b17 100644 --- a/src/AudioAnalysisTools/AcousticEvent.cs +++ b/src/AudioAnalysisTools/AcousticEvent.cs @@ -71,33 +71,41 @@ public AcousticEventClassMap() /// /// Gets the time offset from start of current segment to start of event in seconds. - /// NOTE: AcousticEvents do not have a notion of time offset wrt start of recording ; - only to start of current recording segment. - /// Proxied to EventBase.EventStartSeconds. + /// Proxied to EventBase.EventStartSeconds. /// /// + /// + /// NOTE: is relative to the start of a segment. This notion is obsolete! + /// Events must always be stored relative to start of the recording. + /// /// Note: converted to private setter so we can control how this is set. Recommend using /// after event instantiation to modify bounds. /// + [Obsolete("Bounds relative to the segment are inconsistent with our rules for always measuring from the start of the recording.")] public double TimeStart { get; private set; } /// /// Gets the time offset (in seconds) from start of current segment to end of the event. - /// Written into the csv file under column "EventEndSeconds" /// This field is NOT in EventBase. EventBase only requires TimeStart because it is designed to also accomodate points. /// /// + /// + /// NOTE: is relative to the start of a segment. This notion is obsolete! + /// Events must always be stored relative to start of the recording. + /// /// Note: converted to private setter so we can control how this is set. /// Recommend using after event instantiation to modify bounds. /// + [Obsolete("Bounds relative to the segment are inconsistent with our rules for always measuring from the start of the recording.")] public double TimeEnd { get; private set; } /// - /// Gets the end time of an event WRT the recording/file start. + /// Gets the end time of an event relative to the recording start. /// public double EventEndSeconds => this.TimeEnd + this.SegmentStartSeconds; /// - /// Gets the start time of an event WRT the recording/file start. + /// Gets the start time of an event relative to the recording start. /// public override double EventStartSeconds => this.TimeStart + this.SegmentStartSeconds; @@ -172,13 +180,14 @@ public AcousticEventClassMap() /// Gets or sets second score if required. /// + [Obsolete("We should use another type of Event class to represent this concept")] public double Score2 { get; set; } - /// - /// Gets or sets a list of points that can be used to identifies features in spectrogram relative to the Event. - /// i.e. Points can be outside of events and can have negative values. - /// Point location is relative to the top left corner of the event. - /// + ///// + ///// Gets or sets a list of points that can be used to identifies features in spectrogram relative to the Event. + ///// i.e. Points can be outside of events and can have negative values. + ///// Point location is relative to the top left corner of the event. + ///// //public List Points { get; set; } /// @@ -222,8 +231,8 @@ public AcousticEvent() /// Initializes a new instance of the class. /// This constructor requires the minimum information to establish the temporal and frequency bounds of an acoustic event. /// - /// The start of the current segment with respect to start of recording/file. - /// event start with respect to start of segment. + /// The start of the current segment relative to start of recording. + /// event start with respect to start of segment. /// event end with respect to start of segment. /// Lower frequency bound of event. /// Upper frequency bound of event. @@ -652,43 +661,6 @@ public static double EventFractionalOverlap(AcousticEvent event1, AcousticEvent } */ - //################################################################################################################# - //METHOD TO WRITE ACOUSTIC EVETNS TO FILE - - /// - /// Writes acousitc event data to an excel spread sheet. - /// - public static StringBuilder WriteEvents(List eventList, string str) - { - StringBuilder sb = new StringBuilder(); - if (eventList.Count == 0) - { - string line = string.Format( - str + "\t{0}\t{1,8:f3}\t{2,8:f3}\t{3}\t{4}\t{5:f2}\t{6:f1}\t{7}", "NoEvent", 0.000, 0.000, "N/A", "N/A", 0.000, 0.000, "N/A"); - sb.AppendLine(line); - } - else - { - foreach (AcousticEvent ae in eventList) - { - string line = string.Format( - str + "\t{0}\t{1,8:f3}\t{2,8:f3}\t{3}\t{4}\t{5:f2}\t{6:f1}\t{7}", - ae.Name, - ae.TimeStart, - ae.TimeEnd, - ae.LowFrequencyHertz, - ae.HighFrequencyHertz, - ae.Score, - ae.Score2, - ae.FileName); - - sb.AppendLine(line); - } - } - - return sb; - } - //################################################################################################################# //METHODS FOR SEGMENTATION OF A FREQ BAND BASED ON ACOUSTIC ENERGY @@ -737,7 +709,7 @@ public static Tuple, double, double, double, double[]> GetSe /// Noise reduction is done first. /// /// the full spectrogram. - /// Start of current segment wrt recording start. + /// Start of current segment relative to the recording start. /// Bottom of the required frequency band. /// Top of the required frequency band. /// To smooth the amplitude array. @@ -1239,7 +1211,7 @@ public static List GetEventsAroundMaxima( /// threshold. /// duration of event must exceed this to count as an event. /// duration of event must be less than this to count as an event. - /// offset. + /// offset. /// a list of acoustic events. public static List ConvertScoreArray2Events( double[] scores, @@ -1250,14 +1222,14 @@ public static List ConvertScoreArray2Events( double scoreThreshold, double minDuration, double maxDuration, - TimeSpan segmentStartWrtRecording) + TimeSpan segmentStart) { int count = scores.Length; var events = new List(); double maxPossibleScore = 5 * scoreThreshold; // used to calculate a normalised score between 0 - 1.0 bool isHit = false; double frameOffset = 1 / framesPerSec; - double startTimeWrtSegment = 0.0; // units = seconds + double startTimeInSegment = 0.0; // units = seconds int startFrame = 0; // pass over all frames @@ -1267,7 +1239,7 @@ public static List ConvertScoreArray2Events( { //start of an event isHit = true; - startTimeWrtSegment = i * frameOffset; + startTimeInSegment = i * frameOffset; startFrame = i; } else // check for the end of an event @@ -1276,7 +1248,7 @@ public static List ConvertScoreArray2Events( // this is end of an event, so initialise it isHit = false; double endTime = i * frameOffset; - double duration = endTime - startTimeWrtSegment; + double duration = endTime - startTimeInSegment; // if (duration < minDuration) continue; //skip events with duration shorter than threshold if (duration < minDuration || duration > maxDuration) @@ -1295,7 +1267,7 @@ public static List ConvertScoreArray2Events( av /= i - startFrame + 1; // Initialize the event. - AcousticEvent ev = new AcousticEvent(segmentStartWrtRecording, startTimeWrtSegment, duration, minHz, maxHz); + AcousticEvent ev = new AcousticEvent(segmentStart, startTimeInSegment, duration, minHz, maxHz); ev.SetTimeAndFreqScales(framesPerSec, freqBinWidth); ev.Score = av; diff --git a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEventTests.cs similarity index 96% rename from tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs rename to tests/Acoustics.Test/AudioAnalysisTools/AcousticEventTests.cs index 06bf11ae1..ed2c2c573 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventTests.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/AcousticEventTests.cs @@ -1,8 +1,8 @@ -// +// // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // -namespace Acoustics.Test.AudioAnalysisTools.EventStatistics +namespace Acoustics.Test.AudioAnalysisTools { using System; using System.Collections.Generic; @@ -14,7 +14,7 @@ namespace Acoustics.Test.AudioAnalysisTools.EventStatistics using SixLabors.ImageSharp.PixelFormats; [TestClass] - public class EventTests : GeneratedImageTest + public class AcousticEventTests : GeneratedImageTest { [TestMethod] public void TestEventMerging() diff --git a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventStatisticsCalculateTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/EventStatistics/EventStatisticsCalculateTests.cs similarity index 98% rename from tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventStatisticsCalculateTests.cs rename to tests/Acoustics.Test/AudioAnalysisTools/EventStatistics/EventStatisticsCalculateTests.cs index eb912ab9b..b0cad3c33 100644 --- a/tests/Acoustics.Test/AudioAnalysisTools/AcousticEvents/EventStatisticsCalculateTests.cs +++ b/tests/Acoustics.Test/AudioAnalysisTools/EventStatistics/EventStatisticsCalculateTests.cs @@ -2,7 +2,7 @@ // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // -namespace Acoustics.Test.AudioAnalysisTools.AcousticEvents +namespace Acoustics.Test.AudioAnalysisTools.EventStatistics { using System; using Acoustics.Shared;