Merge pull request #214 from QutEcoacoustics/unsupervised-features-2

Unsupervised features 2
QutEcoacoustics · Feb 4, 2019 · add0de4 · add0de4
2 parents 36a8583 + e47eeff
commit add0de4
Show file tree

Hide file tree

Showing 57 changed files with 3,192 additions and 298 deletions.
diff --git a/src/Acoustics.Shared/Extensions/ArrayExtensions.cs b/src/Acoustics.Shared/Extensions/ArrayExtensions.cs
@@ -165,5 +165,22 @@ public static double GetMaxValue(this double[] data)
 
             return max;
         }
+
+        /// <summary>
+        /// retrieving the min value of a vector
+        /// </summary>
+        public static double GetMinValue(this double[] data)
+        {
+            double min = data[0];
+            for (int i = 1; i < data.Length; i++)
+            {
+                if (data[i] < min)
+                {
+                    min = data[i];
+                }
+            }
+
+            return min;
+        }
     }
 }
diff --git a/src/Acoustics.Shared/Extensions/DoubleSquareArrayExtensions.cs b/src/Acoustics.Shared/Extensions/DoubleSquareArrayExtensions.cs
@@ -221,18 +221,18 @@ public static void MinMax(this double[,] data, out double min, out double max)
         /// <summary>
         /// returns an empty matrix with the same number of rows and columns of the input matrix.
         /// </summary>
-        public static double[,] EmptyCopy(this double[,] matrix)
+        public static T[,] EmptyCopy<T>(this T[,] matrix)
         {
-            return new double[matrix.GetLength(0), matrix.GetLength(1)];
+            return new T[matrix.GetLength(0), matrix.GetLength(1)];
         }
 
         /// <summary>
         /// retrieving a full column of a matrix
         /// columnIndex is the column we want to access
         /// </summary>
-        public static double[] GetColumn(this double[,] matrix, int columnIndex)
+        public static T[] GetColumn<T>(this T[,] matrix, int columnIndex)
         {
-            double[] column = new double[matrix.GetLength(0)];
+            T[] column = new T[matrix.GetLength(0)];
             for (int row = 0; row < matrix.GetLength(0); row++)
             {
                 column[row] = matrix[row, columnIndex];
@@ -245,9 +245,9 @@ public static double[] GetColumn(this double[,] matrix, int columnIndex)
         /// retrieving a full row of a matrix
         /// rowIndex is the row we want to access
         /// </summary>
-        public static double[] GetRow(this double[,] matrix, int rowIndex)
+        public static T[] GetRow<T>(this T[,] matrix, int rowIndex)
         {
-            double[] row = new double[matrix.GetLength(1)];
+            T[] row = new T[matrix.GetLength(1)];
             for (int column = 0; column < matrix.GetLength(1); column++)
             {
                 row[column] = matrix[rowIndex, column];
@@ -270,7 +270,7 @@ public enum MergingDirection
         /// adding a 2D-array to another 2D-array either by "column" or by "row"
         /// </summary>
 
-        public static void AddToArray(double[,] result, double[,] array, MergingDirection mergingDirection, int start = 0)
+        public static void AddToArray<T>(this T[,] result, T[,] array, MergingDirection mergingDirection, int start = 0)
         {
             for (int i = 0; i < array.GetLength(0); i++)
             {

diff --git a/src/AnalysisConfigFiles/FeatureLearningConfig.yml b/src/AnalysisConfigFiles/FeatureLearningConfig.yml
@@ -0,0 +1,53 @@
+---
+# Summary: Generates Clustering Features
+#
+# This analysis outputs:
+#     1. an image of the clusters' centroids
+#     2. a csv file that contains the vectors of the clusters' centroids
+#     3. a csv file that contains clusters' id and size
+#     4. a csv file that contains feature vectors
+#
+# The feature vectors can be used in training the machine learning models.
+
+# The directory for the output of parallel job running on MahnooshSandpit
+# OutputDirectory: "D:\Mahnoosh\Liz\ParallelJobs\"
+
+# The properties to generate Mel scale
+FrequencyScaleType: Mel
+# HertzInterval: 1000
+FrameSize: 1024
+FinalBinCount: 128
+
+# The default values for minFreqBin and maxFreqBin are 1 and FinalBinCount
+# For any other arbitrary frequency bin bounds, these two parameters need to be manually set.
+MinFreqBin: 24
+MaxFreqBin: 82
+
+# The number of frequency band for feature generation process
+numFreqBand: 1
+
+# The width and height of the patches to be taken from the patch sampling set
+# A default patch is a single full-band frame which patchWidth = (maxFreqBin - minFreqBin + 1) / numFreqBand, patchHeight = 1
+# PatchWidth: 5
+PatchHeight: 1
+
+# the number of frames that their feature vectors will be concatenated in order to preserve temporal information.
+FrameWindowLength : 1 
+
+# the step size to make a window of frames
+StepSize : 1
+
+# The number of patches to be selected from each recording of the patch sampling set
+NumRandomPatches: 1000
+
+# the number of clusters to be generated from the selected patch set
+NumClusters: 256
+
+# Applying noise reduction and whitening if these options are set to 'true'
+DoNoiseReduction: true
+DoWhitening: true
+
+# The factor of data downsampling using max pooling
+MaxPoolingFactor: 6
+...
+
diff --git a/src/AnalysisPrograms/AnalysisPrograms.csproj b/src/AnalysisPrograms/AnalysisPrograms.csproj
@@ -52,14 +52,20 @@
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Accord, Version=2.12.0.0, Culture=neutral, PublicKeyToken=fa1a88e29555ccf7, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\Accord.2.12.0.0\lib\Accord.dll</HintPath>
+    <Reference Include="Accord, Version=3.8.0.0, Culture=neutral, PublicKeyToken=fa1a88e29555ccf7, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\Accord.3.8.0\lib\net462\Accord.dll</HintPath>
     </Reference>
-    <Reference Include="Accord.Math, Version=2.12.0.0, Culture=neutral, PublicKeyToken=fa1a88e29555ccf7, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\Accord.Math.2.12.0.0\lib\Accord.Math.dll</HintPath>
+    <Reference Include="Accord.MachineLearning, Version=3.8.0.0, Culture=neutral, PublicKeyToken=fa1a88e29555ccf7, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\Accord.MachineLearning.3.8.0\lib\net462\Accord.MachineLearning.dll</HintPath>
     </Reference>
-    <Reference Include="Accord.Statistics, Version=2.12.0.0, Culture=neutral, PublicKeyToken=fa1a88e29555ccf7, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\Accord.Statistics.2.12.0.0\lib\Accord.Statistics.dll</HintPath>
+    <Reference Include="Accord.Math, Version=3.8.0.0, Culture=neutral, PublicKeyToken=fa1a88e29555ccf7, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\Accord.Math.3.8.0\lib\net462\Accord.Math.dll</HintPath>
+    </Reference>
+    <Reference Include="Accord.Math.Core, Version=3.8.0.0, Culture=neutral, PublicKeyToken=fa1a88e29555ccf7, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\Accord.Math.3.8.0\lib\net462\Accord.Math.Core.dll</HintPath>
+    </Reference>
+    <Reference Include="Accord.Statistics, Version=3.8.0.0, Culture=neutral, PublicKeyToken=fa1a88e29555ccf7, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\Accord.Statistics.3.8.0\lib\net462\Accord.Statistics.dll</HintPath>
     </Reference>
     <Reference Include="AForge, Version=2.2.5.0, Culture=neutral, PublicKeyToken=c1db6ff4eaa06aeb, processorArchitecture=MSIL">
       <HintPath>..\..\packages\AForge.2.2.5\lib\AForge.dll</HintPath>
@@ -94,8 +100,8 @@
     <Reference Include="MathNet.Numerics, Version=3.20.2.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>..\..\packages\MathNet.Numerics.3.20.2\lib\net40\MathNet.Numerics.dll</HintPath>
     </Reference>
-    <Reference Include="McMaster.Extensions.CommandLineUtils, Version=2.2.5.0, Culture=neutral, PublicKeyToken=6f71cb76b82f055d, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\McMaster.Extensions.CommandLineUtils.2.2.5\lib\net45\McMaster.Extensions.CommandLineUtils.dll</HintPath>
+    <Reference Include="McMaster.Extensions.CommandLineUtils, Version=2.3.0.0, Culture=neutral, PublicKeyToken=6f71cb76b82f055d, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\McMaster.Extensions.CommandLineUtils.2.3.0-alpha\lib\net45\McMaster.Extensions.CommandLineUtils.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="Microsoft.Data.Sqlite, Version=2.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
@@ -287,6 +293,7 @@
     <Compile Include="ConcatenateIndexFiles.cs" />
     <Compile Include="Draw\Zooming\DrawZoomingSpectrograms.cs" />
     <Compile Include="HerveGlotinCollaboration.cs" />
+    <Compile Include="MahnooshSandpit.cs" />
     <Compile Include="Production\Arguments\DebugOptions.cs" />
     <Compile Include="Production\Arguments\HelpArgs.cs" />
     <Compile Include="Production\Arguments\ListArgs.cs" />
@@ -488,10 +495,12 @@
     <Error Condition="!Exists('..\..\packages\SQLitePCLRaw.lib.e_sqlite3.linux.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.linux.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\packages\SQLitePCLRaw.lib.e_sqlite3.linux.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.linux.targets'))" />
     <Error Condition="!Exists('..\..\packages\SQLitePCLRaw.lib.e_sqlite3.osx.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.osx.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\packages\SQLitePCLRaw.lib.e_sqlite3.osx.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.osx.targets'))" />
     <Error Condition="!Exists('..\..\packages\SQLitePCLRaw.lib.e_sqlite3.v110_xp.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.v110_xp.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\packages\SQLitePCLRaw.lib.e_sqlite3.v110_xp.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.v110_xp.targets'))" />
+    <Error Condition="!Exists('..\..\packages\Accord.3.8.0\build\Accord.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\packages\Accord.3.8.0\build\Accord.targets'))" />
   </Target>
   <Import Project="..\..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets" Condition="Exists('..\..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" />
   <Import Project="..\..\packages\MSBuildTasks.1.5.0.235\build\MSBuildTasks.targets" Condition="Exists('..\..\packages\MSBuildTasks.1.5.0.235\build\MSBuildTasks.targets')" />
   <Import Project="..\..\packages\SQLitePCLRaw.lib.e_sqlite3.linux.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.linux.targets" Condition="Exists('..\..\packages\SQLitePCLRaw.lib.e_sqlite3.linux.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.linux.targets')" />
   <Import Project="..\..\packages\SQLitePCLRaw.lib.e_sqlite3.osx.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.osx.targets" Condition="Exists('..\..\packages\SQLitePCLRaw.lib.e_sqlite3.osx.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.osx.targets')" />
   <Import Project="..\..\packages\SQLitePCLRaw.lib.e_sqlite3.v110_xp.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.v110_xp.targets" Condition="Exists('..\..\packages\SQLitePCLRaw.lib.e_sqlite3.v110_xp.1.1.9\build\net35\SQLitePCLRaw.lib.e_sqlite3.v110_xp.targets')" />
+  <Import Project="..\..\packages\Accord.3.8.0\build\Accord.targets" Condition="Exists('..\..\packages\Accord.3.8.0\build\Accord.targets')" />
 </Project>
diff --git a/src/AnalysisPrograms/Audio2Sonogram.cs b/src/AnalysisPrograms/Audio2Sonogram.cs
@@ -171,6 +171,10 @@ private static Dictionary<string, string> GetConfigDictionary(FileInfo configFil
             return configDict;
         }
 
+        /// <summary>
+        /// Dictionary string, string> configDict is an obsolete class.
+        /// Should avoid calls to this method.
+        /// </summary>
         public static AudioToSonogramResult GenerateFourSpectrogramImages(
             FileInfo sourceRecording,
             FileInfo path2SoxSpectrogram,