Merge pull request #166 from QutEcoacoustics/AudioCutterFix

Editting AudioCutter

AudioAnalysis.sln.DotSettings

@@ -1,9 +1,12 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
 	<s:Boolean x:Key="/Default/CodeEditing/ContextActionTable/DisabledContextActions/=JetBrains_002EReSharper_002EIntentions_002ECSharp_002EContextActions_002EMisc_002ESurroundWithQuotesAction/@EntryIndexedValue">False</s:Boolean>
+	<s:String x:Key="/Default/CodeInspection/Highlighting/InspectionSeverities/=RedundantCommaInInitializer/@EntryIndexedValue">DO_NOT_SHOW</s:String>
 	<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpCodeStyle/ThisQualifier/INSTANCE_MEMBERS_QUALIFY_MEMBERS/@EntryValue">15</s:String>
+	<s:Int64 x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/BLANK_LINES_BEFORE_SINGLE_LINE_COMMENT/@EntryValue">1</s:Int64>
 	<s:String x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/PLACE_ACCESSORHOLDER_ATTRIBUTE_ON_SAME_LINE_EX/@EntryValue">NEVER</s:String>
 	<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/PLACE_SIMPLE_ACCESSORHOLDER_ON_SINGLE_LINE/@EntryValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/SPACE_AFTER_TYPECAST_PARENTHESES/@EntryValue">False</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/STICK_COMMENT/@EntryValue">False</s:Boolean>
 	<s:String x:Key="/Default/CodeStyle/CSharpFileLayoutPatterns/Pattern/@EntryValue">&lt;?xml version="1.0" encoding="utf-16"?&gt;&#xD;
 &lt;Patterns xmlns="urn:schemas-jetbrains-com:member-reordering-patterns"&gt;&#xD;
   &lt;TypePattern DisplayName="Non-reorderable types"&gt;&#xD;

CONTRIBUTING.md

@@ -10,6 +10,18 @@ Best Practices
 - NEVER commit if the code does not build to the `master` branch
 - Try to work on branches if your code negatively affects production code
 - Write code in American English. Documentation may be written in Australian English.
+- Wherever possible **use un-prefixed SI units for variables**
+    1. Variables with no unit **MUST** be standard units
+        - `var duration = 30` should **always** be 30 seconds
+        - `var bandwidth = 50` should **always be hertz**
+    1. **Never** use imperial units
+    1. **Always** include the full unit in the name if it does not follow our standard
+        - avoid this if possible, see first point
+        - e.g. `var minKiloHertz = 3.5`
+        - e.g.  `var limitHours = 6`
+    1. **Do not** abbreviate units
+    1. It is **recommended** that full units be used in any user facing field name
+        - e.g. `EventEndSeconds` in a CSV file
 
 Required Software
 -----------------

src/Acoustics.Tools/AudioFilePreparer.cs

@@ -76,14 +76,15 @@ public static AudioUtilityModifiedInfo PrepareFile(
             FileInfo source,
             string outputMediaType,
             AudioUtilityRequest request,
-            DirectoryInfo temporaryFilesDirectory)
+            DirectoryInfo temporaryFilesDirectory,
+            bool oldFormat = true)
         {
             var outputFileName = GetFileName(
                 source.Name,
                 outputMediaType,
                 request.OffsetStart,
                 request.OffsetEnd,
-                oldFormat: true);
+                oldFormat: oldFormat);
 
             var outputFile = new FileInfo(Path.Combine(outputDirectory.FullName, outputFileName));
 

src/AnalysisBase/AnalysisSettings.cs

@@ -156,6 +156,7 @@ public DirectoryInfo AnalysisTempDirectoryFallback
         /// Gets or sets the minimum audio file duration the analysis can process.
         /// This is the min duration without including overlap. Overlap will be added.
         /// This should be set to an initial value by an analysis.
+        /// This value is used in <see cref="AnalysisCoordinator.PrepareAnalysisSegments{TSource}"/>
         /// TODO: a chunk of audio without the overlap is a 'segment step'.
         /// </summary>
         public TimeSpan AnalysisMinSegmentDuration { get; set; }

src/AnalysisPrograms/AudioCutter.cs

@@ -74,13 +74,13 @@ public class Arguments : SubCommandBase
             public double? EndOffset { get; set; }
 
             [Option(
-                Description = "The minimum duration of a segmented audio file (in seconds, defaults to 10; must be within [0,3600]).",
+                Description = "The minimum duration of a segmented audio file (in seconds, defaults to 5; must be within [0, 3600]).",
                 ShortName = "")]
             [InRange(1, 3600)]
-            public double SegmentDurationMinimum { get; set; } = 10;
+            public double SegmentDurationMinimum { get; set; } = 5;
 
             [Option(
-                Description = "The duration of a segmented audio file (in seconds, defaults to 60; must be within [0,3600]).",
+                Description = "The duration of a segmented audio file (in seconds, defaults to 60; must be within [0, 3600]).",
                 ShortName = "d")]
             [InRange(1, 3600)]
             public double SegmentDuration { get; set; } = 60;
@@ -98,7 +98,7 @@ public class Arguments : SubCommandBase
             public string SegmentFileExtension { get; set; } = "wav";
 
             [Option(
-                Description = "The sample rate for segmented audio files (in hertz, defaults to 22050; valid values are 17640, 22050, 44100).",
+                Description = "The sample rate for segmented audio files (in hertz, defaults to 22050; valid values are 8000, 17640, 22050, 44100, 48000, 96000).",
                 ShortName = "r")]
             [InRange(8000, 96000)]
             public int SampleRate { get; set; } = 22050;
@@ -120,8 +120,7 @@ public class Arguments : SubCommandBase
 
             public override Task<int> Execute(CommandLineApplication app)
             {
-                AudioCutter.Execute(this);
-                return this.Ok();
+                return AudioCutter.Execute(this);
             }
 
             protected override ValidationResult OnValidate(ValidationContext context, CommandLineContext appContext)
@@ -166,7 +165,7 @@ protected override ValidationResult OnValidate(ValidationContext context, Comman
             }
         }
 
-        public static void Execute(Arguments arguments)
+        public static async Task<int> Execute(Arguments arguments)
         {
             if (arguments == null)
             {
@@ -175,7 +174,7 @@ public static void Execute(Arguments arguments)
 
             var sw = new Stopwatch();
             sw.Start();
-            ISourcePreparer sourcePreparer = new LocalSourcePreparer();
+            ISourcePreparer sourcePreparer = new LocalSourcePreparer(filterShortSegments: true, useOldNamingFormat: false);
 
             //create analysis settings using arguments
             AnalysisSettings settings = new AnalysisSettings()
@@ -185,11 +184,11 @@ public static void Execute(Arguments arguments)
                 AnalysisMinSegmentDuration = TimeSpan.FromSeconds(arguments.SegmentDurationMinimum),
                 SegmentOverlapDuration = TimeSpan.FromSeconds(arguments.SegmentOverlap),
                 AnalysisTargetSampleRate = arguments.SampleRate,
-                AnalysisTempDirectory = arguments.TemporaryFilesDir.ToDirectoryInfo(),
+                AnalysisTempDirectory = (arguments.TemporaryFilesDir ?? arguments.OutputDir).ToDirectoryInfo(),
             };
 
             // create segments from file
-            var fileSegment = new FileSegment(arguments.InputFile.ToFileInfo(), TimeAlignment.None)
+            var fileSegment = new FileSegment(arguments.InputFile.ToFileInfo(), TimeAlignment.None, dateBehavior: FileSegment.FileDateBehavior.None)
             {
                 SegmentStartOffset = TimeSpan.FromSeconds(arguments.StartOffset),
             };
@@ -213,21 +212,25 @@ public static void Execute(Arguments arguments)
             }
             else
             {
-                RunSequential(fileSegments, sourcePreparer, settings, arguments);
+                var runTime = await RunSequential(fileSegments, sourcePreparer, settings, arguments);
             }
 
             sw.Stop();
             LoggedConsole.WriteLine("Took {0}. Done.", sw.Elapsed);
+            return ExceptionLookup.Ok;
         }
 
-        private static void RunSequential(List<ISegment<FileInfo>> fileSegments, ISourcePreparer sourcePreparer, AnalysisSettings settings, Arguments arguments)
+        private static async Task<double> RunSequential(List<ISegment<FileInfo>> fileSegments, ISourcePreparer sourcePreparer, AnalysisSettings settings, Arguments arguments)
         {
             var totalItems = fileSegments.Count;
+            var totalTime = 0.0;
             for (var index = 0; index < fileSegments.Count; index++)
             {
                 var item = fileSegments[index];
-                CreateSegment(sourcePreparer, item, settings, arguments, index + 1, totalItems, arguments.MixDownToMono);
+                totalTime += await CreateSegment(sourcePreparer, item, settings, arguments, index + 1, totalItems, arguments.MixDownToMono);
             }
+
+            return totalTime;
         }
 
         private static void RunParallel(List<ISegment<FileInfo>> fileSegments, ISourcePreparer sourcePreparer, AnalysisSettings settings, Arguments arguments)
@@ -240,11 +243,12 @@ private static void RunParallel(List<ISegment<FileInfo>> fileSegments, ISourcePr
                     var item1 = item;
                     int index1 = Convert.ToInt32(index);
 
-                    CreateSegment(sourcePreparer, item1, settings, arguments, index1 + 1, totalItems, arguments.MixDownToMono);
+                    // call create segment synchronously
+                    CreateSegment(sourcePreparer, item1, settings, arguments, index1 + 1, totalItems, arguments.MixDownToMono).Wait();
                 });
         }
 
-        private static void CreateSegment(
+        private static async Task<double> CreateSegment(
             ISourcePreparer sourcePreparer,
             ISegment<FileInfo> fileSegment,
             AnalysisSettings settings,
@@ -253,23 +257,33 @@ private static void CreateSegment(
             int itemCount,
             bool mixDownToMono)
         {
-            var task = sourcePreparer.PrepareFile(
+            var timer = Stopwatch.StartNew();
+
+            FileSegment preparedFile;
+            try
+            {
+                preparedFile = await sourcePreparer.PrepareFile(
                     arguments.OutputDir.ToDirectoryInfo(),
                     fileSegment,
                     settings.SegmentMediaType,
                     settings.AnalysisTargetSampleRate,
                     settings.AnalysisTempDirectory,
                     null,
                     mixDownToMono);
-
-            task.Wait(120.Seconds());
-            var preparedFile = task.Result;
+            }
+            catch (IOException ioex)
+            {
+                LoggedConsole.WriteError($"Failed to cut segment {itemNumber} of {itemCount}:" + ioex.Message);
+                return double.NaN;
+            }
 
             LoggedConsole.WriteLine(
                 "Created segment {0} of {1}: {2}",
                 itemNumber,
                 itemCount,
                 preparedFile.SourceMetadata.Identifier);
+
+            return timer.Elapsed.TotalSeconds;
         }
     }
 }

src/AnalysisPrograms/SourcePreparers/LocalSourcePreparer.cs

@@ -23,6 +23,14 @@ namespace AnalysisPrograms.SourcePreparers
     public class LocalSourcePreparer : ISourcePreparer
     {
         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+        private readonly bool useOldNamingFormat;
+        private readonly bool filterShortSegments;
+
+        public LocalSourcePreparer(bool filterShortSegments = false, bool useOldNamingFormat = true)
+        {
+            this.filterShortSegments = filterShortSegments;
+            this.useOldNamingFormat = useOldNamingFormat;
+        }
 
         /// <summary>
         /// Prepare an audio file. This will be a single segment of a larger audio file, modified based on the analysisSettings.
@@ -73,7 +81,8 @@ public async Task<FileSegment> PrepareFile(
                     sourceFileInfo,
                     outputMediaType,
                     request,
-                    TempFileHelper.TempDir());
+                    TempFileHelper.TempDir(),
+                    oldFormat: this.useOldNamingFormat);
 
                 return new FileSegment(
                     preparedFile.TargetInfo.SourceFile,
@@ -148,7 +157,7 @@ public IEnumerable<ISegment<TSource>> CalculateSegments<TSource>(
 
                 var analysisSegmentMaxDuration = settings.AnalysisMaxSegmentDuration?.TotalMilliseconds ?? fileSegmentDuration;
 
-                var analysisSegmentMinDuration = settings.AnalysisMinSegmentDuration.TotalMilliseconds;
+                var analysisSegmentMinDuration = this.filterShortSegments ? settings.AnalysisMinSegmentDuration : (TimeSpan?)null;
 
                 // segment into exact chunks - all but the last chunk will be equal to the max duration
                 var segments = AudioFilePreparer.DivideExactLeaveLeftoversAtEnd(
@@ -164,8 +173,19 @@ public IEnumerable<ISegment<TSource>> CalculateSegments<TSource>(
                 {
                     Log.Debug($"Generated fractional segment for time alignment ({startOffset} - {startOffset + startDelta})");
                     var startAlignDelta = Convert.ToInt64(startDelta.TotalMilliseconds);
-                    yield return
-                        (ISegment<TSource>)CreateSegment(ref aggregate, startAlignDelta, fileSegment, startOffset, endOffset, overlap);
+
+                    if (TryCreateSegment(
+                        ref aggregate,
+                        startAlignDelta,
+                        fileSegment,
+                        startOffset,
+                        endOffset,
+                        overlap,
+                        analysisSegmentMinDuration,
+                        out var validFileSegment))
+                    {
+                        yield return (ISegment<TSource>)validFileSegment;
+                    }
                 }
                 else
                 {
@@ -176,8 +196,18 @@ public IEnumerable<ISegment<TSource>> CalculateSegments<TSource>(
                 // yield each normal segment
                 foreach (long offset in segments)
                 {
-                    yield return
-                        (ISegment<TSource>)CreateSegment(ref aggregate, offset, fileSegment, startOffset, endOffset, overlap);
+                    if (TryCreateSegment(
+                        ref aggregate,
+                        offset,
+                        fileSegment,
+                        startOffset,
+                        endOffset,
+                        overlap,
+                        analysisSegmentMinDuration,
+                        out var validFileSegment))
+                    {
+                        yield return (ISegment<TSource>)validFileSegment;
+                    }
                 }
 
                 // include fractional segment cut from time alignment
@@ -186,19 +216,32 @@ public IEnumerable<ISegment<TSource>> CalculateSegments<TSource>(
                 {
                     Log.Debug($"Generated fractional segment for time alignment ({endOffset - endDelta} - {endOffset})");
                     var endAlignDelta = Convert.ToInt64(endDelta.TotalMilliseconds);
-                    yield return
-                        (ISegment<TSource>)CreateSegment(ref aggregate, endAlignDelta, fileSegment, startOffset, endOffset, overlap);
+
+                    if (TryCreateSegment(
+                        ref aggregate,
+                        endAlignDelta,
+                        fileSegment,
+                        startOffset,
+                        endOffset,
+                        overlap,
+                        analysisSegmentMinDuration,
+                        out var validFileSegment))
+                    {
+                        yield return (ISegment<TSource>)validFileSegment;
+                    }
                 }
             }
         }
 
-        internal static ISegment<TSource> CreateSegment<TSource>(
+        internal static bool TryCreateSegment<TSource>(
             ref long aggregate,
-            long offset,
+            in long offset,
             ISegment<TSource> currentSegment,
-            TimeSpan startOffset,
-            TimeSpan endOffset,
-            TimeSpan overlap)
+            in TimeSpan startOffset,
+            in TimeSpan endOffset,
+            in TimeSpan overlap,
+            TimeSpan? minimumDuration,
+            out ISegment<TSource> newSegment)
         {
             var newStart = startOffset.Add(TimeSpan.FromMilliseconds(aggregate));
 
@@ -217,9 +260,23 @@ internal static ISegment<TSource> CreateSegment<TSource>(
 
             var newEnd = segmentEndOffset;
 
+            // The minimum segment filtering is usually taken care of in AnalysisCoordinator.
+            // However some code does not use AnalysisCoordinator so we duplicate the functionality here.
+            if (minimumDuration.HasValue)
+            {
+                if ((newEnd - newStart) < minimumDuration)
+                {
+                    Log.Warn(
+                        $"Omitting short segment {newStart}–{newEnd} because it is less than the minimum {minimumDuration}");
+                    newSegment = null;
+                    return false;
+                }
+            }
+
             // So we aren't actually cutting any files, rather we're preparing to cut files.
             // Thus the clone the object and set new offsets.
-            return currentSegment.SplitSegment(newStart.TotalSeconds, newEnd.TotalSeconds);
+            newSegment = currentSegment.SplitSegment(newStart.TotalSeconds, newEnd.TotalSeconds);
+            return true;
         }
 
         /// <summary>
@@ -274,7 +331,8 @@ public async Task<FileSegment> PrepareFile<TSource>(
                     segment.Source,
                     outputMediaType,
                     request,
-                    temporaryFilesDirectory);
+                    temporaryFilesDirectory,
+                    oldFormat: this.useOldNamingFormat);
 
                 return new FileSegment(
                     preparedFile.TargetInfo.SourceFile,

src/AnalysisPrograms/SourcePreparers/RemoteSourcePreparer.cs

@@ -140,13 +140,19 @@ public IEnumerable<ISegment<TSource>> CalculateSegments<TSource>(
                     // yield each normal segment
                     foreach (long offset in segments)
                     {
-                        yield return LocalSourcePreparer.CreateSegment(
+                        // The null for minimum means, do not filter short segments. Short filtering is done in AnalysisCoordinator
+                        if (LocalSourcePreparer.TryCreateSegment(
                             ref aggregate,
                             offset,
                             segment,
                             startOffset,
                             endOffset,
-                            overlap);
+                            overlap,
+                            null /*settings.AnalysisMinSegmentDuration*/,
+                            out var validFileSegment))
+                        {
+                            yield return (ISegment<TSource>)validFileSegment;
+                        }
                     }
                 }
             }