From 2f97ee4ee69a7180077f669c097a38dc2d117ac2 Mon Sep 17 00:00:00 2001 From: Dave Glick Date: Sat, 10 Apr 2021 22:31:23 -0400 Subject: [PATCH] A little more file tweaking --- src/core/Statiq.Core/IO/Local/LocalFile.cs | 2 +- src/core/Statiq.Core/Modules/IO/CopyFiles.cs | 2 +- src/core/Statiq.Core/Modules/IO/WriteFiles.cs | 90 ++++++++++--------- 3 files changed, 49 insertions(+), 45 deletions(-) diff --git a/src/core/Statiq.Core/IO/Local/LocalFile.cs b/src/core/Statiq.Core/IO/Local/LocalFile.cs index 64d48862..28f6dc66 100644 --- a/src/core/Statiq.Core/IO/Local/LocalFile.cs +++ b/src/core/Statiq.Core/IO/Local/LocalFile.cs @@ -9,7 +9,7 @@ namespace Statiq.Core // Initially based on code from Cake (http://cakebuild.net/) internal class LocalFile : IFile { - private const int BufferSize = 16384; + private const int BufferSize = 8192; private readonly LocalFileProvider _fileProvider; private readonly System.IO.FileInfo _file; diff --git a/src/core/Statiq.Core/Modules/IO/CopyFiles.cs b/src/core/Statiq.Core/Modules/IO/CopyFiles.cs index 19700338..eff57170 100644 --- a/src/core/Statiq.Core/Modules/IO/CopyFiles.cs +++ b/src/core/Statiq.Core/Modules/IO/CopyFiles.cs @@ -98,7 +98,7 @@ protected override async Task> ExecuteConfigAsync(IDocume if (value is object) { // Use a semaphore to limit the write operations so we don't try to do a bunch of writes at once - SemaphoreSlim semaphore = new SemaphoreSlim(10, 10); + SemaphoreSlim semaphore = new SemaphoreSlim(20, 20); // Create copy tasks IEnumerable inputFiles = context.FileSystem.GetInputFiles(value); diff --git a/src/core/Statiq.Core/Modules/IO/WriteFiles.cs b/src/core/Statiq.Core/Modules/IO/WriteFiles.cs index 7dcd1362..cbebc7a6 100644 --- a/src/core/Statiq.Core/Modules/IO/WriteFiles.cs +++ b/src/core/Statiq.Core/Modules/IO/WriteFiles.cs @@ -71,14 +71,14 @@ public WriteFiles Where(Config predicate) protected override async Task> ExecuteContextAsync(IExecutionContext context) { // Use a semaphore to limit the write operations so we don't try to do a bunch of writes at once - SemaphoreSlim semaphore = new SemaphoreSlim(10, 10); + SemaphoreSlim semaphore = new SemaphoreSlim(20, 20); // Get the output file path for each file in sequence and set up action chains // Value = input source string(s) (for reporting a warning if not appending), write action Dictionary, Func>> writesBySource = new Dictionary, Func>>(); foreach (IDocument input in context.Inputs) { - await WriteFileAsync(input); + await AddWriteFileTaskAsync(input, context, writesBySource, semaphore); } // Display a warning for any duplicated outputs if not appending @@ -91,57 +91,61 @@ protected override async Task> ExecuteContextAsync(IExecu } } - // Run the write actions in parallel + // Run the write actions await Task.WhenAll(writesBySource.Values.Select(x => x.Item2())); // Return the input documents return context.Inputs; + } - async Task WriteFileAsync(IDocument input) + private async Task AddWriteFileTaskAsync( + IDocument input, + IExecutionContext context, + Dictionary, Func>> writesBySource, + SemaphoreSlim semaphore) + { + if (await ShouldProcessAsync(input, context) && !input.Destination.IsNull) { - if (await ShouldProcessAsync(input, context) && !input.Destination.IsNull) + if (writesBySource.TryGetValue(input.Destination, out Tuple, Func> value)) { - if (writesBySource.TryGetValue(input.Destination, out Tuple, Func> value)) - { - // This output source was already seen so nest the previous write action in a new one - value.Item1.Add(input.Source.ToSafeDisplayString()); - Func previousWrite = value.Item2; - value = new Tuple, Func>( - value.Item1, - async () => + // This output source was already seen so nest the previous write action in a new one + value.Item1.Add(input.Source.ToSafeDisplayString()); + Func previousWrite = value.Item2; + value = new Tuple, Func>( + value.Item1, + async () => + { + await semaphore.WaitAsync(); + try { - await semaphore.WaitAsync(); - try - { - // Complete the previous write, then do the next one - await previousWrite(); - await WriteAsync(input, context, input.Destination); - } - finally - { - semaphore.Release(); - } - }); - } - else - { - value = new Tuple, Func>( - new List { input.Source.ToSafeDisplayString() }, - async () => + // Complete the previous write, then do the next one + await previousWrite(); + await WriteAsync(input, context, input.Destination); + } + finally { - await semaphore.WaitAsync(); - try - { - await WriteAsync(input, context, input.Destination); - } - finally - { - semaphore.Release(); - } - }); - } - writesBySource[input.Destination] = value; + semaphore.Release(); + } + }); + } + else + { + value = new Tuple, Func>( + new List { input.Source.ToSafeDisplayString() }, + async () => + { + await semaphore.WaitAsync(); + try + { + await WriteAsync(input, context, input.Destination); + } + finally + { + semaphore.Release(); + } + }); } + writesBySource[input.Destination] = value; } }