From a5e39758873e37c1f4b51cf3834ef754ff0354bb Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Mon, 30 Dec 2024 21:28:40 -0800 Subject: [PATCH] Improve non-buffering strategy for FileStream --- Src/IronPython.Modules/nt.cs | 11 +++++------ Src/IronPython/Runtime/PythonFileManager.cs | 13 ++----------- Tests/test_io_stdlib.py | 4 ++-- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/Src/IronPython.Modules/nt.cs b/Src/IronPython.Modules/nt.cs index 4622a62a5..baf77c5a5 100644 --- a/Src/IronPython.Modules/nt.cs +++ b/Src/IronPython.Modules/nt.cs @@ -883,8 +883,6 @@ public static void mkdir(CodeContext context, [NotNone] Bytes path, [ParamDictio public static void mkdir(CodeContext context, object? path, [ParamDictionary, NotNone] IDictionary kwargs, [NotNone] params object[] args) => mkdir(ConvertToFsString(context, path, nameof(path)), kwargs, args); - private const int DefaultBufferSize = 4096; - [Documentation(""" open(path, flags, mode=511, *, dir_fd=None) @@ -940,15 +938,16 @@ public static object open(CodeContext/*!*/ context, [NotNone] string path, int f // open it again w/ just read access. fs = new FileStream(path, fileMode, FileAccess.Write, FileShare.None); fs.Close(); - s = fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, DefaultBufferSize, options); + s = fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 1, options); } else if (access == FileAccess.ReadWrite && fileMode == FileMode.Append) { // .NET doesn't allow Append w/ access != Write, so open the file w/ Write // and a secondary stream w/ Read, then seek to the end. - s = fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite, DefaultBufferSize, options); - rs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, DefaultBufferSize, options); + s = fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite, 1, options); + rs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 1, options); rs.Seek(0L, SeekOrigin.End); } else { - s = fs = new FileStream(path, fileMode, access, FileShare.ReadWrite, DefaultBufferSize, options); + // Use a 1 byte buffer size to disable buffering + s = fs = new FileStream(path, fileMode, access, FileShare.ReadWrite, 1, options); } rs ??= s; diff --git a/Src/IronPython/Runtime/PythonFileManager.cs b/Src/IronPython/Runtime/PythonFileManager.cs index 574ba0eb3..f1efa378f 100644 --- a/Src/IronPython/Runtime/PythonFileManager.cs +++ b/Src/IronPython/Runtime/PythonFileManager.cs @@ -29,19 +29,10 @@ internal sealed class StreamBox { private int _id = -1; private Stream _readStream; private Stream _writeStream; - private readonly bool _writeNeedsFlush; public StreamBox(Stream readStream, Stream writeStream) { _readStream = readStream; _writeStream = writeStream; - - // a hack but it does improve perfomance a lot if flushing is not needed - _writeNeedsFlush = writeStream switch { - FileStream => true, // FileStream is buffered by default, used on Windows and mostly Mono - PosixFileStream => false, // PosixFileStream is unbuffered, used on .NET/Posix - UnixStream => false, // UnixStream is unbuffered, used sometimes on Mono - _ => true // if in doubt, flush - }; } public StreamBox(Stream stream) : this(stream, stream) { } @@ -162,8 +153,8 @@ public int Write(IPythonBuffer buffer) { count = buffer.NumBytes(); _writeStream.Write(bytes, 0, count); #endif - if (_writeNeedsFlush) { - _writeStream.Flush(); // IO at this level is not supposed to buffer so we need to call Flush. + if (ClrModule.IsMono && count == 1) { + _writeStream.Flush(); // IO at this level is not supposed to buffer so we need to call Flush (only needed on Mono) } if (!IsSingleStream) { _readStream.Seek(_writeStream.Position, SeekOrigin.Begin); diff --git a/Tests/test_io_stdlib.py b/Tests/test_io_stdlib.py index 9c24dd4dd..344eef760 100644 --- a/Tests/test_io_stdlib.py +++ b/Tests/test_io_stdlib.py @@ -6,7 +6,7 @@ ## Run selected tests from test_io from StdLib ## -from iptest import is_ironpython, is_mono, generate_suite, run_test +from iptest import is_ironpython, is_mono, is_windows, generate_suite, run_test import test.test_io @@ -100,7 +100,7 @@ def load_tests(loader, standard_tests, pattern): test.test_io.PyMiscIOTest('test_warn_on_dealloc_fd'), # AssertionError: ResourceWarning not triggered ] - if is_mono: + if is_mono or is_windows: failing_tests += [ test.test_io.PyTextIOWrapperTest('test_seek_append_bom'), # OSError: [Errno -2146232800] Unable seek backward to overwrite data that previously existed in a file opened in Append mode. ]