Skip to content

Commit

Permalink
Improve non-buffering strategy for FileStream
Browse files Browse the repository at this point in the history
  • Loading branch information
BCSharp committed Dec 31, 2024
1 parent 86adef5 commit a5e3975
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 19 deletions.
11 changes: 5 additions & 6 deletions Src/IronPython.Modules/nt.cs
Original file line number Diff line number Diff line change
Expand Up @@ -883,8 +883,6 @@ public static void mkdir(CodeContext context, [NotNone] Bytes path, [ParamDictio
public static void mkdir(CodeContext context, object? path, [ParamDictionary, NotNone] IDictionary<string, object> kwargs, [NotNone] params object[] args)
=> mkdir(ConvertToFsString(context, path, nameof(path)), kwargs, args);

private const int DefaultBufferSize = 4096;

[Documentation("""
open(path, flags, mode=511, *, dir_fd=None)
Expand Down Expand Up @@ -940,15 +938,16 @@ public static object open(CodeContext/*!*/ context, [NotNone] string path, int f
// open it again w/ just read access.
fs = new FileStream(path, fileMode, FileAccess.Write, FileShare.None);
fs.Close();
s = fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, DefaultBufferSize, options);
s = fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 1, options);
} else if (access == FileAccess.ReadWrite && fileMode == FileMode.Append) {
// .NET doesn't allow Append w/ access != Write, so open the file w/ Write
// and a secondary stream w/ Read, then seek to the end.
s = fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite, DefaultBufferSize, options);
rs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, DefaultBufferSize, options);
s = fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite, 1, options);
rs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 1, options);
rs.Seek(0L, SeekOrigin.End);
} else {
s = fs = new FileStream(path, fileMode, access, FileShare.ReadWrite, DefaultBufferSize, options);
// Use a 1 byte buffer size to disable buffering
s = fs = new FileStream(path, fileMode, access, FileShare.ReadWrite, 1, options);
}
rs ??= s;

Expand Down
13 changes: 2 additions & 11 deletions Src/IronPython/Runtime/PythonFileManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,19 +29,10 @@ internal sealed class StreamBox {
private int _id = -1;
private Stream _readStream;
private Stream _writeStream;
private readonly bool _writeNeedsFlush;

public StreamBox(Stream readStream, Stream writeStream) {
_readStream = readStream;
_writeStream = writeStream;

// a hack but it does improve perfomance a lot if flushing is not needed
_writeNeedsFlush = writeStream switch {
FileStream => true, // FileStream is buffered by default, used on Windows and mostly Mono
PosixFileStream => false, // PosixFileStream is unbuffered, used on .NET/Posix
UnixStream => false, // UnixStream is unbuffered, used sometimes on Mono
_ => true // if in doubt, flush
};
}

public StreamBox(Stream stream) : this(stream, stream) { }
Expand Down Expand Up @@ -162,8 +153,8 @@ public int Write(IPythonBuffer buffer) {
count = buffer.NumBytes();
_writeStream.Write(bytes, 0, count);
#endif
if (_writeNeedsFlush) {
_writeStream.Flush(); // IO at this level is not supposed to buffer so we need to call Flush.
if (ClrModule.IsMono && count == 1) {
_writeStream.Flush(); // IO at this level is not supposed to buffer so we need to call Flush (only needed on Mono)
}
if (!IsSingleStream) {
_readStream.Seek(_writeStream.Position, SeekOrigin.Begin);
Expand Down
4 changes: 2 additions & 2 deletions Tests/test_io_stdlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
## Run selected tests from test_io from StdLib
##

from iptest import is_ironpython, is_mono, generate_suite, run_test
from iptest import is_ironpython, is_mono, is_windows, generate_suite, run_test

import test.test_io

Expand Down Expand Up @@ -100,7 +100,7 @@ def load_tests(loader, standard_tests, pattern):
test.test_io.PyMiscIOTest('test_warn_on_dealloc_fd'), # AssertionError: ResourceWarning not triggered
]

if is_mono:
if is_mono or is_windows:
failing_tests += [
test.test_io.PyTextIOWrapperTest('test_seek_append_bom'), # OSError: [Errno -2146232800] Unable seek backward to overwrite data that previously existed in a file opened in Append mode.
]
Expand Down

0 comments on commit a5e3975

Please sign in to comment.