Skip to content

Commit

Permalink
Fix open append (#370)
Browse files Browse the repository at this point in the history
* Fix append in file_cache by saving flags and using append correctly

* Add unit test for file append

* Cleanup comment, revert change to OpenFile
  • Loading branch information
jfantinhardesty authored Nov 18, 2024
1 parent 2459128 commit a99fbc1
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 5 deletions.
13 changes: 12 additions & 1 deletion component/file_cache/file_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -965,6 +965,11 @@ func (fc *FileCache) OpenFile(options internal.OpenFileOptions) (*handlemap.Hand
// create handle and record openFileOptions for later
handle := handlemap.NewHandle(options.Name)
handle.SetValue("openFileOptions", openFileOptions{flags: options.Flags, fMode: options.Mode})

if options.Flags&os.O_APPEND != 0 {
handle.Flags.Set(handlemap.HandleOpenedAppend)
}

// Increment the handle count in this lock item as there is one handle open for this now
flock.Inc()

Expand Down Expand Up @@ -1117,7 +1122,13 @@ func (fc *FileCache) WriteFile(options internal.WriteFileOptions) (int, error) {

// Removing Pwrite as it is not supported on Windows
// bytesWritten, err := syscall.Pwrite(options.Handle.FD(), options.Data, options.Offset)
bytesWritten, err := f.WriteAt(options.Data, options.Offset)

var bytesWritten int
if options.Handle.Flags.IsSet(handlemap.HandleOpenedAppend) {
bytesWritten, err = f.Write(options.Data)
} else {
bytesWritten, err = f.WriteAt(options.Data, options.Offset)
}

if err == nil {
// Mark the handle dirty so the file is written back to storage on FlushFile.
Expand Down
9 changes: 5 additions & 4 deletions internal/handlemap/handle_map.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,11 @@ const InvalidHandleID HandleID = 0

// Flags represented in BitMap for various flags in the handle
const (
HandleFlagUnknown uint16 = iota
HandleFlagDirty // File has been modified with write operation or is a new file
HandleFlagFSynced // User has called fsync on the file explicitly
HandleFlagCached // File is cached in the local system by cloudfuse
HandleFlagUnknown uint16 = iota
HandleFlagDirty // File has been modified with write operation or is a new file
HandleFlagFSynced // User has called fsync on the file explicitly
HandleFlagCached // File is cached in the local system by cloudfuse
HandleOpenedAppend // File is opened for Append
)

// Structure to hold in memory cache for streaming layer
Expand Down
28 changes: 28 additions & 0 deletions test/e2e_tests/file_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,34 @@ func (suite *fileTestSuite) TestFileCreateLabel() {
suite.fileTestCleanup([]string{fileName})
}

func (suite *fileTestSuite) TestFileAppend() {
fileName := filepath.Join(suite.testPath, "append_test.txt")
initialContent := []byte("Initial content\n")
appendContent := []byte("Appended content\n")

// Create and write initial content to the file
srcFile, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0777)
suite.NoError(err)
_, err = srcFile.Write(initialContent)
suite.NoError(err)
srcFile.Close()

// Open the file with O_APPEND and append new content
appendFile, err := os.OpenFile(fileName, os.O_APPEND|os.O_WRONLY, 0777)
suite.NoError(err)
_, err = appendFile.Write(appendContent)
suite.NoError(err)
appendFile.Close()

// Read the file and verify the content
data, err := os.ReadFile(fileName)
suite.NoError(err)
expectedContent := append(initialContent, appendContent...)
suite.Equal(expectedContent, data)

suite.fileTestCleanup([]string{fileName})
}

// # Write a small file
func (suite *fileTestSuite) TestFileWriteSmall() {
fileName := filepath.Join(suite.testPath, "small_write.txt")
Expand Down

0 comments on commit a99fbc1

Please sign in to comment.