From 6d44046f168e784761180dc02cd5e3d3440c74b6 Mon Sep 17 00:00:00 2001 From: myxo Date: Sun, 10 Mar 2024 15:22:10 +0700 Subject: [PATCH] First somewhat working solution --- .gitignore | 2 + README.md | 12 ++ go.mod | 11 ++ go.sum | 13 ++ gofs.go | 245 +++++++++++++++++++++++++++- memory.go | 403 ++++++++++++++++++++++++++++++++++++++++++++- memory_fs.go | 203 +++++++++++++++++++++++ memory_test.go | 434 ++++++++++++++++++++++++++++++++++++++++++++++++- util.go | 55 +++++++ util_test.go | 21 +++ 10 files changed, 1394 insertions(+), 5 deletions(-) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 go.sum create mode 100644 memory_fs.go create mode 100644 util.go create mode 100644 util_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..385d6a8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.out +*.test diff --git a/README.md b/README.md new file mode 100644 index 0000000..403748b --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +Features: +- performance in non mock path (no interfaces) +- profound testing (we use property based tsting to ensure simularity with real implementation + +Non features: +- make a varaity of different backends (like NetFS, s2, etc.). I try to keep package as clean from dependencies as possible +- simulating of concurrent effect of filesystem (e.g. concurrent ReadDir with file removing) + +TODO: +- [ ] Make count in test to see how much function envocation we have +- [ ] Document what fileMode are supported +- [ ] O_APPEND diff --git a/go.mod b/go.mod index 1711862..2bf1059 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,14 @@ module github.com/myxo/gofs go 1.22.0 + +require ( + github.com/stretchr/testify v1.9.0 + pgregory.net/rapid v1.1.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0af4125 --- /dev/null +++ b/go.sum @@ -0,0 +1,13 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw= +pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/gofs.go b/gofs.go index 2d184c7..ea81c11 100644 --- a/gofs.go +++ b/gofs.go @@ -1,7 +1,250 @@ package gofs +import ( + "io" + "os" +) + type FS interface { - + Create(name string) (*File, error) + CreateTemp(dir, pattern string) (*File, error) + // NewFile(fd uintptr, name string) *File // TODO: ??? + Open(name string) (*File, error) + OpenFile(name string, flag int, perm os.FileMode) (*File, error) + Chdir(dir string) error + Chmod(name string, mode os.FileMode) error + Chown(name string, uid, gid int) error + Mkdir(name string, perm os.FileMode) error + MkdirAll(path string, perm os.FileMode) error + MkdirTemp(dir, pattern string) (string, error) + ReadFile(name string) ([]byte, error) + Readlink(name string) (string, error) + ReadDir(name string) ([]os.DirEntry, error) + Remove(name string) error + RemoveAll(path string) error + Rename(oldpath, newpath string) error + Truncate(name string, size int64) error + WriteFile(name string, data []byte, perm os.FileMode) error +} + +type File struct { + mockFile *FakeFile + osFile *os.File +} + +func NewFromOs(fp *os.File) *File { + return &File{osFile: fp} +} + +func (f *File) Fd() uintptr { + if f.osFile != nil { + return f.osFile.Fd() + } + return 0 +} + +func (f *File) Chdir() error { + if f.osFile != nil { + return f.osFile.Chdir() + } + return f.mockFile.Chdir() +} + +func (f *File) Chmod(mode os.FileMode) error { + if f.osFile != nil { + return f.osFile.Chmod(mode) + } + return f.mockFile.Chmod(mode) +} + +func (f *File) Chown(uid, gid int) error { panic("todo") } + +func (f *File) Close() error { + if f.osFile != nil { + return f.osFile.Close() + } + return f.mockFile.Close() +} + +func (f *File) Name() string { + if f.osFile != nil { + return f.osFile.Name() + } + return f.mockFile.Name() +} + +func (f *File) Read(b []byte) (n int, err error) { + if f.osFile != nil { + return f.osFile.Read(b) + } + return f.mockFile.Read(b) +} + +func (f *File) ReadAt(b []byte, off int64) (n int, err error) { + if f.osFile != nil { + return f.osFile.ReadAt(b, off) + } + return f.mockFile.ReadAt(b, off) } +func (f *File) ReadDir(n int) ([]os.DirEntry, error) { + if f.osFile != nil { + return f.osFile.ReadDir(n) + } + return f.mockFile.ReadDir(n) +} + +func (f *File) ReadFrom(r io.Reader) (n int64, err error) { + if f.osFile != nil { + return f.osFile.ReadFrom(r) + } + return f.mockFile.ReadFrom(r) +} + +func (f *File) Readdir(n int) ([]os.FileInfo, error) { + if f.osFile != nil { + return f.osFile.Readdir(n) + } + return f.mockFile.Readdir(n) +} + +func (f *File) Readdirnames(n int) (names []string, err error) { + if f.osFile != nil { + return f.osFile.Readdirnames(n) + } + return f.mockFile.Readdirnames(n) +} + +func (f *File) Seek(offset int64, whence int) (ret int64, err error) { + if f.osFile != nil { + return f.osFile.Seek(offset, whence) + } + return f.mockFile.Seek(offset, whence) +} + +func (f *File) Stat() (os.FileInfo, error) { + if f.osFile != nil { + return f.osFile.Stat() + } + return f.mockFile.Stat() +} + +func (f *File) Sync() error { + if f.osFile != nil { + return f.osFile.Sync() + } + return f.mockFile.Sync() +} + +func (f *File) Truncate(size int64) error { + if f.osFile != nil { + return f.osFile.Truncate(size) + } + return f.mockFile.Truncate(size) +} +func (f *File) Write(b []byte) (n int, err error) { + if f.osFile != nil { + return f.osFile.Write(b) + } + return f.mockFile.Write(b) +} + +func (f *File) WriteAt(b []byte, off int64) (n int, err error) { + if f.osFile != nil { + return f.osFile.WriteAt(b, off) + } + return f.mockFile.WriteAt(b, off) +} + +func (f *File) WriteString(s string) (n int, err error) { + if f.osFile != nil { + return f.osFile.WriteString(s) + } + return f.mockFile.WriteString(s) +} +func (f *File) WriteTo(w io.Writer) (n int64, err error) { panic("todo") } + +//func (f *File) SetDeadline(t time.Time) error{ panic("todo") } +//func (f *File) SetReadDeadline(t time.Time) error{ panic("todo") } +//func (f *File) SetWriteDeadline(t time.Time) error{ panic("todo") } + +type OsFs struct{} + +var _ FS = &OsFs{} + +func (OsFs) Create(name string) (*File, error) { + fp, err := os.Create(name) + return NewFromOs(fp), err +} + +func (OsFs) CreateTemp(dir, pattern string) (*File, error) { + fp, err := os.CreateTemp(dir, pattern) + return NewFromOs(fp), err +} + +func (OsFs) Open(name string) (*File, error) { + fp, err := os.Open(name) + return NewFromOs(fp), err +} + +func (OsFs) OpenFile(name string, flag int, perm os.FileMode) (*File, error) { + fp, err := os.OpenFile(name, flag, perm) + return NewFromOs(fp), err +} + +func (OsFs) Chdir(dir string) error { + return os.Chdir(dir) +} + +func (OsFs) Chmod(name string, mode os.FileMode) error { + return os.Chmod(name, mode) +} + +func (OsFs) Chown(name string, uid, gid int) error { + return os.Chown(name, uid, gid) +} + +func (OsFs) Mkdir(name string, perm os.FileMode) error { + return os.Mkdir(name, perm) +} + +func (OsFs) MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + +func (OsFs) MkdirTemp(dir, pattern string) (string, error) { + return os.MkdirTemp(dir, pattern) +} + +func (OsFs) ReadFile(name string) ([]byte, error) { + return os.ReadFile(name) +} + +func (OsFs) Readlink(name string) (string, error) { + return os.Readlink(name) +} + +func (OsFs) Remove(name string) error { + return os.Remove(name) +} + +func (OsFs) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +func (OsFs) Rename(oldpath, newpath string) error { + return os.Rename(oldpath, newpath) +} + +func (OsFs) Truncate(name string, size int64) error { + return os.Truncate(name, size) +} + +func (OsFs) WriteFile(name string, data []byte, perm os.FileMode) error { + return os.WriteFile(name, data, perm) +} + +func (OsFs) ReadDir(name string) ([]os.DirEntry, error) { + return os.ReadDir(name) +} diff --git a/memory.go b/memory.go index b4d1297..056ab92 100644 --- a/memory.go +++ b/memory.go @@ -1,7 +1,406 @@ package gofs -type data struct { - +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sync" + "time" + "unsafe" +) + +var filePool = sync.Pool{New: func() any { + return &mockData{ + buff: make([]byte, 0, 32*1024), + } +}} + +type interval struct { + from, to int64 +} + +type mockData struct { + buff []byte + realName string // this is synced with inodes map + isDirectory bool + dirContent []*mockData // if isDirectory is true + perm os.FileMode + dyrtyPages []interval // well... it's not exactly pages... +} + +// TODO: do I use this? +type Entry interface { + os.FileInfo + os.DirEntry +} + +func (m *mockData) reset() { + clear(m.buff[:cap(m.buff)]) // Zero all elements + m.buff = m.buff[:0] + m.perm = 0 + m.isDirectory = false + m.dirContent = nil +} + +func (m *mockData) Size() int64 { + return int64(len(m.buff)) +} + +func (m *mockData) hasWritePerm() bool { + const anyWritePerm = 0222 + return m.perm&anyWritePerm != 0 +} + +func (m *mockData) hasReadPerm() bool { + const anyReadPerm = 0444 + return m.perm&anyReadPerm != 0 +} + +// Kinda like descriptor +type FakeFile struct { + data *mockData + name string + flag int + cursor int64 + fs FS + readDirSlice []os.DirEntry // non empty only on directory iteration with ReadDir function + readDirSlice2 []os.FileInfo // non empty only on directory iteration with ReadDir function +} + +func (f *FakeFile) Chdir() error { panic("todo") } + +func (f *FakeFile) Chmod(mode os.FileMode) error { + if f.data == nil { + return fmt.Errorf("file already closed") + } + f.data.perm = mode & fs.ModePerm + return nil +} + +func (f *FakeFile) Chown(uid, gid int) error { panic("todo") } + +func (f *FakeFile) Close() error { + if f.data == nil { + return fmt.Errorf("file already closed") + } + if err := f.Sync(); err != nil { + // TODO: should I release memory here? + return err + } + // cannot reset all variables, since go implementation does not do it + f.data = nil + clear(f.readDirSlice) + clear(f.readDirSlice2) + return nil +} + +func (f *FakeFile) Name() string { + return f.name +} + +func (f *FakeFile) Read(b []byte) (n int, err error) { + n, err = f.pread(b, f.cursor) + f.cursor += int64(n) + return n, err +} + +func (f *FakeFile) ReadAt(b []byte, off int64) (n int, err error) { + if off < 0 { + return 0, fmt.Errorf("negative offset") + } + // Mimic weird implementation of ReadAt from stdlib + for len(b) > 0 { + m, e := f.pread(b, off) + if e != nil { + err = e + break + } + n += m + b = b[m:] + off += int64(m) + } + return n, err +} + +func (f *FakeFile) pread(b []byte, off int64) (n int, err error) { + if f.data == nil { + return 0, fmt.Errorf("file already closed") + } + if off < 0 { + return 0, fmt.Errorf("negative offset") + } + if len(b) == 0 { + return 0, nil + } + if !hasReadPerm(f.flag) { + return 0, fmt.Errorf("file %q open wiithout write permission", f.name) + } + if off > int64(len(f.data.buff)) { + return 0, io.ErrUnexpectedEOF + } + n = copy(b, f.data.buff[off:]) + if n == 0 { + return 0, io.EOF + } + return n, nil +} + +func (f *FakeFile) ReadDir(n int) ([]os.DirEntry, error) { + if f.data == nil { + return nil, fmt.Errorf("file already closed") + } + if !f.data.isDirectory { + return nil, fmt.Errorf("file %q not a directory", f.name) + } + if f.readDirSlice == nil { + for i := range f.data.dirContent { + f.readDirSlice = append(f.readDirSlice, NewInfoDataFromNode(f.data.dirContent[i], f.data.dirContent[i].realName)) + } + } + if n > 0 { + n = min(n, len(f.readDirSlice)) + } else { + n = len(f.readDirSlice) + } + ret := f.readDirSlice[:n] + f.readDirSlice = f.readDirSlice[n:] + if len(f.readDirSlice) == 0 { + f.readDirSlice = nil + } + return ret, nil +} + +func (f *FakeFile) Readdir(n int) ([]os.FileInfo, error) { + if f.data == nil { + return nil, fmt.Errorf("file already closed") + } + if !f.data.isDirectory { + return nil, fmt.Errorf("file %q not a directory", f.data.realName) + } + if f.readDirSlice2 == nil { + for i := range f.data.dirContent { + f.readDirSlice2 = append(f.readDirSlice2, NewInfoDataFromNode(f.data.dirContent[i], f.data.dirContent[i].realName)) + } + } + if n > 0 { + n = min(n, len(f.readDirSlice2)) + } else { + n = len(f.readDirSlice2) + } + ret := f.readDirSlice2[:n] + f.readDirSlice2 = f.readDirSlice2[n:] + if len(f.readDirSlice2) == 0 { + f.readDirSlice2 = nil + } + return ret, nil +} + +func (f *FakeFile) Readdirnames(n int) (names []string, err error) { + di, err := f.ReadDir(n) + out := make([]string, len(di)) + for i := range di { + out[i] = di[i].Name() + } + return out, err +} + +func (f *FakeFile) ReadFrom(r io.Reader) (n int64, err error) { + return io.Copy(fileWithoutReadFrom{FakeFile: f}, r) +} + +// Hack copypasted from stdlib +// noReadFrom can be embedded alongside another type to +// hide the ReadFrom method of that other type. +type noReadFrom struct{} + +// ReadFrom hides another ReadFrom method. +// It should never be called. +func (noReadFrom) ReadFrom(io.Reader) (int64, error) { + panic("can't happen") +} + +// fileWithoutReadFrom implements all the methods of *File other +// than ReadFrom. This is used to permit ReadFrom to call io.Copy +// without leading to a recursive call to ReadFrom. +type fileWithoutReadFrom struct { + noReadFrom + *FakeFile +} + +func (f *FakeFile) Seek(offset int64, whence int) (ret int64, err error) { + if f.data == nil { + return 0, fmt.Errorf("file already closed") + } + newOffset := int64(0) + start := int64(0) + switch whence { + case io.SeekStart: + start = 0 + case io.SeekCurrent: + start = f.cursor + case io.SeekEnd: + start = f.data.Size() + } + newOffset = start + offset + if newOffset < 0 { + return 0, fmt.Errorf("seek offset is negative") + } + f.cursor = newOffset + return newOffset, nil +} + +func (f *FakeFile) Stat() (os.FileInfo, error) { + if f.data == nil { + return nil, fmt.Errorf("file already closed") + } + // TODO: check read persmissions? + info := NewInfoDataFromNode(f.data, f.name) + return info, nil +} + +func (f *FakeFile) Sync() error { + if f.data == nil { + return fmt.Errorf("file already closed") + } + clear(f.data.dyrtyPages) + return nil +} + +func (f *FakeFile) Truncate(size int64) error { + if f.data == nil { + return fmt.Errorf("file already closed") + } + if size < 0 { + return fmt.Errorf("negative truncate size") + } + if !hasWritePerm(f.flag) { + return fmt.Errorf("file open without write permission") + } + f.data.buff = resizeSlice(f.data.buff, int(size)) + clear(f.data.buff[len(f.data.buff):cap(f.data.buff)]) + return nil +} + +func (f *FakeFile) Write(b []byte) (n int, err error) { + if f.data == nil { + return 0, fmt.Errorf("file already closed") + } + writePos := f.cursor + if isAppend(f.flag) { + writePos = f.data.Size() + } + n, err = f.pwrite(b, writePos) + // what with cursor with append flag? It doesn't matter? + f.cursor = writePos + int64(n) + return n, err +} + +func (f *FakeFile) WriteAt(b []byte, off int64) (n int, err error) { + if off < 0 { + return 0, fmt.Errorf("negative offset") + } + if isAppend(f.flag) { + return 0, fmt.Errorf("invalid use of WriteAt on file opened with O_APPEND") + } + + // Mimic weird WriteAt implementation of stdlib + for len(b) > 0 { + m, e := f.pwrite(b, off) + if e != nil { + err = e + break + } + n += m + b = b[m:] + off += int64(m) + } + return n, err +} + +func (f *FakeFile) pwrite(b []byte, off int64) (n int, err error) { + if f.data == nil { + return 0, fmt.Errorf("file already closed") + } + if off < 0 { + return 0, fmt.Errorf("negative offset") + } + + if !isReadWrite(f.flag) && !isWriteOnly(f.flag) { + return 0, fmt.Errorf("file %q open wiithout write permission", f.name) + } + + if len(b) == 0 { + return 0, nil + } + + if len(f.data.buff) < int(off)+len(b) { + f.data.buff = resizeSlice(f.data.buff, int(off)+len(b)) + } + n = copy(f.data.buff[off:], b) + + f.data.dyrtyPages = append(f.data.dyrtyPages, interval{from: off, to: off + int64(n)}) + return n, nil +} + +func (f *FakeFile) WriteString(s string) (n int, err error) { + b := unsafe.Slice(unsafe.StringData(s), len(s)) + return f.Write(b) +} + +func (f *FakeFile) WriteTo(w io.Writer) (n int64, err error) { panic("todo") } + +type infoData struct { + name string + size int64 + mode os.FileMode + modTime time.Time + isDir bool // TODO: feels like it may be in mode } +var _ os.FileInfo = &infoData{} +var _ os.DirEntry = &infoData{} + +func NewInfoDataFromNode(inode *mockData, name string) *infoData { + // TODO: think how we can use sync.Pool here + var info infoData + info.name = filepath.Base(name) + info.size = inode.Size() + info.mode = inode.perm + info.isDir = inode.isDirectory + // info.modTime = // TODO + return &info +} +func (m *infoData) Name() string { + return filepath.Base(m.name) +} + +func (m *infoData) Size() int64 { + return m.size +} + +func (m *infoData) Mode() os.FileMode { + return m.mode +} + +func (m *infoData) ModTime() time.Time { + panic("todo") +} + +func (m *infoData) IsDir() bool { + return m.isDir +} + +func (m *infoData) Sys() any { + return nil +} + +func (m *infoData) Type() os.FileMode { + return m.mode +} + +func (m *infoData) Info() (os.FileInfo, error) { + return m, nil +} diff --git a/memory_fs.go b/memory_fs.go new file mode 100644 index 0000000..ae04bea --- /dev/null +++ b/memory_fs.go @@ -0,0 +1,203 @@ +package gofs + +import ( + "fmt" + "io" + "os" + "path/filepath" +) + +type FakeFS struct { + inodes map[string]*mockData +} + +var _ FS = &FakeFS{} + +const rootDir = "/" + +func NewMemoryFs() *FakeFS { + fs := &FakeFS{ + inodes: map[string]*mockData{}, + } + fs.inodes[rootDir] = &mockData{ + realName: rootDir, + isDirectory: true, + perm: 0666, + } + return fs +} + +func (f *FakeFS) Create(path string) (*File, error) { + return f.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) +} + +func (f *FakeFS) CreateTemp(dir, pattern string) (*File, error) { + panic("todo") +} + +func (f *FakeFS) Open(name string) (*File, error) { + return f.OpenFile(name, os.O_RDONLY, 0) +} + +func checkOpenPerm(flag int, inode *mockData) error { + if hasWritePerm(flag) && !inode.hasWritePerm() { + return fmt.Errorf("file does not have write perm") + } + if hasReadPerm(flag) && !inode.hasReadPerm() { + return fmt.Errorf("file does not have read perm") + } + return nil +} + +func (f *FakeFS) OpenFile(name string, flag int, perm os.FileMode) (*File, error) { + dirPath := filepath.Dir(name) + dir, dirExist := f.inodes[dirPath] + if !dirExist || !dir.isDirectory { + return nil, fmt.Errorf("dir not exist") + } + inode, ok := f.inodes[name] + if !ok { + if !isCreate(flag) { + return nil, fmt.Errorf("no such file %q", name) + } + // TODO: check directory perms + inode = filePool.Get().(*mockData) + inode.reset() + inode.realName = name + inode.perm = perm + if !isCreate(flag) { // read and write allowed with any perm if you just created the file + if err := checkOpenPerm(flag, inode); err != nil { + return nil, err + } + } + f.inodes[name] = inode + dir.dirContent = append(dir.dirContent, inode) + } else { + if err := checkOpenPerm(flag, inode); err != nil { + return nil, err + } + if isCreate(flag) && isExclusive(flag) { + return nil, fmt.Errorf("file already exist") + } + if isTruncate(flag) { + if !inode.hasWritePerm() { + return nil, fmt.Errorf("file does not have write perm") + } + clear(inode.buff) + inode.buff = inode.buff[:0] + } + } + + return &File{ + mockFile: &FakeFile{ + name: name, + data: inode, + flag: flag, + fs: f, + }, + }, nil +} + +func (f *FakeFS) Chdir(dir string) error { panic("TODO") } +func (f *FakeFS) Chmod(name string, mode os.FileMode) error { panic("TODO") } +func (f *FakeFS) Chown(name string, uid, gid int) error { panic("TODO") } + +func (f *FakeFS) Mkdir(name string, perm os.FileMode) error { + parentPath := filepath.Dir(name) + parent, parentExist := f.inodes[parentPath] + if !parentExist || !parent.isDirectory { + return fmt.Errorf("parent %q not exist", parentPath) + } + + if _, exist := f.inodes[name]; exist { + return fmt.Errorf("already exist") + } + + inode := &mockData{ + realName: name, + isDirectory: true, + perm: perm, + } + f.inodes[name] = inode + parent.dirContent = append(parent.dirContent, inode) + return nil +} + +func (f *FakeFS) MkdirAll(path string, perm os.FileMode) error { panic("TODO") } +func (f *FakeFS) MkdirTemp(dir, pattern string) (string, error) { panic("TODO") } + +func (f *FakeFS) ReadFile(name string) ([]byte, error) { + fp, err := f.Open(name) + if err != nil { + return nil, err + } + defer fp.Close() + + return io.ReadAll(fp) +} + +func (f *FakeFS) ReadDir(name string) ([]os.DirEntry, error) { + panic("todo") +} + +func (f *FakeFS) Readlink(name string) (string, error) { panic("TODO") } +func (f *FakeFS) Remove(name string) error { panic("TODO") } +func (f *FakeFS) RemoveAll(path string) error { panic("TODO") } + +func (f *FakeFS) Rename(oldpath, newpath string) error { + inode, ok := f.inodes[oldpath] + if !ok { + return fmt.Errorf("file not exist (%s)", oldpath) + } + dir := filepath.Dir(newpath) + dirNode, ok := f.inodes[dir] + if !ok { + return fmt.Errorf("directory on new path does not exis (%s)", newpath) + } + + if !dirNode.isDirectory { + return fmt.Errorf("%s is not a directory", dir) + } + + delete(f.inodes, oldpath) + f.inodes[newpath] = inode + inode.realName = newpath + return nil +} + +func (f *FakeFS) Truncate(name string, size int64) error { + inode, ok := f.inodes[name] + if !ok { + return fmt.Errorf("file not exist") + } + if !inode.hasWritePerm() { + return fmt.Errorf("file does not have write perm") + } + // TODO: check write permission + inode.buff = resizeSlice(inode.buff, int(size)) + clear(inode.buff[len(inode.buff):cap(inode.buff)]) + return nil +} + +func (f *FakeFS) WriteFile(name string, data []byte, perm os.FileMode) error { + fp, err := f.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + _, err = fp.Write(data) + if err1 := fp.Close(); err1 != nil && err == nil { + err = err1 + } + return err +} + +// Release return all memory to pool and clear file map. All File structs should be destroyed by this moment, +// accessing them is UB. This function is useful, for example in end of a test +func (f *FakeFS) Release() { + for _, v := range f.inodes { + if v != nil { + filePool.Put(v) + } + } + clear(f.inodes) +} diff --git a/memory_test.go b/memory_test.go index ada7443..a08487b 100644 --- a/memory_test.go +++ b/memory_test.go @@ -1,7 +1,437 @@ package gofs -import "testing" +import ( + "bytes" + "cmp" + "crypto/rand" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "testing" -func TestTmp (t *testing.T) { + "github.com/stretchr/testify/require" + "pgregory.net/rapid" +) + +func checkSyncError(t *rapid.T, errOs error, errFake error) { + t.Helper() + if (errOs != nil) != (errFake != nil) { + t.Fatalf("os and fake impl produce different error os:%q fake=%q", errOs, errFake) + } +} + +func listOsFiles(dirPath string) { + files, err := os.ReadDir(dirPath) + if err != nil { + panic(err) + } + + for _, file := range files { + fmt.Println(file.Name(), file.IsDir()) + } +} + +func TestFS(t *testing.T) { + dir := t.TempDir() + possibleFilenames := []string{"/a/test.file.1", "/a/test.file.2", "/b/test.file.1", "/b/test.file.2"} + possibleDirs := []string{"/a", "/b"} + + rapid.Check(t, func(t *rapid.T) { + fs := NewMemoryFs() + var osFiles []*File + var fakeFiles []*File + fileCount := 0 + + defer func() { + for i := range osFiles { + osFiles[i].Close() + } + os.RemoveAll(filepath.Join(dir, "a")) + os.RemoveAll(filepath.Join(dir, "b")) + fs.Release() + }() + err := fs.Mkdir("/a", 0777) + require.NoError(t, err) + err = os.MkdirAll(filepath.Join(dir, "a"), 0777) + require.NoError(t, err) + createFiles := func() { + fpOs, err := os.Create(filepath.Join(dir, possibleFilenames[0])) + require.NoError(t, err) + fpFake, err := fs.Create(filepath.Join("/", possibleFilenames[0])) + require.NoError(t, err) + + osFiles = append(osFiles, NewFromOs(fpOs)) + fakeFiles = append(fakeFiles, fpFake) + fileCount++ + } + createFiles() + + getFiles := func() (*File, *File) { + i := rapid.IntRange(0, len(osFiles)-1).Draw(t, "file index") + return osFiles[i], fakeFiles[i] + } + + t.Repeat(map[string]func(*rapid.T){ + "write": func(t *rapid.T) { + fpOs, fpFake := getFiles() + n := rapid.IntRange(0, 1024).Draw(t, "write size") + buff := make([]byte, n) + rand.Read(buff) + nOs, errOs := fpOs.Write(buff) + nFake, errFake := fpFake.Write(buff) + checkSyncError(t, errOs, errFake) + if nOs != nFake { + t.Fatalf("os impl return %d, we %d", nOs, nFake) + } + }, + "writeAt": func(t *rapid.T) { + fpOs, fpFake := getFiles() + offset := rapid.Int64Range(-1, 1024).Draw(t, "write at offset") + n := rapid.IntRange(0, 1024).Draw(t, "write at size") + buff := make([]byte, n) + rand.Read(buff) + nOs, errOs := fpOs.WriteAt(buff, offset) + nFake, errFake := fpFake.WriteAt(buff, offset) + checkSyncError(t, errOs, errFake) + if nOs != nFake { + t.Fatalf("os impl return %d, we %d", nOs, nFake) + } + }, + "read": func(t *rapid.T) { + fpOs, fpFake := getFiles() + n := rapid.IntRange(0, 10*1024).Draw(t, "read size") + buffOs := make([]byte, n) + buffFake := make([]byte, n) + nOs, errOs := fpOs.Read(buffOs) + nFake, errFake := fpFake.Read(buffFake) + checkSyncError(t, errOs, errFake) + require.Equal(t, nOs, nFake) + require.Equal(t, buffOs[:nOs], buffFake[:nFake]) + }, + "readAt": func(t *rapid.T) { + fpOs, fpFake := getFiles() + offset := rapid.Int64().Draw(t, "read at offset") + n := rapid.IntRange(0, 10*1024).Draw(t, "read size") + buffOs := make([]byte, n) + buffFake := make([]byte, n) + nOs, errOs := fpOs.ReadAt(buffOs, offset) + nFake, errFake := fpFake.ReadAt(buffFake, offset) + checkSyncError(t, errOs, errFake) + require.Equal(t, nOs, nFake) + require.Equal(t, buffOs[:nOs], buffFake[:nFake]) + }, + "Chmod": func(t *rapid.T) { + // we do not check execute permission + // we do not check permission for different user groups + possibleModes := []os.FileMode{0666, 0222, 0444} // rw, w-only, r-only + fpOs, fpFake := getFiles() + mode := rapid.SampledFrom(possibleModes).Draw(t, "file mode") + errOs := fpOs.Chmod(mode) + errFake := fpFake.Chmod(mode) + checkSyncError(t, errOs, errFake) + }, + // //"Chown": func(t *rapid.T) {}, // TODO: ?? + // //"Chdir": func(t *rapid.T) {}, // TODO: ?? + "Close": func(t *rapid.T) { + fpOs, fpFake := getFiles() + errOs := fpOs.Close() + errFake := fpFake.Close() + checkSyncError(t, errOs, errFake) + }, + "ReadDir": func(t *rapid.T) { + fpOs, fpFake := getFiles() + n := rapid.IntRange(-1, 3).Draw(t, "readDir n") + diOs, errOs := fpOs.ReadDir(n) + diFake, errFake := fpFake.ReadDir(n) + checkSyncError(t, errOs, errFake) + slices.SortFunc(diOs, func(a os.DirEntry, b os.DirEntry) int { return cmp.Compare(a.Name(), b.Name()) }) + slices.SortFunc(diFake, func(a os.DirEntry, b os.DirEntry) int { return cmp.Compare(a.Name(), b.Name()) }) + require.Equal(t, len(diOs), len(diFake)) + for i := range diOs { + require.Equal(t, diOs[i].Name(), diFake[i].Name()) + require.Equal(t, diOs[i].IsDir(), diFake[i].IsDir()) + infoOs, errOs := diOs[i].Info() + infoFake, errFake := diFake[i].Info() + checkSyncError(t, errOs, errFake) + if errOs != nil { + require.Equal(t, infoOs.Size(), infoFake.Size()) + require.Equal(t, infoOs.Mode(), infoFake.Mode()) + } + } + }, + "ReadFrom": func(t *rapid.T) { + fpOs, fpFake := getFiles() + var buffOs bytes.Buffer + var buffFake bytes.Buffer + nOs, errOs := fpOs.ReadFrom(&buffOs) + nFake, errFake := fpFake.ReadFrom(&buffFake) + checkSyncError(t, errOs, errFake) + require.Equal(t, nOs, nFake) + require.Equal(t, buffOs.Bytes(), buffFake.Bytes()) + }, + "Readdir": func(t *rapid.T) { + fpOs, fpFake := getFiles() + n := rapid.IntRange(-1, 3).Draw(t, "readdir n") + infoOs, errOs := fpOs.Readdir(n) + infoFake, errFake := fpFake.Readdir(n) + checkSyncError(t, errOs, errFake) + slices.SortFunc(infoOs, func(a os.FileInfo, b os.FileInfo) int { return cmp.Compare(a.Name(), b.Name()) }) + slices.SortFunc(infoFake, func(a os.FileInfo, b os.FileInfo) int { return cmp.Compare(a.Name(), b.Name()) }) + require.Equal(t, len(infoOs), len(infoFake)) + for i := range infoOs { + require.Equal(t, infoOs[i].Name(), infoFake[i].Name()) + require.Equal(t, infoOs[i].IsDir(), infoFake[i].IsDir()) + require.Equal(t, infoOs[i].Size(), infoFake[i].Size()) + require.Equal(t, infoOs[i].Mode(), infoFake[i].Mode()) + } + }, + "Readdirnames": func(t *rapid.T) { + fpOs, fpFake := getFiles() + n := rapid.IntRange(-1, 3).Draw(t, "readdir n") + infoOs, errOs := fpOs.Readdirnames(n) + infoFake, errFake := fpFake.Readdirnames(n) + checkSyncError(t, errOs, errFake) + slices.Sort(infoOs) + slices.Sort(infoFake) + require.Equal(t, infoOs, infoFake) + }, + "Seek": func(t *rapid.T) { + fpOs, fpFake := getFiles() + offset := rapid.Int64Range(-1, 1024).Draw(t, "seek offset") + whence := rapid.SampledFrom([]int{io.SeekStart, io.SeekCurrent, io.SeekEnd}).Draw(t, "seek whence") + retOs, errOs := fpOs.Seek(offset, whence) + retFake, errFake := fpFake.Seek(offset, whence) + checkSyncError(t, errOs, errFake) + require.Equal(t, retOs, retFake) + }, + "Stat": func(t *rapid.T) { + fpOs, fpFake := getFiles() + fiOs, errOs := fpOs.Stat() + fiFake, errFake := fpFake.Stat() + checkSyncError(t, errOs, errFake) + if fiOs != nil { + require.Equal(t, fiOs.Name(), fiFake.Name()) + require.Equal(t, fiOs.IsDir(), fiFake.IsDir()) + require.Equal(t, fiOs.Size(), fiFake.Size()) + // we do not compare mode, since it depends on parent fs directory, so hard to check in property test + // We do not compare time, since it's hard to mock, and not really relevant + } + }, + "Sync": func(t *rapid.T) { + fpOs, fpFake := getFiles() + errOs := fpOs.Sync() + errFake := fpFake.Sync() + checkSyncError(t, errOs, errFake) + }, + "Truncate": func(t *rapid.T) { + fpOs, fpFake := getFiles() + offset := rapid.Int64Range(-1, 1024).Draw(t, "truncate offset") + errOs := fpOs.Truncate(offset) + errFake := fpFake.Truncate(offset) + checkSyncError(t, errOs, errFake) + }, + "FS_Create": func(t *rapid.T) { + p := rapid.SampledFrom(possibleFilenames).Draw(t, "file to create") + fpOs, errOs := os.Create(filepath.Join(dir, p)) + fpFake, errFake := fs.Create(filepath.Join("/", p)) + checkSyncError(t, errOs, errFake) + if errOs == nil { + osFiles = append(osFiles, NewFromOs(fpOs)) + fakeFiles = append(fakeFiles, fpFake) + } + }, + // "FS_CreateTemp": func(t *rapid.T) {}, + "FS_Open": func(t *rapid.T) { + p := rapid.SampledFrom(possibleFilenames).Draw(t, "file to create") + fpOs, errOs := os.Open(filepath.Join(dir, p)) + fpFake, errFake := fs.Open(filepath.Join("/", p)) + checkSyncError(t, errOs, errFake) + if errOs == nil { + osFiles = append(osFiles, NewFromOs(fpOs)) + fakeFiles = append(fakeFiles, fpFake) + } + }, + "FS_OpenFile": func(t *rapid.T) { + p := rapid.SampledFrom(possibleFilenames).Draw(t, "file to open") + possibleModes := []os.FileMode{0666, 0222, 0444} // rw, w-only, r-only + perm := rapid.SampledFrom(possibleModes).Draw(t, "file perm") + flagMap := map[string]int{"readonly": os.O_RDONLY, "writeonly": os.O_WRONLY, "RDWR": os.O_RDWR} + possibleFlags := []string{"readonly", "writeonly", "RDWR"} + flag := flagMap[rapid.SampledFrom(possibleFlags).Draw(t, "file mode")] + //if rapid.Bool().Draw(t, "append bit") { + // flag |= os.O_APPEND + //} + if rapid.Bool().Draw(t, "create bit") { + flag |= os.O_CREATE + } + if rapid.Bool().Draw(t, "excl bit") { + flag |= os.O_EXCL + } + if rapid.Bool().Draw(t, "trunc bit") { + flag |= os.O_TRUNC + } + + fpOs, errOs := os.OpenFile(filepath.Join(dir, p), flag, perm) + fpFake, errFake := fs.OpenFile(filepath.Join("/", p), flag, perm) + checkSyncError(t, errOs, errFake) + if errOs == nil { + osFiles = append(osFiles, NewFromOs(fpOs)) + fakeFiles = append(fakeFiles, fpFake) + } + }, + // "FS_Chdir": func(t *rapid.T) {}, + "FS_Chmod": func(t *rapid.T) { + p := rapid.SampledFrom(possibleFilenames).Draw(t, "file to create") + fpOs, errOs := os.Open(filepath.Join(dir, p)) + fpFake, errFake := fs.Open(filepath.Join("/", p)) + checkSyncError(t, errOs, errFake) + if errOs == nil { + possibleModes := []os.FileMode{0666, 0222, 0444} // rw, w-only, r-only + mode := rapid.SampledFrom(possibleModes).Draw(t, "file mode") + errOs := fpOs.Chmod(mode) + errFake := fpFake.Chmod(mode) + checkSyncError(t, errOs, errFake) + } + }, + // "FS_Chown": func(t *rapid.T) {}, + "FS_Mkdir": func(t *rapid.T) { + p := rapid.SampledFrom(possibleDirs).Draw(t, "dir to create") + errOs := os.Mkdir(filepath.Join(dir, p), 0777) + errFake := fs.Mkdir(filepath.Join("/", p), 0777) + checkSyncError(t, errOs, errFake) + }, + // "FS_MkdirAll": func(t *rapid.T) {}, + // "FS_MkdirTemp": func(t *rapid.T) {}, + "FS_ReadFile": func(t *rapid.T) { + p := rapid.SampledFrom(possibleFilenames).Draw(t, "file to read") + contOs, errOs := os.ReadFile(filepath.Join(dir, p)) + contFake, errFake := fs.ReadFile(filepath.Join("/", p)) + checkSyncError(t, errOs, errFake) + require.Equal(t, contOs, contFake) + }, + // "FS_Readlink": func(t *rapid.T) {}, + // "FS_Remove": func(t *rapid.T) {}, + // "FS_RemoveAll": func(t *rapid.T) {}, + "FS_Rename": func(t *rapid.T) { + oldname := rapid.SampledFrom(possibleFilenames).Draw(t, "file to write") + newname := rapid.SampledFrom(possibleFilenames).Draw(t, "file to write") + + oldOsPath := filepath.Join(dir, oldname) + newOsPath := filepath.Join(dir, newname) + oldFakePath := filepath.Join("/", oldname) + newFakePath := filepath.Join("/", newname) + + errOs := os.Rename(oldOsPath, newOsPath) + errFake := fs.Rename(oldFakePath, newFakePath) + checkSyncError(t, errOs, errFake) + }, + "FS_Truncate": func(t *rapid.T) { + p := rapid.SampledFrom(possibleFilenames).Draw(t, "file to write") + n := rapid.Int64Range(0, 1024).Draw(t, "truncate size") + errOs := os.Truncate(filepath.Join(dir, p), n) + errFake := fs.Truncate(filepath.Join("/", p), n) + checkSyncError(t, errOs, errFake) + }, + "FS_WriteFile": func(t *rapid.T) { + p := rapid.SampledFrom(possibleFilenames).Draw(t, "file to write") + n := rapid.IntRange(0, 1024).Draw(t, "write size") + buff := make([]byte, n) + rand.Read(buff) + errOs := os.WriteFile(filepath.Join(dir, p), buff, 0777) + errFake := fs.WriteFile(filepath.Join("/", p), buff, 0777) + checkSyncError(t, errOs, errFake) + }, + }) + }) +} + +func TestTmp(t *testing.T) { + dir := t.TempDir() + fs := NewMemoryFs() + var osFiles []*File + var fakeFiles []*File + fileCount := 0 + + defer func() { + for i := range osFiles { + osFiles[i].Close() + } + os.RemoveAll(filepath.Join(dir, "a")) + os.RemoveAll(filepath.Join(dir, "b")) + fs.Release() + }() + err := fs.Mkdir("/a", 0777) + require.NoError(t, err) + err = os.MkdirAll(filepath.Join(dir, "a"), 0777) + require.NoError(t, err) + createFiles := func() { + fpOs, err := os.Create(filepath.Join(dir, "/a/file.test")) + require.NoError(t, err) + fpFake, err := fs.Create(filepath.Join("/", "/a/file.test")) + require.NoError(t, err) + + osFiles = append(osFiles, NewFromOs(fpOs)) + fakeFiles = append(fakeFiles, fpFake) + fileCount++ + } + createFiles() + + p := "/a/file.test" + fpOs, errOs := os.Open(filepath.Join(dir, p)) + fpFake, errFake := fs.Open(filepath.Join("/", p)) + fmt.Printf("oserr:%q, fakeErr:%q\n", errOs, errFake) + + fmt.Println("chmod") + mode := os.FileMode(0222) + errOs = fpOs.Chmod(mode) + errFake = fpFake.Chmod(mode) + fmt.Printf("oserr:%q, fakeErr:%q\n", errOs, errFake) + + fmt.Println("writeat") + offset := int64(0) + n := 1 + buff := make([]byte, n) + rand.Read(buff) + nOs, errOs := fpOs.WriteAt(buff, offset) + nFake, errFake := fpFake.WriteAt(buff, offset) + fmt.Printf("oserr:%q, fakeErr:%q\n", errOs, errFake) + + fmt.Println("FS_create") + fpOs, errOs = os.Create(filepath.Join(dir, p)) + fpFake, errFake = fs.Create(filepath.Join("/", p)) + fmt.Printf("oserr:%q, fakeErr:%q\n", errOs, errFake) + + n = 1 + buff = make([]byte, n) + rand.Read(buff) + nOs, errOs = fpOs.Write(nil) + nFake, errFake = fpFake.Write(nil) + fmt.Printf("oserr:%q, fakeErr:%q\n", errOs, errFake) + if nOs != nFake { + t.Fatalf("os impl return %d, we %d", nOs, nFake) + } +} + +func TestTmp2(t *testing.T) { + dir := t.TempDir() + //dir := "/Users/myxo/tmp" + path := filepath.Join(dir, "test2") + fp, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0444) + fmt.Println("open", err) + buff := []byte("hello") + _, err = fp.Read(buff) + fmt.Println("read", err) + _, err = fp.WriteAt(buff, 0) + fmt.Println("write", err) + n, err := fp.ReadAt(buff, 0) + fmt.Println("read", n, err, buff[:n]) + fmt.Println("close") + fp.Close() + fp, err = os.OpenFile(path, os.O_RDONLY, 0222) + fmt.Println("open", err) + n, err = fp.ReadAt(buff, 0) + fmt.Println("read", n, err, buff[:n]) } diff --git a/util.go b/util.go new file mode 100644 index 0000000..acde4ac --- /dev/null +++ b/util.go @@ -0,0 +1,55 @@ +package gofs + +import "os" + +func resizeSlice(s []byte, size int) []byte { + if size < 0 { + panic("negative size") + } + if size > 1024*1024 { + panic(size) + } + if cap(s) < size { + s = append(s[:cap(s)], make([]byte, size-cap(s))...) + } else { + s = s[:size] + } + return s +} + +func isWriteOnly(flag int) bool { + return flag&os.O_WRONLY != 0 +} + +func isReadOnly(flag int) bool { + const mask = 0x3 + return flag&mask == 0 // read obly is an absend of bits +} + +func isReadWrite(flag int) bool { + return flag&os.O_RDWR != 0 +} + +func hasWritePerm(flag int) bool { + return isWriteOnly(flag) || isReadWrite(flag) +} + +func hasReadPerm(flag int) bool { + return isReadOnly(flag) || isReadWrite(flag) +} + +func isAppend(flag int) bool { + return flag&os.O_APPEND != 0 +} + +func isCreate(flag int) bool { + return flag&os.O_CREATE != 0 +} + +func isExclusive(flag int) bool { + return flag&os.O_EXCL != 0 +} + +func isTruncate(flag int) bool { + return flag&os.O_TRUNC != 0 +} diff --git a/util_test.go b/util_test.go new file mode 100644 index 0000000..12300ae --- /dev/null +++ b/util_test.go @@ -0,0 +1,21 @@ +package gofs + +import ( + "testing" + + "pgregory.net/rapid" +) + +func TestResize(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + origLen := rapid.IntRange(0, 100).Draw(t, "orig len") + capacity := rapid.IntRange(origLen, 100).Draw(t, "capacity") + newLen := rapid.IntRange(0, 100).Draw(t, "orig len") + + buff := make([]byte, origLen, capacity) + newBuff := resizeSlice(buff, newLen) + if len(newBuff) != newLen { + t.Fail() + } + }) +}