From 478877b8946934832cc9ce59c059994b4f93b4cb Mon Sep 17 00:00:00 2001 From: Evan Date: Thu, 9 Apr 2020 18:44:20 -0400 Subject: [PATCH 1/3] pager checkreserved should use the reserved byte offset --- db/pager_unix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/pager_unix.go b/db/pager_unix.go index 7b86eaa..10725b9 100644 --- a/db/pager_unix.go +++ b/db/pager_unix.go @@ -106,7 +106,7 @@ func (f *filePager) CheckReservedLock() (bool, error) { lock := &unix.Flock_t{ Type: unix.F_WRLCK, Whence: seek_set, - Start: sqlite_shared_first, + Start: sqlite_reserved_byte, Len: 1, } err := unix.FcntlFlock(f.f.Fd(), unix.F_GETLK, lock) From 3b19a4681eea2a288513e14c7e49b8bcf5bcf686 Mon Sep 17 00:00:00 2001 From: Evan Date: Thu, 9 Apr 2020 18:46:05 -0400 Subject: [PATCH 2/3] add a pager for windows --- db/pager_unix.go | 2 + db/pager_windows.go | 144 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 db/pager_windows.go diff --git a/db/pager_unix.go b/db/pager_unix.go index 10725b9..ca21c79 100644 --- a/db/pager_unix.go +++ b/db/pager_unix.go @@ -1,3 +1,5 @@ +// +build !windows + // unix implementation of the `pager` interface (the file reader) with POSIX // advisory locking diff --git a/db/pager_windows.go b/db/pager_windows.go new file mode 100644 index 0000000..be7d802 --- /dev/null +++ b/db/pager_windows.go @@ -0,0 +1,144 @@ +// windows implementation of the `pager` interface (the file reader) with LockFileEx +// locking + +// references: +// winapi docs: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-lockfileex +// sqlite3 winlock: https://github.com/sqlite/sqlite/blob/c398c65bee850b6b8f24a44852872a27f114535d/src/os_win.c#L3236 + +package db + +import ( + "errors" + "fmt" + "os" + "time" + + "golang.org/x/exp/mmap" + "golang.org/x/sys/windows" +) + +const ( + sqlite_pending_byte = 0x40000000 + sqlite_reserved_byte = sqlite_pending_byte + 1 + sqlite_shared_first = sqlite_pending_byte + 2 + sqlite_shared_size = 510 +) + +const ( + reserved uint32 = 0 +) + +// LockState is used to represent the program's opinion of whether a file is locked or not +// this does not represent the lock state of the file - we can't distinguish between +// a file that is locked by another process and failing to acquire a lock +type LockState int + +const ( + Unlocked LockState = iota + Locked +) + +type filePager struct { + f *os.File + mm *mmap.ReaderAt + state LockState +} + +func newFilePager(file string) (*filePager, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + mm, err := mmap.Open(file) + if err != nil { + f.Close() + return nil, err + } + return &filePager{ + f: f, + mm: mm, + state: Unlocked, + }, nil +} + +// pages start counting at 1 +func (f *filePager) page(id int, pagesize int) ([]byte, error) { + buf := make([]byte, pagesize) + _, err := f.mm.ReadAt(buf[:], int64(id-1)*int64(pagesize)) + return buf, err +} + +func (f *filePager) lock(start, len uint32) error { + ol := new(windows.Overlapped) + ol.Offset = start + return windows.LockFileEx(windows.Handle(f.f.Fd()), windows.LOCKFILE_FAIL_IMMEDIATELY|windows.LOCKFILE_EXCLUSIVE_LOCK, reserved, len, 0, ol) +} + +func (f *filePager) unlock(start, len uint32) error { + ol := new(windows.Overlapped) + ol.Offset = start + return windows.UnlockFileEx(windows.Handle(f.f.Fd()), reserved, len, 0, ol) +} + +func (f *filePager) RLock() error { + // Set a 'SHARED' lock, following winLock() logic from sqlite3.c + if f.state == Locked { + return errors.New("trying to lock a locked lock") // panic? + } + + // Try 3 times to get the pending lock. This is needed to work + // around problems caused by indexing and/or anti-virus software on + // Windows systems. + // If you are using this code as a model for alternative VFSes, do not + // copy this retry logic. It is a hack intended for Windows only. + // + // source: https://github.com/sqlite/sqlite/blob/c398c65bee850b6b8f24a44852872a27f114535d/src/os_win.c#L3280 + for i := 0; i < 3; i++ { + err := f.lock(sqlite_pending_byte, 1) + if err == nil { + break + } + if errors.Is(err, windows.ERROR_INVALID_HANDLE) { + return fmt.Errorf("failed to get lock, invalid handle") + } + time.Sleep(time.Microsecond) + } + + defer func() { + // - drop the pending lock. No idea what to do with the error :/ + f.unlock(sqlite_pending_byte, 1) + }() + + // Get the read-lock + if err := f.lock(sqlite_shared_first, sqlite_shared_size); err != nil { + return err + } + f.state = Locked + return nil +} + +func (f *filePager) RUnlock() error { + if f.state == Unlocked { + return errors.New("trying to unlock an unlocked lock") // panic? + } + if err := f.unlock(sqlite_shared_first, sqlite_shared_size); err != nil { + return err + } + f.state = Unlocked + return nil +} + +// True if there is a 'reserved' lock on the database, by any process. +func (f *filePager) CheckReservedLock() (bool, error) { + // per SQLite's winCheckReservedLock() + err := f.lock(sqlite_reserved_byte, 1) + if err == nil { + f.unlock(sqlite_reserved_byte, 1) + } + return err == nil, err +} + +func (f *filePager) Close() error { + f.f.Close() + return f.mm.Close() +} From 8532e881a37e43148cd841016a60c7e4e1014fb0 Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 10 Apr 2020 10:34:28 -0400 Subject: [PATCH 3/3] refactor sqlite locking constants constants are shared between os lock implementations, so move them out of os-specific files also camelcases them --- db/pager.go | 7 +++++++ db/pager_unix.go | 20 ++++++++------------ db/pager_windows.go | 19 ++++++------------- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/db/pager.go b/db/pager.go index 03d037f..bdcf2ad 100644 --- a/db/pager.go +++ b/db/pager.go @@ -1,5 +1,12 @@ package db +const ( + sqlitePendingByte = 0x40000000 + sqliteReservedByte = sqlitePendingByte + 1 + sqliteSharedFirst = sqlitePendingByte + 2 + sqliteSharedSize = 510 +) + type pager interface { // load a page from storage. page(n int, pagesize int) ([]byte, error) diff --git a/db/pager_unix.go b/db/pager_unix.go index ca21c79..4db7dc5 100644 --- a/db/pager_unix.go +++ b/db/pager_unix.go @@ -14,11 +14,7 @@ import ( ) const ( - seek_set = 0 // should be defined in syscall - sqlite_pending_byte = 0x40000000 - sqlite_reserved_byte = sqlite_pending_byte + 1 - sqlite_shared_first = sqlite_pending_byte + 2 - sqlite_shared_size = 510 + seekSet = 0 // should be defined in syscall ) type filePager struct { @@ -64,8 +60,8 @@ func (f *filePager) RLock() error { // - get PENDING lock pending := &unix.Flock_t{ Type: unix.F_RDLCK, - Whence: seek_set, - Start: sqlite_pending_byte, + Whence: seekSet, + Start: sqlitePendingByte, Len: 1, } if err := f.lock(pending); err != nil { @@ -81,9 +77,9 @@ func (f *filePager) RLock() error { // Get the read-lock read := &unix.Flock_t{ Type: unix.F_RDLCK, - Whence: seek_set, - Start: sqlite_shared_first, - Len: sqlite_shared_size, + Whence: seekSet, + Start: sqliteSharedFirst, + Len: sqliteSharedSize, } if err := f.lock(read); err != nil { return err @@ -107,8 +103,8 @@ func (f *filePager) CheckReservedLock() (bool, error) { // per SQLite's unixCheckReservedLock() lock := &unix.Flock_t{ Type: unix.F_WRLCK, - Whence: seek_set, - Start: sqlite_reserved_byte, + Whence: seekSet, + Start: sqliteReservedByte, Len: 1, } err := unix.FcntlFlock(f.f.Fd(), unix.F_GETLK, lock) diff --git a/db/pager_windows.go b/db/pager_windows.go index be7d802..ba4e500 100644 --- a/db/pager_windows.go +++ b/db/pager_windows.go @@ -17,13 +17,6 @@ import ( "golang.org/x/sys/windows" ) -const ( - sqlite_pending_byte = 0x40000000 - sqlite_reserved_byte = sqlite_pending_byte + 1 - sqlite_shared_first = sqlite_pending_byte + 2 - sqlite_shared_size = 510 -) - const ( reserved uint32 = 0 ) @@ -94,7 +87,7 @@ func (f *filePager) RLock() error { // // source: https://github.com/sqlite/sqlite/blob/c398c65bee850b6b8f24a44852872a27f114535d/src/os_win.c#L3280 for i := 0; i < 3; i++ { - err := f.lock(sqlite_pending_byte, 1) + err := f.lock(sqlitePendingByte, 1) if err == nil { break } @@ -106,11 +99,11 @@ func (f *filePager) RLock() error { defer func() { // - drop the pending lock. No idea what to do with the error :/ - f.unlock(sqlite_pending_byte, 1) + f.unlock(sqlitePendingByte, 1) }() // Get the read-lock - if err := f.lock(sqlite_shared_first, sqlite_shared_size); err != nil { + if err := f.lock(sqliteSharedFirst, sqliteSharedSize); err != nil { return err } f.state = Locked @@ -121,7 +114,7 @@ func (f *filePager) RUnlock() error { if f.state == Unlocked { return errors.New("trying to unlock an unlocked lock") // panic? } - if err := f.unlock(sqlite_shared_first, sqlite_shared_size); err != nil { + if err := f.unlock(sqliteSharedFirst, sqliteSharedSize); err != nil { return err } f.state = Unlocked @@ -131,9 +124,9 @@ func (f *filePager) RUnlock() error { // True if there is a 'reserved' lock on the database, by any process. func (f *filePager) CheckReservedLock() (bool, error) { // per SQLite's winCheckReservedLock() - err := f.lock(sqlite_reserved_byte, 1) + err := f.lock(sqliteReservedByte, 1) if err == nil { - f.unlock(sqlite_reserved_byte, 1) + f.unlock(sqliteReservedByte, 1) } return err == nil, err }