diff --git a/.gitignore b/.gitignore index f1c181e..554df3a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out + +# IDE files +.idea/* diff --git a/.travis.yml b/.travis.yml index 4c65ea5..57e1681 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,12 +2,14 @@ language: go go: - 1.12 - - 1.11 os: - osx - linux +env: + - GO111MODULE=on + script: - go test -race -coverprofile=coverage.txt -covermode=atomic diff --git a/immap.go b/immap.go deleted file mode 100644 index ff4bcf5..0000000 --- a/immap.go +++ /dev/null @@ -1,17 +0,0 @@ -package mmap - -import "io" - -// IMmap provides an interface to a memory mapped file -type IMmap interface { - io.ReaderAt - io.WriterAt - - Lock() error - Unlock() error - Advise(advice int) error - ReadUint64At(offset int64) uint64 - WriteUint64At(num uint64, offset int64) - Flush(flags int) error - Unmap() error -} diff --git a/mmap.go b/mmap.go index cd6e67e..8471f1f 100644 --- a/mmap.go +++ b/mmap.go @@ -2,7 +2,9 @@ package mmap import ( "errors" + "io" "os" + "strings" "syscall" ) @@ -13,8 +15,25 @@ var ( ErrIndexOutOfBound = errors.New("offset out of mapped region") ) -// Mmap provides abstraction around a memory mapped file -type Mmap struct { +// File provides an interface to a memory mapped file +type File interface { + io.ReaderAt + io.WriterAt + + ReadStringAt(dest *strings.Builder, offset int64) int + WriteStringAt(src string, offset int64) int + ReadUint64At(offset int64) uint64 + WriteUint64At(num uint64, offset int64) + + Lock() error + Unlock() error + Advise(advice int) error + Flush(flags int) error + Unmap() error +} + +// mmapFile provides abstraction around a memory mapped file +type mmapFile struct { data []byte length int64 } @@ -26,13 +45,13 @@ type Mmap struct { // then all the mapped memory is accessible // case 2 => if file size <= memory region (offset + length) // then from offset to file size memory region is accessible -func NewSharedFileMmap(f *os.File, offset int64, length int, prot int) (IMmap, error) { +func NewSharedFileMmap(f *os.File, offset int64, length int, prot int) (File, error) { data, err := syscall.Mmap(int(f.Fd()), offset, length, prot, syscall.MAP_SHARED) if err != nil { return nil, err } - return &Mmap{ + return &mmapFile{ data: data, length: int64(length), }, nil @@ -40,7 +59,7 @@ func NewSharedFileMmap(f *os.File, offset int64, length int, prot int) (IMmap, e // Unmap unmaps the memory mapped file. An error will be returned // if any of the functions are called on Mmap after calling Unmap -func (m *Mmap) Unmap() error { +func (m *mmapFile) Unmap() error { err := syscall.Munmap(m.data) m.data = nil return err diff --git a/mmap_data.go b/mmap_data.go index 7dff9a1..efabd8c 100644 --- a/mmap_data.go +++ b/mmap_data.go @@ -2,72 +2,86 @@ package mmap import ( "encoding/binary" + "strings" "syscall" "unsafe" ) -// ReadAt copies data to dest slice from mapped region starting at -// given offset and returns number of bytes copied to the dest slice. -// There are two possibilities - -// Case 1: len(dest) >= (len(m.data) - offset) -// => copies (len(m.data) - offset) bytes to dest from mapped region -// Case 2: len(dest) < (len(m.data) - offset) -// => copies len(dest) bytes to dest from mapped region -// err is always nil, hence, can be ignored -func (m *Mmap) ReadAt(dest []byte, offset int64) (int, error) { +func (m *mmapFile) boundaryChecks(offset, numBytes int64) { if m.data == nil { panic(ErrUnmappedMemory) - } else if offset >= m.length || offset < 0 { + } else if offset+numBytes > m.length || offset < 0 { panic(ErrIndexOutOfBound) } +} +// ReadAt copies data to dest slice from mapped region starting at +// given offset and returns number of bytes copied to the dest slice. +// There are two possibilities - +// Case 1: len(dest) >= (m.length - offset) +// => copies (m.length - offset) bytes to dest from mapped region +// Case 2: len(dest) < (m.length - offset) +// => copies len(dest) bytes to dest from mapped region +// err is always nil, hence, can be ignored +func (m *mmapFile) ReadAt(dest []byte, offset int64) (int, error) { + m.boundaryChecks(offset, 1) return copy(dest, m.data[offset:]), nil } // WriteAt copies data to mapped region from the src slice starting at // given offset and returns number of bytes copied to the mapped region. // There are two possibilities - -// Case 1: len(src) >= (len(m.data) - offset) -// => copies (len(m.data) - offset) bytes to the mapped region from src -// Case 2: len(src) < (len(m.data) - offset) +// Case 1: len(src) >= (m.length - offset) +// => copies (m.length - offset) bytes to the mapped region from src +// Case 2: len(src) < (m.length - offset) // => copies len(src) bytes to the mapped region from src // err is always nil, hence, can be ignored -func (m *Mmap) WriteAt(src []byte, offset int64) (int, error) { - if m.data == nil { - panic(ErrUnmappedMemory) - } else if offset >= m.length || offset < 0 { - panic(ErrIndexOutOfBound) - } - +func (m *mmapFile) WriteAt(src []byte, offset int64) (int, error) { + m.boundaryChecks(offset, 1) return copy(m.data[offset:], src), nil } -// ReadUint64At reads uint64 from offset -func (m *Mmap) ReadUint64At(offset int64) uint64 { - if m.data == nil { - panic(ErrUnmappedMemory) - } else if offset+8 > m.length || offset < 0 { - panic(ErrIndexOutOfBound) +// ReadStringAt copies data to dest string builder from mapped region starting at +// given offset until the min value of (length - offset) or (dest.Cap() - dest.Len()) +// and returns number of bytes copied to the dest slice. +func (m *mmapFile) ReadStringAt(dest *strings.Builder, offset int64) int { + m.boundaryChecks(offset, 1) + + dataLength := m.length - offset + emptySpace := int64(dest.Cap() - dest.Len()) + end := m.length + if dataLength > emptySpace { + end = offset + emptySpace } + n, _ := dest.Write(m.data[offset:end]) + return n +} + +// WriteStringAt copies data to mapped region from the src string starting at +// given offset and returns number of bytes copied to the mapped region. +// See github.com/grandecola/mmap/#Mmap.WriteAt for more details. +func (m *mmapFile) WriteStringAt(src string, offset int64) int { + m.boundaryChecks(offset, 1) + return copy(m.data[offset:], src) +} + +// ReadUint64At reads uint64 from offset +func (m *mmapFile) ReadUint64At(offset int64) uint64 { + m.boundaryChecks(offset, 8) return binary.LittleEndian.Uint64(m.data[offset : offset+8]) } // WriteUint64At writes num at offset -func (m *Mmap) WriteUint64At(num uint64, offset int64) { - if m.data == nil { - panic(ErrUnmappedMemory) - } else if offset+8 > m.length || offset < 0 { - panic(ErrIndexOutOfBound) - } - +func (m *mmapFile) WriteUint64At(num uint64, offset int64) { + m.boundaryChecks(offset, 8) binary.LittleEndian.PutUint64(m.data[offset:offset+8], num) } // Flush flushes the memory mapped region to disk -func (m *Mmap) Flush(flags int) error { +func (m *mmapFile) Flush(flags int) error { _, _, err := syscall.Syscall(syscall.SYS_MSYNC, - uintptr(unsafe.Pointer(&m.data[0])), uintptr(len(m.data)), uintptr(flags)) + uintptr(unsafe.Pointer(&m.data[0])), uintptr(m.length), uintptr(flags)) if err != 0 { return err } diff --git a/mmap_page.go b/mmap_page.go index 1c4cd3a..3eeacca 100644 --- a/mmap_page.go +++ b/mmap_page.go @@ -6,9 +6,9 @@ import ( ) // Advise provides hints to kernel regarding the use of memory mapped region -func (m *Mmap) Advise(advice int) error { +func (m *mmapFile) Advise(advice int) error { _, _, err := syscall.Syscall(syscall.SYS_MADVISE, - uintptr(unsafe.Pointer(&m.data[0])), uintptr(len(m.data)), uintptr(advice)) + uintptr(unsafe.Pointer(&m.data[0])), uintptr(m.length), uintptr(advice)) if err != 0 { return err } @@ -17,9 +17,9 @@ func (m *Mmap) Advise(advice int) error { } // Lock locks all the mapped memory to RAM, preventing the pages from swapping out -func (m *Mmap) Lock() error { +func (m *mmapFile) Lock() error { _, _, err := syscall.Syscall(syscall.SYS_MLOCK, - uintptr(unsafe.Pointer(&m.data[0])), uintptr(len(m.data)), 0) + uintptr(unsafe.Pointer(&m.data[0])), uintptr(m.length), 0) if err != 0 { return err } @@ -28,9 +28,9 @@ func (m *Mmap) Lock() error { } // Unlock unlocks the mapped memory from RAM, enabling swapping out of RAM if required -func (m *Mmap) Unlock() error { +func (m *mmapFile) Unlock() error { _, _, err := syscall.Syscall(syscall.SYS_MUNLOCK, - uintptr(unsafe.Pointer(&m.data[0])), uintptr(len(m.data)), 0) + uintptr(unsafe.Pointer(&m.data[0])), uintptr(m.length), 0) if err != 0 { return err } diff --git a/mmap_test.go b/mmap_test.go index 4ed3bf9..26e202b 100644 --- a/mmap_test.go +++ b/mmap_test.go @@ -5,6 +5,7 @@ import ( "bytes" "io/ioutil" "os" + "strings" "syscall" "testing" ) @@ -15,22 +16,31 @@ var ( testPath = "/tmp/m.txt" ) -func init() { +func setup(t *testing.T) { f, err := os.OpenFile(testPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { - panic(err) + t.Fatalf("error in opening file :: %v", err) } if _, err := f.Write(testData); err != nil { - panic(err) + t.Fatalf("error in writing to file :: %v", err) } if err := f.Close(); err != nil { - panic(err) + t.Fatalf("error in closing file :: %v", err) + } +} + +func tearDown(t *testing.T) { + if err := os.Remove(testPath); err != nil { + t.Fatalf("error in deleting file :: %v", err) } } func TestUnmap(t *testing.T) { + setup(t) + defer tearDown(t) + f, err := os.OpenFile(testPath, os.O_RDWR, 0644) if err != nil { t.Fatalf("error in opening file :: %v", err) @@ -52,6 +62,9 @@ func TestUnmap(t *testing.T) { } func TestReadWrite(t *testing.T) { + setup(t) + defer tearDown(t) + f, errFile := os.OpenFile(testPath, os.O_RDWR, 0644) if errFile != nil { t.Fatalf("error in opening file :: %v", errFile) @@ -120,7 +133,9 @@ func TestReadWrite(t *testing.T) { if errFile != nil { t.Fatalf("error in reading file :: %s", errFile) } - f1.Close() + if err := f1.Close(); err != nil { + t.Fatalf("error in closing file :: %v", err) + } if !bytes.Equal(fileData, []byte("012345678aABCDEFGHIJKLMNOPQRSTUVWXYZ")) { t.Fatalf("no modification in file :: %v", string(fileData)) } @@ -159,7 +174,155 @@ func TestReadWrite(t *testing.T) { }() } +func TestReadStringAt(t *testing.T) { + setup(t) + defer tearDown(t) + + f, errFile := os.OpenFile(testPath, os.O_RDWR, 0644) + if errFile != nil { + t.Fatalf("error in opening file :: %v", errFile) + } + defer func() { + if err := f.Close(); err != nil { + t.Fatalf("error in closing file :: %v", err) + } + }() + + m, errMmap := NewSharedFileMmap(f, 0, len(testData), protPage) + if errMmap != nil { + t.Fatalf("error in mapping :: %v", errMmap) + } + defer func() { + if err := m.Unmap(); err != nil { + t.Fatalf("error in calling unmap :: %v", err) + } + }() + + // Read + sb := &strings.Builder{} + sb.Grow(len(testData)) + if n := m.ReadStringAt(sb, 0); n != len(testData) { + t.Fatalf("expected to read string of length %v, read of length %v", len(testData), n) + } + if string(testData) != sb.String() { + t.Fatalf("mapped data is not equal testData: %v, %v", sb.String(), string(testData)) + } + + // Read data smaller than the mapped region + sb.Reset() + sb.Grow(len(testData) - 2) + if n := m.ReadStringAt(sb, 0); n != len(testData)-2 { + t.Fatalf("expected to read string of length %v, read of length %v", len(testData), n) + } + expectedData := string(testData[:len(testData)-2]) + if expectedData != sb.String() { + t.Fatalf("mapped data is not equal testData: %v, %v", sb.String(), expectedData) + } + + // Read slice bigger than mapped region after offset + sb.Reset() + sb.Grow(len(testData) + 10) + if n := m.ReadStringAt(sb, 0); n != len(testData) { + t.Fatalf("expected to read string of length %v, read of length %v", len(testData), n) + } + if string(testData) != sb.String() { + t.Fatalf("mapped data is not equal testData: %v, %v", sb.String(), string(testData)) + } + + // Read offset larger than size of mapped region + func() { + defer func() { + if err := recover(); err != ErrIndexOutOfBound { + t.Fatalf("unexpected error in reading from mmaped region :: %v", err) + } + }() + + sb.Reset() + sb.Grow(10) + _ = m.ReadStringAt(sb, 100) + }() +} + +func TestWriteStringAt(t *testing.T) { + setup(t) + defer tearDown(t) + + f, errFile := os.OpenFile(testPath, os.O_RDWR, 0644) + if errFile != nil { + t.Fatalf("error in opening file :: %v", errFile) + } + defer func() { + if err := f.Close(); err != nil { + t.Fatalf("error in closing file :: %v", err) + } + }() + + m, errMmap := NewSharedFileMmap(f, 0, len(testData), protPage) + if errMmap != nil { + t.Fatalf("error in mapping :: %v", errMmap) + } + defer func() { + if err := m.Unmap(); err != nil { + t.Fatalf("error in calling unmap :: %v", err) + } + }() + + // Write + _ = m.WriteStringAt("a", 9) + if err := m.Flush(syscall.MS_SYNC); err != nil { + t.Fatalf("error in calling flush :: %v", err) + } + f1, errFile := os.OpenFile(testPath, os.O_RDWR, 0644) + if errFile != nil { + t.Fatalf("error in opening file :: %v", errFile) + } + fileData, errFile := ioutil.ReadAll(f1) + if errFile != nil { + t.Fatalf("error in reading file :: %s", errFile) + } + if err := f1.Close(); err != nil { + t.Fatalf("error in closing file :: %v", err) + } + if !bytes.Equal(fileData, []byte("012345678aABCDEFGHIJKLMNOPQRSTUVWXYZ")) { + t.Fatalf("no modification in file :: %v", string(fileData)) + } + + // Write slice bigger than mapped region after offset + _ = m.WriteStringAt("abc", 34) + if err := m.Flush(syscall.MS_SYNC); err != nil { + t.Fatalf("error in flushing mapped region :: %v", err) + } + f2, err := os.OpenFile(testPath, os.O_RDWR, 0644) + if err != nil { + t.Fatalf("error in opening file :: %v", err) + } + fileData, err = ioutil.ReadAll(f2) + if err != nil { + t.Fatalf("error in reading file :: %s", err) + } + if err := f2.Close(); err != nil { + t.Fatalf("error in closing file :: %s", err) + } + if !bytes.Equal(fileData, []byte("012345678aABCDEFGHIJKLMNOPQRSTUVWXab")) { + t.Fatalf("no modification in file :: %v", string(fileData)) + } + + // Write offset larger than size of mapped region + func() { + defer func() { + if err := recover(); err != ErrIndexOutOfBound { + t.Fatalf("unexpected error in writing to mmaped region :: %v", err) + } + }() + + _ = m.WriteStringAt("a", 100) + }() +} + func TestAdvise(t *testing.T) { + setup(t) + defer tearDown(t) + f, err := os.OpenFile(testPath, os.O_RDWR, 0644) if err != nil { t.Fatalf("error in opening file :: %v", err) @@ -186,6 +349,9 @@ func TestAdvise(t *testing.T) { } func TestLockUnlock(t *testing.T) { + setup(t) + defer tearDown(t) + f, err := os.OpenFile(testPath, os.O_RDWR, 0644) if err != nil { t.Fatalf("error in opening file :: %v", err) @@ -215,6 +381,9 @@ func TestLockUnlock(t *testing.T) { } func TestReadUint64At(t *testing.T) { + setup(t) + defer tearDown(t) + f, err := os.OpenFile(testPath, os.O_RDWR, 0644) if err != nil { t.Fatalf("error in opening file :: %v", err) @@ -251,6 +420,9 @@ func TestReadUint64At(t *testing.T) { } func TestWriteUint64At(t *testing.T) { + setup(t) + defer tearDown(t) + f, err := os.OpenFile(testPath, os.O_RDWR, 0644) if err != nil { t.Fatalf("error in opening file :: %v", err) @@ -289,6 +461,9 @@ func TestWriteUint64At(t *testing.T) { } func TestFailScenarios(t *testing.T) { + setup(t) + defer tearDown(t) + f, err := os.OpenFile(testPath, os.O_RDWR, 0644) if err != nil { t.Fatalf("error in opening file :: %v", err)