Skip to content
Permalink
Browse files

fix needle Append return offset to avoid uint32 overflow

  • Loading branch information...
bingoohuang committed Jan 16, 2019
1 parent 7081261 commit 27093bc2e5cf3d61149e5eda14b98b1e527e5a28
Showing with 71 additions and 8 deletions.
  1. +5 −4 weed/storage/needle_read_write.go
  2. +62 −0 weed/storage/needle_read_write_test.go
  3. +4 −4 weed/storage/volume_read_write.go
@@ -3,6 +3,7 @@ package storage
import (
"errors"
"fmt"
"io"
"os"

"github.com/chrislusf/seaweedfs/weed/glog"
@@ -27,16 +28,16 @@ func (n *Needle) DiskSize(version Version) int64 {
return getActualSize(n.Size, version)
}

func (n *Needle) Append(w *os.File, version Version) (offset Offset, size uint32, actualSize int64, err error) {
if end, e := w.Seek(0, 2); e == nil {
func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32, actualSize int64, err error) {
if end, e := w.Seek(0, io.SeekEnd); e == nil {
defer func(w *os.File, off int64) {
if err != nil {
if te := w.Truncate(end); te != nil {
glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te)
}
}
}(w, end)
offset = Offset(end)
offset = uint64(end)
} else {
err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
return
@@ -175,7 +176,7 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
}
n.ParseNeedleHeader(bytes)
if n.Size != size {
return fmt.Errorf("File Entry Not Found. Needle id %d expected size %d Memory %d", n.Id, n.Size, size)
return fmt.Errorf("File Entry Not Found. offset %d, Needle id %d expected size %d Memory %d", offset, n.Id, n.Size, size)
}
switch version {
case Version1:
@@ -0,0 +1,62 @@
package storage

import (
"crypto/rand"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"io"
"io/ioutil"
"os"
"testing"
)

func TestAppend(t *testing.T) {
n := &Needle{

Cookie: types.Cookie(123), // Cookie Cookie `comment:"random number to mitigate brute force lookups"`
Id: types.NeedleId(123), // Id NeedleId `comment:"needle id"`
Size: 8, // Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
DataSize: 4, // DataSize uint32 `comment:"Data size"` //version2
Data: []byte("abcd"), // Data []byte `comment:"The actual file data"`
Flags: 0, // Flags byte `comment:"boolean flags"` //version2
NameSize: 0, // NameSize uint8 //version2
Name: nil, // Name []byte `comment:"maximum 256 characters"` //version2
MimeSize: 0, // MimeSize uint8 //version2
Mime: nil, // Mime []byte `comment:"maximum 256 characters"` //version2
PairsSize: 0, // PairsSize uint16 //version2
Pairs: nil, // Pairs []byte `comment:"additional name value pairs, json format, maximum 6
LastModified: 123, // LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes
Ttl: nil, // Ttl *TTL
Checksum: 123, // Checksum CRC `comment:"CRC32 to check integrity"`
AppendAtNs: 123, // AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
Padding: nil, // Padding []byte `comment:"Aligned to 8 bytes"`
}

tempFile, err := ioutil.TempFile("", ".dat")
if err != nil {
t.Errorf("Fail TempFile. %v", err)
return
}

/*
uint8 : 0 to 255
uint16 : 0 to 65535
uint32 : 0 to 4294967295
uint64 : 0 to 18446744073709551615
int8 : -128 to 127
int16 : -32768 to 32767
int32 : -2147483648 to 2147483647
int64 : -9223372036854775808 to 9223372036854775807
*/

fileSize := int64(4294967295) + 10000
io.CopyN(tempFile, rand.Reader, fileSize)
defer func() {
tempFile.Close()
os.Remove(tempFile.Name())
}()

offset, _, _, _ := n.Append(tempFile, CurrentVersion)
if offset != uint64(fileSize) {
t.Errorf("Fail to Append Needle.")
}
}
@@ -76,7 +76,7 @@ func (v *Volume) AppendBlob(b []byte) (offset int64, err error) {
return
}

func (v *Volume) writeNeedle(n *Needle) (offset Offset, size uint32, err error) {
func (v *Volume) writeNeedle(n *Needle) (offset uint64, size uint32, err error) {
glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String())
if v.readOnly {
err = fmt.Errorf("%s is read-only", v.dataFile.Name())
@@ -96,8 +96,8 @@ func (v *Volume) writeNeedle(n *Needle) (offset Offset, size uint32, err error)
}

nv, ok := v.nm.Get(n.Id)
if !ok || Offset(nv.Offset)*NeedlePaddingSize < offset {
if err = v.nm.Put(n.Id, offset/NeedlePaddingSize, n.Size); err != nil {
if !ok || uint64(nv.Offset)*NeedlePaddingSize < offset {
if err = v.nm.Put(n.Id, Offset(offset/NeedlePaddingSize), n.Size); err != nil {
glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err)
}
}
@@ -124,7 +124,7 @@ func (v *Volume) deleteNeedle(n *Needle) (uint32, error) {
if err != nil {
return size, err
}
if err = v.nm.Delete(n.Id, offset/NeedlePaddingSize); err != nil {
if err = v.nm.Delete(n.Id, Offset(offset/NeedlePaddingSize)); err != nil {
return size, err
}
return size, err

0 comments on commit 27093bc

Please sign in to comment.
You can’t perform that action at this time.