diff --git a/internal/allocator/allocator.go b/internal/allocator/allocator.go index 5956f48e..5a8de519 100644 --- a/internal/allocator/allocator.go +++ b/internal/allocator/allocator.go @@ -20,6 +20,7 @@ type Allocator struct { type File struct { Storage storage.File Name string + Padding bool } // Progress about the allocation. @@ -64,16 +65,20 @@ func (a *Allocator) Run(info *metainfo.Info, sto storage.Storage, progressC chan for i, f := range info.Files { var sf storage.File var exists bool - sf, exists, a.Error = sto.Open(f.Path, f.Length) - if a.Error != nil { - return - } - a.Files[i] = File{Storage: sf, Name: f.Path} - if exists { - a.HasExisting = true + if f.Padding { + sf = storage.NewPaddingFile(f.Length) } else { - a.HasMissing = true + sf, exists, a.Error = sto.Open(f.Path, f.Length) + if a.Error != nil { + return + } + if exists { + a.HasExisting = true + } else { + a.HasMissing = true + } } + a.Files[i] = File{Storage: sf, Name: f.Path, Padding: f.Padding} allocatedSize += f.Length a.sendProgress(progressC, allocatedSize) } diff --git a/internal/bufferpool/bufferpool.go b/internal/bufferpool/bufferpool.go index 44a01673..3346a55a 100644 --- a/internal/bufferpool/bufferpool.go +++ b/internal/bufferpool/bufferpool.go @@ -35,11 +35,16 @@ type Buffer struct { } func newBuffer(buf *[]byte, length int, pool *Pool) Buffer { - return Buffer{ + b := Buffer{ Data: (*buf)[:length], buf: buf, pool: pool, } + // Clear contents before returning to the caller. + for i := range b.Data { + b.Data[i] = 0 + } + return b } // Release the Buffer and return it to the Pool. diff --git a/internal/filesection/section.go b/internal/filesection/section.go index 6bca0645..eb9f582a 100644 --- a/internal/filesection/section.go +++ b/internal/filesection/section.go @@ -8,6 +8,9 @@ type FileSection struct { Offset int64 Length int64 Name string + // Padding indicates that the file is used as padding so it should not be requested from peers. + // The contents of padding files are always zero. + Padding bool } // ReadWriterAt combines the io.ReaderAt and io.WriterAt interfaces. diff --git a/internal/filesection/section_test.go b/internal/filesection/section_test.go index d18d34d7..f90dba72 100644 --- a/internal/filesection/section_test.go +++ b/internal/filesection/section_test.go @@ -32,10 +32,10 @@ func TestFiles(t *testing.T) { } } files := []FileSection{ - {osFiles[0], 2, 2, ""}, - {osFiles[1], 0, 1, ""}, - {osFiles[2], 0, 0, ""}, - {osFiles[3], 0, 2, ""}, + {osFiles[0], 2, 2, "", false}, + {osFiles[1], 0, 1, "", false}, + {osFiles[2], 0, 0, "", false}, + {osFiles[3], 0, 2, "", false}, } pf := Piece(files) diff --git a/internal/metainfo/info.go b/internal/metainfo/info.go index b6faa9d9..039cddb2 100644 --- a/internal/metainfo/info.go +++ b/internal/metainfo/info.go @@ -40,25 +40,53 @@ type Info struct { type File struct { Length int64 Path string + // https://www.bittorrent.org/beps/bep_0047.html + Padding bool } type file struct { Length int64 `bencode:"length"` Path []string `bencode:"path"` PathUTF8 []string `bencode:"path.utf-8,omitempty"` + Attr string `bencode:"attr"` } -// NewInfo returns info from bencoded bytes in b. -func NewInfo(b []byte, utf8 bool) (*Info, error) { - var ib struct { - PieceLength uint32 `bencode:"piece length"` - Pieces []byte `bencode:"pieces"` - Name string `bencode:"name"` - NameUTF8 string `bencode:"name.utf-8,omitempty"` - Private bencode.RawMessage `bencode:"private"` - Length int64 `bencode:"length"` // Single File Mode - Files []file `bencode:"files"` // Multiple File mode +func (f *file) isPadding() bool { + // BEP 0047 + if strings.ContainsRune(f.Attr, 'p') { + return true + } + // BitComet convention that do not conform BEP 0047 + if len(f.Path) > 0 && strings.HasPrefix(f.Path[len(f.Path)-1], "_____padding_file") { + return true + } + return false +} + +type infoType struct { + PieceLength uint32 `bencode:"piece length"` + Pieces []byte `bencode:"pieces"` + Name string `bencode:"name"` + NameUTF8 string `bencode:"name.utf-8,omitempty"` + Private bencode.RawMessage `bencode:"private"` + Length int64 `bencode:"length"` // Single File Mode + Files []file `bencode:"files"` // Multiple File mode +} + +func (ib *infoType) overrideUTF8Keys() { + if len(ib.NameUTF8) > 0 { + ib.Name = ib.NameUTF8 } + for i := range ib.Files { + if len(ib.Files[i].PathUTF8) > 0 { + ib.Files[i].Path = ib.Files[i].PathUTF8 + } + } +} + +// NewInfo returns info from bencoded bytes in b. +func NewInfo(b []byte, utf8 bool, pad bool) (*Info, error) { + var ib infoType if err := bencode.DecodeBytes(b, &ib); err != nil { return nil, err } @@ -72,15 +100,8 @@ func NewInfo(b []byte, utf8 bool) (*Info, error) { if numPieces == 0 { return nil, errZeroPieces } - if utf8 { // override name and path from utf8 keys - if len(ib.NameUTF8) > 0 { - ib.Name = ib.NameUTF8 - } - for i := range ib.Files { - if len(ib.Files[i].PathUTF8) > 0 { - ib.Files[i].Path = ib.Files[i].PathUTF8 - } - } + if utf8 { + ib.overrideUTF8Keys() } // ".." is not allowed in file names for _, file := range ib.Files { @@ -137,6 +158,9 @@ func NewInfo(b []byte, utf8 bool) (*Info, error) { Path: filepath.Join(parts...), Length: f.Length, } + if pad { + i.Files[j].Padding = f.isPadding() + } } } else { i.Files = []File{{Path: cleanName(i.Name), Length: i.Length}} diff --git a/internal/metainfo/metainfo.go b/internal/metainfo/metainfo.go index fbc87472..a1a04ed6 100644 --- a/internal/metainfo/metainfo.go +++ b/internal/metainfo/metainfo.go @@ -36,7 +36,7 @@ func New(r io.Reader) (*MetaInfo, error) { if len(t.Info) == 0 { return nil, errors.New("no info dict in torrent file") } - info, err := NewInfo(t.Info, true) + info, err := NewInfo(t.Info, true, true) if err != nil { return nil, err } diff --git a/internal/piece/piece.go b/internal/piece/piece.go index 52bdf78c..1d0ed725 100644 --- a/internal/piece/piece.go +++ b/internal/piece/piece.go @@ -25,9 +25,8 @@ type Piece struct { // Block is part of a Piece that is specified in peerprotocol.Request messages. type Block struct { - Index int // index in piece - Begin uint32 // offset in piece - Length uint32 // always equal to BlockSize except the last block of a piece. + Begin uint32 // Offset in piece + Length uint32 // Cannot exceed BlockSize. It's shorter for last block or if the next file is a padding file. } // NewPieces returns a slice of Pieces by mapping files to the pieces. @@ -70,10 +69,11 @@ func NewPieces(info *metainfo.Info, files []allocator.File) []Piece { n := uint32(min(int64(left), fileLeft())) // number of bytes to write file := filesection.FileSection{ - File: files[fileIndex].Storage, - Offset: fileOffset, - Length: int64(n), - Name: files[fileIndex].Name, + File: files[fileIndex].Storage, + Offset: fileOffset, + Length: int64(n), + Name: files[fileIndex].Name, + Padding: files[fileIndex].Padding, } sections = append(sections, file) @@ -97,8 +97,10 @@ func NewPieces(info *metainfo.Info, files []allocator.File) []Piece { return pieces } -// NumBlocks returns the number of blocks in the piece. -func (p *Piece) NumBlocks() int { +// numBlocks returns the number of blocks in the piece. +// The calculation is only correct when there is no padding in piece. +// It is only used in per-allocation of blocks slice in CalculateBlocks(). +func (p *Piece) numBlocks() int { div, mod := divmod(p.Length, BlockSize) numBlocks := div if mod != 0 { @@ -107,44 +109,61 @@ func (p *Piece) NumBlocks() int { return int(numBlocks) } -// GetBlock returns the Block at index i. -func (p *Piece) GetBlock(i int) (b Block, ok bool) { - div, mod := divmod(p.Length, BlockSize) - numBlocks := int(div) - if mod != 0 { - numBlocks++ - } - if i >= numBlocks { - return - } - var blen uint32 - if mod != 0 && i == numBlocks-1 { - blen = mod - } else { - blen = BlockSize - } - return Block{ - Index: i, - Begin: uint32(i) * BlockSize, - Length: blen, - }, true +func (p *Piece) CalculateBlocks() []Block { + return p.calculateBlocks(BlockSize) } -// FindBlock returns the block at offset `begin` and length `length`. -func (p *Piece) FindBlock(begin, length uint32) (b Block, ok bool) { - idx, mod := divmod(begin, BlockSize) - if mod != 0 { - return +func (p *Piece) calculateBlocks(blockSize uint32) []Block { + blocks := make([]Block, 0, p.numBlocks()) + + secIndex := 0 + sec := p.Data[secIndex] + blk := Block{ + Begin: 0, + Length: 0, + } + pieceOffset := uint32(0) + secOffset := uint32(0) + blkLeft := func() uint32 { return blockSize - blk.Length } + secLeft := func() uint32 { return uint32(sec.Length) - secOffset } + nextBlock := func() { + if blk.Length == 0 { + return + } + blocks = append(blocks, blk) + blk.Begin = pieceOffset + blk.Length = 0 } - b, ok = p.GetBlock(int(idx)) - if !ok { - return + hasNextSection := true + nextSection := func() { + secIndex++ + if secIndex == len(p.Data) { + hasNextSection = false + return + } + secOffset = 0 + sec = p.Data[secIndex] } - if b.Length != length { - ok = false - return + for hasNextSection { + if sec.Padding { + pieceOffset += uint32(sec.Length) + nextBlock() + nextSection() + continue + } + n := min(secLeft(), blkLeft()) + blk.Length += n + pieceOffset += n + secOffset += n + if blkLeft() == 0 { + nextBlock() + } + if secLeft() == 0 { + nextSection() + } } - return b, true + nextBlock() + return blocks } // VerifyHash returns true if hash of piece data in buffer `buf` matches the hash of Piece. diff --git a/internal/piece/piece_test.go b/internal/piece/piece_test.go index 5cb8f875..f7994f3f 100644 --- a/internal/piece/piece_test.go +++ b/internal/piece/piece_test.go @@ -3,79 +3,172 @@ package piece import ( "testing" + "github.com/cenkalti/rain/internal/filesection" "github.com/stretchr/testify/assert" ) func TestNumBlocks(t *testing.T) { p := Piece{Length: 2 * 16 * 1024} - assert.Equal(t, 2, p.NumBlocks()) + assert.Equal(t, 2, p.numBlocks()) p = Piece{Length: 2*16*1024 + 42} - assert.Equal(t, 3, p.NumBlocks()) + assert.Equal(t, 3, p.numBlocks()) } -func TestGetBlock(t *testing.T) { +func TestFindBlock(t *testing.T) { p := Piece{ - Index: 1, - Length: 2 * 16 * 1024, + Length: 2*BlockSize + 42, + Data: []filesection.FileSection{ + { + Length: 2*BlockSize + 42, + }, + }, } - - _, ok := p.GetBlock(2) - assert.False(t, ok) - - b, ok := p.GetBlock(0) - assert.True(t, ok) - assert.Equal(t, Block{Index: 0, Begin: 0, Length: 16 * 1024}, b) - - b, ok = p.GetBlock(1) - assert.True(t, ok) - assert.Equal(t, Block{Index: 1, Begin: 16 * 1024, Length: 16 * 1024}, b) - - p = Piece{ - Index: 1, - Length: 2*16*1024 + 42, + blocks := p.CalculateBlocks() + findBlock := func(begin, length uint32) bool { + for _, blk := range blocks { + if blk.Begin == begin && blk.Length == length { + return true + } + } + return false } - - _, ok = p.GetBlock(3) - assert.False(t, ok) - - b, ok = p.GetBlock(0) - assert.True(t, ok) - assert.Equal(t, Block{Index: 0, Begin: 0, Length: 16 * 1024}, b) - - b, ok = p.GetBlock(1) - assert.True(t, ok) - assert.Equal(t, Block{Index: 1, Begin: 16 * 1024, Length: 16 * 1024}, b) - - b, ok = p.GetBlock(2) - assert.True(t, ok) - assert.Equal(t, Block{Index: 2, Begin: 2 * 16 * 1024, Length: 42}, b) + assert.False(t, findBlock(55, BlockSize)) + assert.False(t, findBlock(3*BlockSize, BlockSize)) + assert.False(t, findBlock(0, 1234)) + assert.True(t, findBlock(0, BlockSize)) + assert.True(t, findBlock(BlockSize, BlockSize)) + assert.True(t, findBlock(2*BlockSize, 42)) } -func TestFindBlock(t *testing.T) { - p := Piece{ - Index: 1, - Length: 2*BlockSize + 42, +func TestCalculateBlocks(t *testing.T) { + const blockSize = 40 + testCases := []struct { + piece Piece + expected []Block + }{ + { + piece: Piece{ + Length: 100, + Data: []filesection.FileSection{ + {Length: 80, Padding: false}, + {Length: 20, Padding: true}, + }, + }, + expected: []Block{ + { + Begin: 0, + Length: 40, + }, + { + Begin: 40, + Length: 40, + }, + }, + }, + { + piece: Piece{ + Length: 100, + Data: []filesection.FileSection{ + {Length: 90, Padding: false}, + {Length: 10, Padding: true}, + }, + }, + expected: []Block{ + { + Begin: 0, + Length: 40, + }, + { + Begin: 40, + Length: 40, + }, + { + Begin: 80, + Length: 10, + }, + }, + }, + { + piece: Piece{ + Length: 100, + Data: []filesection.FileSection{ + {Length: 30, Padding: false}, + {Length: 60, Padding: false}, + {Length: 10, Padding: true}, + }, + }, + expected: []Block{ + { + Begin: 0, + Length: 40, + }, + { + Begin: 40, + Length: 40, + }, + { + Begin: 80, + Length: 10, + }, + }, + }, + { + piece: Piece{ + Length: 100, + Data: []filesection.FileSection{ + {Length: 30, Padding: false}, + {Length: 30, Padding: false}, + {Length: 40, Padding: true}, + }, + }, + expected: []Block{ + { + Begin: 0, + Length: 40, + }, + { + Begin: 40, + Length: 20, + }, + }, + }, + // This case is not very realistic because it contains data after padding in the same piece. + { + piece: Piece{ + Length: 100, + Data: []filesection.FileSection{ + {Length: 30, Padding: false}, + {Length: 20, Padding: true}, + {Length: 50, Padding: false}, + }, + }, + expected: []Block{ + { + Begin: 0, + Length: 30, + }, + { + Begin: 50, + Length: 40, + }, + { + Begin: 90, + Length: 10, + }, + }, + }, + } + for i, tc := range testCases { + dataLen := int64(0) + for _, sec := range tc.piece.Data { + dataLen += sec.Length + } + if dataLen != int64(tc.piece.Length) { + t.Errorf("error in test case #%d: piece and data length do not match", i) + continue + } + blocks := tc.piece.calculateBlocks(blockSize) + assert.Equal(t, tc.expected, blocks, "test case #%d", i) } - - _, ok := p.FindBlock(55, BlockSize) - assert.False(t, ok) - - _, ok = p.FindBlock(3*BlockSize, BlockSize) - assert.False(t, ok) - - _, ok = p.FindBlock(0, 1234) - assert.False(t, ok) - - b, ok := p.FindBlock(0, BlockSize) - assert.True(t, ok) - assert.Equal(t, Block{Index: 0, Begin: 0, Length: BlockSize}, b) - - b, ok = p.FindBlock(BlockSize, BlockSize) - assert.True(t, ok) - assert.Equal(t, Block{Index: 1, Begin: BlockSize, Length: BlockSize}, b) - - b, ok = p.FindBlock(2*BlockSize, 42) - assert.True(t, ok) - assert.Equal(t, Block{Index: 2, Begin: 2 * BlockSize, Length: 42}, b) } diff --git a/internal/piecedownloader/piecedownloader.go b/internal/piecedownloader/piecedownloader.go index f295fd83..20125002 100644 --- a/internal/piecedownloader/piecedownloader.go +++ b/internal/piecedownloader/piecedownloader.go @@ -12,6 +12,8 @@ var ( ErrBlockDuplicate = errors.New("received duplicate block") // ErrBlockNotRequested is returned from PieceDownloader.GotBlock method when the received block is not requested yet. ErrBlockNotRequested = errors.New("received not requested block") + // ErrBlockInvalid is returned from PieceDownloader.GotBlock method when the received block is not in request list. + ErrBlockInvalid = errors.New("received block is invalid") ) // PieceDownloader downloads all blocks of a piece from a peer. @@ -21,9 +23,12 @@ type PieceDownloader struct { AllowedFast bool Buffer bufferpool.Buffer - remaining []int - pending map[int]struct{} // in-flight requests - done map[int]struct{} // downloaded requests + // blocks contains blocks that needs to be downloaded from peers. + // It does not contain the parts that belong to padding files. + blocks map[uint32]uint32 // begin -> length + remaining []uint32 // blocks to be downloaded from peers in consecutive order. + pending map[uint32]struct{} // in-flight requests + done map[uint32]struct{} // downloaded requests } // Peer of a Torrent. @@ -35,19 +40,33 @@ type Peer interface { // New returns a new PieceDownloader. func New(pi *piece.Piece, pe Peer, allowedFast bool, buf bufferpool.Buffer) *PieceDownloader { - remaining := make([]int, pi.NumBlocks()) - for i := range remaining { - remaining[i] = i - } + blocks := pi.CalculateBlocks() return &PieceDownloader{ Piece: pi, Peer: pe, AllowedFast: allowedFast, Buffer: buf, - remaining: remaining, - pending: make(map[int]struct{}), - done: make(map[int]struct{}), + blocks: makeBlocks(blocks), + remaining: makeRemaining(blocks), + pending: make(map[uint32]struct{}, len(blocks)), + done: make(map[uint32]struct{}, len(blocks)), + } +} + +func makeBlocks(blocks []piece.Block) map[uint32]uint32 { + ret := make(map[uint32]uint32, len(blocks)) + for _, blk := range blocks { + ret[blk.Begin] = blk.Length + } + return ret +} + +func makeRemaining(blocks []piece.Block) []uint32 { + ret := make([]uint32, len(blocks)) + for i, blk := range blocks { + ret[i] = blk.Begin } + return ret } // Choked must be called when the peer has choked us. This will cancel pending reuqests. @@ -65,58 +84,72 @@ func (d *PieceDownloader) Choked() { } } -// GotBlock must be called when a block is received from the piece. -func (d *PieceDownloader) GotBlock(block piece.Block, data []byte) error { - var err error - if _, ok := d.done[block.Index]; ok { +func (d *PieceDownloader) findBlock(begin, length uint32) bool { + blockLength, ok := d.blocks[begin] + return ok && blockLength == length +} + +// GotBlock must be called when a block is received from the peer. +func (d *PieceDownloader) GotBlock(begin uint32, data []byte) error { + if !d.findBlock(begin, uint32(len(data))) { + return ErrBlockInvalid + } + if _, ok := d.done[begin]; ok { return ErrBlockDuplicate - } else if _, ok := d.pending[block.Index]; !ok { - err = ErrBlockNotRequested } - copy(d.Buffer.Data[block.Begin:block.Begin+block.Length], data) - delete(d.pending, block.Index) - d.done[block.Index] = struct{}{} - return err + copy(d.Buffer.Data[begin:begin+uint32(len(data))], data) + d.done[begin] = struct{}{} + if _, ok := d.pending[begin]; !ok { + // We got the block data although we didn't request it. + // Data is still saved but error returned here to notify the caller about the issue. + return ErrBlockNotRequested + } + delete(d.pending, begin) + return nil } // Rejected must be called when the peer has rejected a piece request. -func (d *PieceDownloader) Rejected(block piece.Block) { - delete(d.pending, block.Index) - d.remaining = append(d.remaining, block.Index) +func (d *PieceDownloader) Rejected(begin, length uint32) bool { + if !d.findBlock(begin, length) { + return false + } + delete(d.pending, begin) + d.remaining = append(d.remaining, begin) + return true } // CancelPending is called to cancel pending requests to the peer. // Must be called when remaining blocks are downloaded from another peer. func (d *PieceDownloader) CancelPending() { - for i := range d.pending { - b, ok := d.Piece.GetBlock(i) + for begin := range d.pending { + length, ok := d.blocks[begin] if !ok { panic("cannot get block") } - d.Peer.CancelPiece(d.Piece.Index, b.Begin, b.Length) + d.Peer.CancelPiece(d.Piece.Index, begin, length) } } // RequestBlocks is called to request remaining blocks of the piece up to `queueLength`. func (d *PieceDownloader) RequestBlocks(queueLength int) { remaining := d.remaining - for _, i := range remaining { + for _, begin := range remaining { if len(d.pending) >= queueLength { break } - b, ok := d.Piece.GetBlock(i) + length, ok := d.blocks[begin] if !ok { panic("cannot get block") } - if _, ok := d.done[i]; !ok { - d.Peer.RequestPiece(d.Piece.Index, b.Begin, b.Length) + if _, ok := d.done[begin]; !ok { + d.Peer.RequestPiece(d.Piece.Index, begin, length) } d.remaining = d.remaining[1:] - d.pending[i] = struct{}{} + d.pending[begin] = struct{}{} } } // Done returns true if all blocks of the piece has been downloaded. func (d *PieceDownloader) Done() bool { - return len(d.done) == d.Piece.NumBlocks() + return len(d.done) == len(d.blocks) } diff --git a/internal/piecedownloader/piecedownloader_test.go b/internal/piecedownloader/piecedownloader_test.go index 436dd297..d67c9630 100644 --- a/internal/piecedownloader/piecedownloader_test.go +++ b/internal/piecedownloader/piecedownloader_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/cenkalti/rain/internal/bufferpool" + "github.com/cenkalti/rain/internal/filesection" "github.com/cenkalti/rain/internal/peerprotocol" "github.com/cenkalti/rain/internal/piece" "github.com/stretchr/testify/assert" @@ -44,16 +45,25 @@ func TestPieceDownloader(t *testing.T) { pi := &piece.Piece{ Index: 1, Length: 10*blockSize + 42, + Data: []filesection.FileSection{ + { + Length: 9*blockSize + 21, + }, + { + Length: blockSize + 21, + Padding: true, + }, + }, } pe := &TestPeer{} d := New(pi, pe, false, buf) - assert.Equal(t, 11, len(d.remaining)) + assert.Equal(t, 10, len(d.remaining)) assert.Equal(t, 0, len(d.pending)) assert.Equal(t, 0, len(d.done)) assert.False(t, d.Done()) d.RequestBlocks(4) - assert.Equal(t, 7, len(d.remaining)) + assert.Equal(t, 6, len(d.remaining)) assert.Equal(t, 4, len(d.pending)) assert.Equal(t, 0, len(d.done)) assert.False(t, d.Done()) @@ -65,7 +75,7 @@ func TestPieceDownloader(t *testing.T) { }, pe.requested) d.RequestBlocks(4) - assert.Equal(t, 7, len(d.remaining)) + assert.Equal(t, 6, len(d.remaining)) assert.Equal(t, 4, len(d.pending)) assert.Equal(t, 0, len(d.done)) assert.False(t, d.Done()) @@ -76,15 +86,15 @@ func TestPieceDownloader(t *testing.T) { {Index: 1, Begin: 3 * blockSize, Length: blockSize}, }, pe.requested) - assert.Nil(t, d.GotBlock(piece.Block{Index: 0, Begin: 0, Length: blockSize}, make([]byte, blockSize))) + assert.Nil(t, d.GotBlock(0, make([]byte, blockSize))) assert.Equal(t, 4, len(pe.requested)) - assert.Equal(t, 7, len(d.remaining)) + assert.Equal(t, 6, len(d.remaining)) assert.Equal(t, 3, len(d.pending)) assert.Equal(t, 1, len(d.done)) assert.False(t, d.Done()) d.RequestBlocks(4) - assert.Equal(t, 6, len(d.remaining)) + assert.Equal(t, 5, len(d.remaining)) assert.Equal(t, 4, len(d.pending)) assert.Equal(t, 1, len(d.done)) assert.False(t, d.Done()) @@ -96,17 +106,17 @@ func TestPieceDownloader(t *testing.T) { {Index: 1, Begin: 4 * blockSize, Length: blockSize}, }, pe.requested) - assert.Nil(t, d.GotBlock(piece.Block{Index: 1, Begin: 1 * blockSize, Length: blockSize}, make([]byte, blockSize))) - assert.Nil(t, d.GotBlock(piece.Block{Index: 2, Begin: 2 * blockSize, Length: blockSize}, make([]byte, blockSize))) - assert.Nil(t, d.GotBlock(piece.Block{Index: 3, Begin: 3 * blockSize, Length: blockSize}, make([]byte, blockSize))) - assert.Nil(t, d.GotBlock(piece.Block{Index: 4, Begin: 4 * blockSize, Length: blockSize}, make([]byte, blockSize))) - assert.Equal(t, 6, len(d.remaining)) + assert.Nil(t, d.GotBlock(1*blockSize, make([]byte, blockSize))) + assert.Nil(t, d.GotBlock(2*blockSize, make([]byte, blockSize))) + assert.Nil(t, d.GotBlock(3*blockSize, make([]byte, blockSize))) + assert.Nil(t, d.GotBlock(4*blockSize, make([]byte, blockSize))) + assert.Equal(t, 5, len(d.remaining)) assert.Equal(t, 0, len(d.pending)) assert.Equal(t, 5, len(d.done)) assert.False(t, d.Done()) d.RequestBlocks(4) - assert.Equal(t, 2, len(d.remaining)) + assert.Equal(t, 1, len(d.remaining)) assert.Equal(t, 4, len(d.pending)) assert.Equal(t, 5, len(d.done)) assert.False(t, d.Done()) @@ -122,26 +132,25 @@ func TestPieceDownloader(t *testing.T) { {Index: 1, Begin: 8 * blockSize, Length: blockSize}, }, pe.requested) - assert.Nil(t, d.GotBlock(piece.Block{Index: 5, Begin: 5 * blockSize, Length: blockSize}, make([]byte, blockSize))) - assert.Equal(t, 2, len(d.remaining)) + assert.Nil(t, d.GotBlock(5*blockSize, make([]byte, blockSize))) + assert.Equal(t, 1, len(d.remaining)) assert.Equal(t, 3, len(d.pending)) assert.Equal(t, 6, len(d.done)) assert.False(t, d.Done()) d.Choked() - assert.Equal(t, 5, len(d.remaining)) + assert.Equal(t, 4, len(d.remaining)) assert.Equal(t, 0, len(d.pending)) assert.Equal(t, 6, len(d.done)) assert.False(t, d.Done()) d.RequestBlocks(99) - assert.Nil(t, d.GotBlock(piece.Block{Index: 6, Begin: 6 * blockSize, Length: blockSize}, make([]byte, blockSize))) - assert.Nil(t, d.GotBlock(piece.Block{Index: 7, Begin: 7 * blockSize, Length: blockSize}, make([]byte, blockSize))) - assert.Nil(t, d.GotBlock(piece.Block{Index: 8, Begin: 8 * blockSize, Length: blockSize}, make([]byte, blockSize))) - assert.Nil(t, d.GotBlock(piece.Block{Index: 9, Begin: 9 * blockSize, Length: blockSize}, make([]byte, blockSize))) - assert.Nil(t, d.GotBlock(piece.Block{Index: 10, Begin: 10 * blockSize, Length: 42}, make([]byte, 42))) + assert.Nil(t, d.GotBlock(6*blockSize, make([]byte, blockSize))) + assert.Nil(t, d.GotBlock(7*blockSize, make([]byte, blockSize))) + assert.Nil(t, d.GotBlock(8*blockSize, make([]byte, blockSize))) + assert.Nil(t, d.GotBlock(9*blockSize, make([]byte, 21))) assert.Equal(t, 0, len(d.remaining)) assert.Equal(t, 0, len(d.pending)) - assert.Equal(t, 11, len(d.done)) + assert.Equal(t, 10, len(d.done)) assert.True(t, d.Done()) } diff --git a/internal/resumer/boltdbresumer/boltdbresumer.go b/internal/resumer/boltdbresumer/boltdbresumer.go index 3fe2b213..a5d7ea54 100644 --- a/internal/resumer/boltdbresumer/boltdbresumer.go +++ b/internal/resumer/boltdbresumer/boltdbresumer.go @@ -12,7 +12,7 @@ import ( ) // LatestVersion is incremented every time there is a backwards-incompatible change in the interpretation of existing resumer data. -const LatestVersion = 2 +const LatestVersion = 3 // Keys for the persisten storage. var Keys = struct { diff --git a/internal/storage/padding.go b/internal/storage/padding.go new file mode 100644 index 00000000..66d28890 --- /dev/null +++ b/internal/storage/padding.go @@ -0,0 +1,25 @@ +package storage + +type PaddingFile struct{} + +func NewPaddingFile(length int64) File { + return PaddingFile{} +} + +var _ File = PaddingFile{} + +func (f PaddingFile) ReadAt(p []byte, off int64) (n int, err error) { + // Need to serve zeroes to be backwards compatibile with clients that do not support padding files. + for i := range p { + p[i] = 0 + } + return len(p), nil +} + +func (f PaddingFile) WriteAt(p []byte, off int64) (n int, err error) { + panic("attempt to write padding file") +} + +func (f PaddingFile) Close() error { + return nil +} diff --git a/torrent/session_load.go b/torrent/session_load.go index 52398437..373e3c67 100644 --- a/torrent/session_load.go +++ b/torrent/session_load.go @@ -41,14 +41,18 @@ func (s *Session) loadExistingTorrents(ids []string) { func (s *Session) parseInfo(b []byte, version int) (*metainfo.Info, error) { var useUTF8Keys bool + var hidePaddings bool switch version { case 1: case 2: useUTF8Keys = true + case 3: + useUTF8Keys = true + hidePaddings = true default: return nil, fmt.Errorf("unknown resume data version: %d", version) } - i, err := metainfo.NewInfo(b, useUTF8Keys) + i, err := metainfo.NewInfo(b, useUTF8Keys, hidePaddings) if err != nil { return nil, err } diff --git a/torrent/torrent_messagehandler.go b/torrent/torrent_messagehandler.go index 696bf813..0ce4ab22 100644 --- a/torrent/torrent_messagehandler.go +++ b/torrent/torrent_messagehandler.go @@ -48,35 +48,33 @@ func (t *torrent) handlePieceMessage(pm peer.PieceMessage) { return } piece := pd.Piece - block, ok := piece.FindBlock(msg.Begin, uint32(len(msg.Buffer.Data))) - if !ok { - pe.Logger().Errorln("invalid piece index:", msg.Index, "begin:", msg.Begin, "length:", len(msg.Buffer.Data)) + err := pd.GotBlock(msg.Begin, msg.Buffer.Data) + switch err { + case piecedownloader.ErrBlockInvalid: + pe.Logger().Errorln("received invalid piece index:", msg.Index, "begin:", msg.Begin, "length:", len(msg.Buffer.Data)) t.bytesWasted.Inc(l) t.closePeer(pe) msg.Buffer.Release() return - } - err := pd.GotBlock(block, msg.Buffer.Data) - switch err { case piecedownloader.ErrBlockDuplicate: if pe.FastEnabled { - pe.Logger().Warningln("received duplicate block:", block.Index) + pe.Logger().Warningf("received duplicate block index:", msg.Index, "begin:", msg.Begin, "length:", len(msg.Buffer.Data)) } else { // If peer does not support fast extension, we cancel all pending requests on choke message. // After an unchoke we request them again. Some clients appears to be sending the same block // if we request it twice. - pe.Logger().Debugln("received duplicate block:", block.Index) + pe.Logger().Debugf("received duplicate block index:", msg.Index, "begin:", msg.Begin, "length:", len(msg.Buffer.Data)) } t.bytesWasted.Inc(l) msg.Buffer.Release() return case piecedownloader.ErrBlockNotRequested: if pe.FastEnabled { - pe.Logger().Warningln("received not requested block:", block.Index) + pe.Logger().Warningf("received not requested block index:", msg.Index, "begin:", msg.Begin, "length:", len(msg.Buffer.Data)) } else { // If peer does not support fast extension, we cancel all pending requests on choke message. // That's why we think that we have received an unrequested block. - pe.Logger().Debugln("received not requested block:", block.Index) + pe.Logger().Debugf("received not requested block index:", msg.Index, "begin:", msg.Begin, "length:", len(msg.Buffer.Data)) } case nil: default: @@ -278,13 +276,12 @@ func (t *torrent) handlePeerMessage(pm peer.Message) { if pd.Piece.Index != msg.Index { break } - block, ok := pd.Piece.FindBlock(msg.Begin, msg.Length) + ok = pd.Rejected(msg.Begin, msg.Length) if !ok { pe.Logger().Errorln("invalid reject index:", msg.Index, "begin:", msg.Begin, "length:", msg.Length) t.closePeer(pe) break } - pd.Rejected(block) case peerprotocol.CancelMessage: if t.pieces == nil || t.bitfield == nil { pe.Logger().Error("cancel received but we don't have info")