From 3d82c01b25f6098bc1f98d26342fe1f9ec5fbae3 Mon Sep 17 00:00:00 2001 From: Niels Zeilemaker Date: Mon, 10 Jun 2013 14:22:51 +0200 Subject: [PATCH] Will now verify all pieces, except first and last pieces. Removed filepiecereanges dependency, now uses libtorrent info. --- .../Core/Libtorrent/LibtorrentDownloadImpl.py | 100 +++++++++++++----- Tribler/Video/VideoServer.py | 2 +- 2 files changed, 77 insertions(+), 25 deletions(-) diff --git a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py index bc0d800c4ae..525f6849b66 100644 --- a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py +++ b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py @@ -36,15 +36,16 @@ class VODFile(object): - def __init__(self, f, d, filepiecerange): + def __init__(self, f, d): self._file = f self._download = d pieces = self._download.tdef.get_pieces() self.pieces = [pieces[x:x + 20]for x in xrange(0, len(pieces), 20)] self.piecesize = self._download.tdef.get_piece_length() - self.startpiece = filepiecerange[0] - self.startoffset = filepiecerange[2] + + self.startpiece = self._download.handle.get_torrent_info().map_file(self._download.get_vod_fileindex(), 0, 0) + self.endpiece = self._download.handle.get_torrent_info().map_file(self._download.get_vod_fileindex(), self._download.get_vod_filesize(), 0) def read(self, *args): oldpos = self._file.tell() @@ -54,7 +55,8 @@ def read(self, *args): time.sleep(1) result = self._file.read(*args) print >> sys.stderr, 'VODFile, got bytes', oldpos, '-', self._file.tell() - self.verify_pieces(result, oldpos, self._file.tell()) + assert self.verify_pieces(result, oldpos, self._file.tell()) + return result def seek(self, *args): @@ -68,28 +70,76 @@ def seek(self, *args): print >> sys.stderr, self._download.handle.piece_priorities() - def verify_pieces(self, data, frompos, topos): - # let's see if we read a complete piece - frompos += self.startoffset - topos += self.startoffset + def verify_pieces(self, original_data, frompos, topos): + allpiecesok = True + _frompos = frompos + _topos = topos - startpos = int(ceil(frompos / float(self.piecesize)) * self.piecesize) - endpos = int(floor(topos / float(self.piecesize)) * self.piecesize) + frompiece = self._download.handle.get_torrent_info().map_file(self._download.get_vod_fileindex(), frompos, 0) + topiece = self._download.handle.get_torrent_info().map_file(self._download.get_vod_fileindex(), topos, 0) + print >> sys.stderr, "LibTorrent says we read pieces", frompiece.piece, topiece.piece - while startpos != endpos: - startindex = startpos - frompos - endindex = startindex + self.piecesize - pieceindex = (startpos / self.piecesize) + self.startpiece + if frompiece.start: + if frompos - frompiece.start < 0: + print >> sys.stderr, "Cannot verify ", frompos, "-", frompos + self.piecesize - frompiece.start - piecehash = sha(data[startindex:endindex]).digest() - if piecehash != self.pieces[pieceindex]: - print >> sys.stderr, "Incorrect piece read", pieceindex, piecehash, self.pieces[pieceindex] + # cannot read partial piece, skipping first X bytes + frompos += self.piecesize - frompiece.start + frompiece = frompiece.piece + 1 else: - print >> sys.stderr, "Correct piece read", pieceindex + # need to read more than this partial piece, extending with X bytes + frompos -= frompiece.start + frompiece = frompiece.piece + + if topiece.piece == self.endpiece.piece: + print >> sys.stderr, "Cannot verify ", topos - topiece.start, "-", topos + + # cannot read partial piece, truncating last X bytes + topos -= topiece.start + topiece = topiece.piece - 1 + + else: + if topiece.start: + topos += self.piecesize - topiece.start + topiece = topiece.piece + + if topiece >= frompiece: + oldpos = self._file.tell() + self._file.seek(frompos) + read_data = self._file.read(topos - frompos) + self._file.seek(oldpos) + + assert len(read_data) == topos - frompos - startpos += self.piecesize + # align two arrays + data_offsets = [0, len(read_data)] + original_data_offsets = [0, (len(original_data))] - print >> sys.stderr, "Cannot verify", frompos - self.startoffset, startpos, endpos, topos - self.startoffset + if frompos > _frompos: + original_data_offsets[0] = frompos - _frompos + elif frompos < _frompos: + data_offsets[0] = _frompos - frompos + + if topos > _topos: + data_offsets[1] -= topos - _topos + elif topos < _topos: + original_data_offsets[1] -= _topos - topos + + assert data_offsets[1] - data_offsets[0] == original_data_offsets[1] - original_data_offsets[0], (data_offsets[1] - data_offsets[0], original_data_offsets[1] - original_data_offsets[0]) + assert read_data[data_offsets[0]:data_offsets[1]] == original_data[original_data_offsets[0]:original_data_offsets[1]] + + startindex = 0 + for piece in range(frompiece, topiece + 1): + piecehash = sha(read_data[startindex:startindex + self.piecesize]).digest() + + if piecehash == self.pieces[piece]: + print >> sys.stderr, "Correct piece read", piece + else: + print >> sys.stderr, "Incorrect piece read", piece, piecehash, self.pieces[piece] + allpiecesok = False + startindex += self.piecesize + + return allpiecesok def close(self, *args): self._file.close(*args) @@ -277,9 +327,6 @@ def set_vod_mode(self): self.videoinfo['mimetype'] = self.get_mimetype(filename) self.videoinfo['usercallback'] = lambda event, params : self.session.uch.perform_vod_usercallback(self, self.dlconfig['vod_usercallback'], event, params) self.videoinfo['userevents'] = self.dlconfig['vod_userevents'][:] - for t, tl, o, f in self.filepieceranges: - if f == self.videoinfo['inpath']: - self.videoinfo['filepieceranges'] = [t, tl, o] self.handle.set_sequential_download(True) @@ -328,6 +375,11 @@ def get_vod_duration(self): def get_vod_fileindex(self): return self.handle.file_priorities().index(1) + def get_vod_filesize(self): + fileindex = self.get_vod_fileindex() + file_entry = self.handle.get_torrent_info().file_at(fileindex) + return file_entry.size + def get_piece_progress(self, pieces): with self.dllock: if self.handle: @@ -399,7 +451,7 @@ def start_vod(self, complete=False): {"complete": complete, "filename": self.videoinfo["outpath"][0] if complete else None, "mimetype": self.videoinfo["mimetype"], - "stream": None if complete else VODFile(open(self.videoinfo['outpath'][0], 'rb'), self, self.videoinfo['filepieceranges']), + "stream": None if complete else VODFile(open(self.videoinfo['outpath'][0], 'rb'), self), "length": self.videoinfo['outpath'][1], "bitrate": self.videoinfo["bitrate"]}) diff --git a/Tribler/Video/VideoServer.py b/Tribler/Video/VideoServer.py index f4731b4bc31..a9d284fec55 100644 --- a/Tribler/Video/VideoServer.py +++ b/Tribler/Video/VideoServer.py @@ -408,7 +408,7 @@ def do_GET(self): # 6. Send body (completely, a Range: or an infinite stream in chunked encoding done = False while True: - data = stream.read(blocksize * 10) + data = stream.read(blocksize) if len(data) == 0: done = True