[CLOSED] Implement File.createReadableStream

[feross]

Issue by fisch0920
Wednesday May 14, 2014 at 08:53 GMT
Originally opened as https://github.com/feross/bittorrent-client/issues/12

---

Aiming to use torrent_stream's implementation as a starting point.

@feross I've already started delving into this, and it's not hard with Storage containing a flat Buffer of all the data, but I really think Storage needs to be mostly redone to support asynchronous writes/reads ala torrent_stream for when Storage is backed by a set of files and not a huge Buffer.

In a perfect world, we could use one implementation that utilized browserify-fs, but AFAIK there is no node fs replacement with solid cross-browser support yet :(

My current plan is to refactor Storage to have an async interface with a synchronous default implementation similar to what exists currently and then use a subclass of Storage in the webtorrent repo that's implemented on top of fs so bittorrent-client will remain compatible with browserify, but I'd really like input here since this is a heftier design change.

[feross]

Comment by fisch0920
Wednesday May 14, 2014 at 10:05 GMT

---

To make my proposal above more concrete, here is a preview of the changes I'm suggesting to Storage (untested):

/**
 * Storage for a torrent download. Handles the complexities of reading and writing
 * to pieces and files.
 *
 * @param {Object} parsedTorrent
 * @param {Object} opts
 */
function Storage (parsedTorrent, opts) {
  var self = this
  EventEmitter.call(self)
  opts = opts || {}

  self.bitfield = new BitField(parsedTorrent.pieces.length)

  self.done = false
  self.log = opts.log || console.log
  self.selection = []
  self.critical = []

  self._init(parsedTorrent, opts)
}

...

/**
 * Initializes the underlying backing store used by this Storage, including 
 * this.pieces and this.files.
 *
 * @param {Object} parsedTorrent
 * @param {Object} opts
 */
Storage.prototype._init = function (parsedTorrent, opts) {
  var self = this

  var pieceLength = parsedTorrent.pieceLength
  var lastPieceLength = parsedTorrent.lastPieceLength
  var numPieces = parsedTorrent.pieces.length

  self.buffer = new Buffer(parsedTorrent.length)

  self.pieces = parsedTorrent.pieces.map(function (hash, index) {
    var start = index * pieceLength
    var end = start + (index === numPieces - 1 ? lastPieceLength : pieceLength)
    var buffer = self.buffer.slice(start, end) // references same memory

    var piece = new Piece(index, hash, buffer)
    piece.on('done', self._onPieceDone.bind(self, piece))
    return piece
  })

  self.files = parsedTorrent.files.map(function (fileObj) {
    var start = fileObj.offset
    var end = start + fileObj.length
    var buffer = self.buffer.slice(start, end) // references same memory

    var startPiece = Math.floor(start / pieceLength)
    var endPiece = Math.floor((end - 1) / pieceLength)
    var pieces = self.pieces.slice(startPiece, endPiece + 1)

    var file = new File(self, fileObj, buffer, pieces, pieceLength)
    file.on('done', self._onFileDone.bind(self, file))
    return file
  })
}

/**
 * Reads a block from a piece.
 * 
 * @param {number}    index    piece index
 * @param {number}    offset   byte offset within piece
 * @param {number}    length   length in bytes to read from piece
 */
Storage.prototype.readBlock = function (index, offset, length, cb) {
  var self = this
  if (!cb) return console.error('Storage.readBlock requires a callback')

  var piece = self.pieces[index]
  if (!piece) return cb(new Error("invalid piece index " + index))

  piece.readBlock(offset, length, cb)
}

/**
 * Writes a block to a piece.
 * 
 * @param {number}  index    piece index
 * @param {number}  offset   byte offset within piece
 * @param {Buffer}  buffer   buffer to write
 */
Storage.prototype.writeBlock = function (index, offset, buffer, cb) {
  var self = this
  if (!cb) cb = noop

  var piece = self.pieces[index]
  if (!piece) return cb(new Error("invalid piece index " + index))

  piece.writeBlock(offset, buffer, cb)
}

/**
 * Reads a piece or a range of a piece.
 * 
 * @param {number}  index         piece index
 * @param {Object}  range         optional range within piece
 * @param {number}  range.offset
 * @param {number}  range.length
 * @param {number}  length        length in bytes to read from piece
 */
Storage.prototype.read = function (index, range, cb) {
  var self = this
  if (!cb) return console.error('Storage.read requires a callback')

  var piece = self.pieces[index]
  if (!piece) return cb(new Error("invalid piece index " + index))
  if (!piece.done) return cb(new Error("Storage.read called on incomplete piece " + index))

  var offset = 0
  var length = piece.length

  if (typeof range === "function") {
    cb = range
  } else if (range) {
    offset = range.offset || 0
    length = range.length || length
  }

  if (piece.buffer) {
    // shortcut for piece with static backing buffer
    return cb(null, piece.buffer.slice(offset, offset + length))
  }

  var blocks = []
  function readNextBlock = function () {
    if (length <= 0) return cb(null, Buffer.concat(blocks))

    var blockOffset = offset
    var blockLength = Math.min(BLOCK_LENGTH, length)

    offset += blockLength
    length -= blockLength

    self.readBlock(index, blockOffset, blockLength, function (err, block) {
      if (err) return cb(err)

      blocks.push(block)
      readNextBlock()
    })
  }

  readNextBlock()
}
...

Webtorrent would then contain a FSStorage subclass which just overrides _init.

[feross]

Comment by feross
Sunday May 18, 2014 at 07:19 GMT

---

The browserify fs situation will be interesting to solve. What implementations have you looked at?

[feross]

Comment by fisch0920
Sunday May 18, 2014 at 09:20 GMT

---

https://www.npmjs.org/package/browserify-fs seems to be the frontrunner, but it depends on level.js which is currently only supported on chrome and opera.

https://github.com/js-platform/filer also seems like a working solution, though again I'd be hesitant to add a dependency that seems so immature before the browserify community has chosen a direction.

[feross]

Comment by fisch0920
Friday May 23, 2014 at 16:50 GMT

---

This should probably be closed; if we want to support browser-compatible disk backed storage, it deserves its own issue.