From 704bdb5d675bab8f277a6e8198230f69538f7612 Mon Sep 17 00:00:00 2001 From: Paul Frazee Date: Sat, 6 Jun 2020 14:04:49 -0500 Subject: [PATCH] Update hyper:// to stream responses (WIP) --- app/bg/protocols/hyper.js | 137 +++++++++++--------------------------- app/package.json | 2 +- 2 files changed, 40 insertions(+), 99 deletions(-) diff --git a/app/bg/protocols/hyper.js b/app/bg/protocols/hyper.js index 8f96f29636..0a31564730 100644 --- a/app/bg/protocols/hyper.js +++ b/app/bg/protocols/hyper.js @@ -1,9 +1,10 @@ import { parseDriveUrl } from '../../lib/urls' +import { PassThrough, Transform } from 'stream' import parseRange from 'range-parser' import once from 'once' +import pump from 'pump' import * as logLib from '../logger' const logger = logLib.child({category: 'hyper', subcategory: 'hyper-scheme'}) -// import intoStream from 'into-stream' import markdown from '../../lib/markdown' import * as drives from '../hyper/drives' import * as filesystem from '../filesystem/index' @@ -21,27 +22,6 @@ const md = markdown({ highlight: undefined }) -/** - * HACK - * Electron has an issue that's causing file read streams to fail to serve - * Reading into memory seems to resolve the issue - * https://github.com/electron/electron/issues/21018 - * -prf - */ -import { PassThrough } from 'stream' -function intoStream (text) { - const rv = new PassThrough() - rv.push(text) - rv.push(null) - return rv -} - -// constants -// = - -// how long till we give up? -const REQUEST_TIMEOUT_MS = 30e3 // 30 seconds - // exported api // = @@ -79,8 +59,6 @@ export const protocolHandler = async function (request, respond) { respond({statusCode: code}) } } - var fileReadStream - var headersSent = false var drive var cspHeader = undefined @@ -273,28 +251,21 @@ export const protocolHandler = async function (request, respond) { } } - // TODO - // Electron is being really aggressive about caching and not following the headers correctly - // caching is disabled till we can figure out why - // -prf - // caching if-match - // const ETag = 'block-' + entry.offset - // if (request.headers['if-none-match'] === ETag) { - // return respondError(304, 'Not Modified') - // } - // handle range headers['Accept-Ranges'] = 'bytes' + var length var range = request.headers.Range || request.headers.range if (range) range = parseRange(entry.size, range) if (range && range.type === 'bytes') { range = range[0] // only handle first range given statusCode = 206 + length = (range.end - range.start + 1) + headers['Content-Length'] = '' + length headers['Content-Range'] = 'bytes ' + range.start + '-' + range.end + '/' + entry.size - headers['Content-Length'] = '' + (range.end - range.start + 1) } else { if (entry.size) { - headers['Content-Length'] = '' + (entry.size) + length = entry.size + headers['Content-Length'] = '' + length } } @@ -322,80 +293,50 @@ export const protocolHandler = async function (request, respond) { : content return respond({ statusCode: 200, - headers: Object.assign(headers, { - 'Content-Type': contentType - }), + headers: Object.assign(headers, {'Content-Type': contentType}), data: intoStream(content) }) } - // fetch the entry and stream the response - // HACK solution until electron issue resolved -prf - headersSent = true var mimeType = mime.identify(entry.path) if (!canExecuteHTML && mimeType.includes('text/html')) { mimeType = 'text/plain' } - Object.assign(headers, {'Content-Type': mimeType}) - var data = await checkoutFS.pda.readFile(entry.path, 'binary') - if (range) { - data = data.slice(range.start, range.end + 1) + headers['Content-Type'] = mimeType + if (request.method === 'HEAD') { + respond({statusCode: 204, headers, data: intoStream('')}) + } else { + respond({ + statusCode, + headers, + data: pump( + checkoutFS.session.drive.createReadStream(entry.path, range), + chunkLogger(entry.path, length) + ) + }) } - respond({ - statusCode, - headers, - data: intoStream(data) - }) }) - /*fileReadStream = checkoutFS.pda.createReadStream(entry.path, range) - var dataStream = fileReadStream - .pipe(mime.identifyStream(entry.path, mimeType => { - - // disable html as needed - if (!canExecuteHTML && mimeType.includes('html')) { - mimeType = 'text/plain' - } - - // send headers, now that we can identify the data - headersSent = true - Object.assign(headers, { - 'Content-Type': mimeType - }) - // TODO - // Electron is being really aggressive about caching and not following the headers correctly - // caching is disabled till we can figure out why - // -prf - // if (ETag) { - // Object.assign(headers, {ETag}) - // } else { - // Object.assign(headers, {'Cache-Control': 'no-cache'}) - // } +} - if (request.method === 'HEAD') { - dataStream.destroy() // stop reading data - respond({statusCode: 204, headers, data: intoStream('')}) - } else { - respond({statusCode, headers, data: dataStream}) - } - })) +function intoStream (text) { + const rv = new PassThrough() + rv.push(text) + rv.push(null) + return rv +} - // handle empty files - fileReadStream.once('end', () => { - if (!headersSent) { - respond({ - statusCode: 200, - headers: { - 'Content-Security-Policy': cspHeader, - 'Access-Control-Allow-Origin': '*' - }, - data: intoStream('') - }) +// DEBUG +function chunkLogger (path, len) { + var n = 0 + return new Transform({ + transform (chunk, encoding, callback) { + console.log(path, 'len', len, 'n', n, 'chunk', chunk.length) + n += chunk.length + return callback(null, chunk) + this.push(chunk) + n += chunk.length + if (len && n >= len) this.push(null) + callback() } }) - - // handle read-stream errors - fileReadStream.once('error', err => { - logger.warn('Error reading file', {url: drive.url, path: entry.path, err}) - if (!headersSent) respondError(500, 'Failed to read file') - })*/ } diff --git a/app/package.json b/app/package.json index c669056723..4b62222030 100644 --- a/app/package.json +++ b/app/package.json @@ -34,7 +34,7 @@ "fs-jetpack": "^1.3.1", "fs-reverse": "0.0.3", "function-queue": "0.0.12", - "hyperdrive-daemon": "^1.13.16", + "hyperdrive-daemon": "^1.13.18", "hyperdrive-daemon-client": "^1.15.2", "icojs": "^0.12.3", "identify-filetype": "^1.0.0",