Permalink
Browse files

Refactor HTTP file streaming.

  • Loading branch information...
1 parent 61c9d6e commit 88d3ec8181705727ddb22ee1b3e72c1d069eb3c9 @reid committed Sep 6, 2011
Showing with 450 additions and 0 deletions.
  1. +46 −0 lib/middleware/onyx.js
  2. +287 −0 lib/onyx.js
  3. +117 −0 test/onyx.js
View
@@ -0,0 +1,46 @@
+/*!
+ * YUI Mocha
+ * Copyright 2011 Yahoo! Inc.
+ * Licensed under the BSD license.
+ */
+
+/**
+ * Provides streaming file functions.
+ *
+ * Connect middleware.
+ */
+module.exports = function onyxProvider (onyx) {
+
+ return function streamFilesFilter (req, res, next) {
+
+ /**
+ * Respond to this request by sending files.
+ *
+ * @param {Array|String} file The fully-qualified path to the file.
+ */
+ res.streamFiles = function streamFilesResponder (files, config) {
+ if (!config) {
+ config = {};
+ }
+
+ config.req = req;
+ config.res = res;
+ config.files = files;
+
+ onyx.streamFiles(config, function (err) {
+ if (err) {
+ if (err.code === "ENOENT" || err.code === "EISDIR") {
+ next();
+ } else {
+ next(err);
+ }
+ return;
+ }
+ res.end();
+ });
+ };
+
+ next();
+ };
+
+};
View
@@ -0,0 +1,287 @@
+/*!
+ * YUI Onyx
+ * Copyright 2011 Yahoo! Inc.
+ * Licensed under the BSD license.
+ */
+
+/**
+ * Onyx is a library for streaming files
+ * over HTTP.
+ *
+ * @module onyx
+ */
+
+/**
+ * Dependencies.
+ */
+var fs = require("fs");
+var path = require("path");
+var util = require("util");
+var assert = require("assert");
+var EventEmitter = require("events").EventEmitter;
+var mime = require("./mime");
+
+/**
+ * Onyx streams a collection of files
+ * as a response to an HTTP request.
+ */
+
+function Onyx () {
+}
+
+util.inherits(Onyx, EventEmitter);
+
+var proto = Onyx.prototype;
+
+/**
+ * Returns the values of an object as an array.
+ *
+ * @method makeArray
+ * @protected
+ * @param {Object} obj
+ * @return {Array}
+ */
+proto.makeArray = function (obj) {
+ var arr = [], key;
+ for (key in obj) {
+ arr.push(obj[key]);
+ }
+ return arr;
+}
+
+proto.statCollector = function statCollector (files, source, cb) {
+ var file = files.shift();
+ var onyx = this;
+
+ if (file) {
+ fs.stat(file, function (err, stat) {
+ if (err) {
+ return cb(err);
+ }
+ source[file] = stat;
+ statCollector.call(onyx, files, source, cb);
+ });
+ return;
+ }
+
+ var stats = onyx.makeArray(source);
+
+ var result = {
+ size : stats.reduce(function (total, stat) {
+ return total + stat.size;
+ }, 0),
+ mtime : stats.reduce(function (latest, stat) {
+ return (latest > stat.mtime) ? latest : stat.mtime;
+ }, 0),
+ files : source
+ };
+
+ cb(null, result);
+};
+
+/**
+ * Aggregate fs#stat across multiple files.
+ *
+ * The callback recieves an object with:
+ * - size: total size of all files
+ * - mtime: latest mtime
+ * - files: individual stat objects per-file
+ * @method mstat
+ * @param {Array} files
+ * @param {Function} cb Callback
+ */
+proto.mstat = function (files, cb) {
+ if (!Array.isArray(files)) {
+ throw new Error("files must be an array");
+ }
+
+ this.statCollector(files.slice(0), {}, cb);
+};
+
+/**
+ * Implementation of the actual file transfer.
+ */
+function xfer (res, files, cb) {
+ var file = files.shift();
+
+ if (!file) {
+ return cb(null);
+ }
+
+ fs.createReadStream(file
+ ).on("data", function (chunk) {
+ res.write(chunk);
+ }).on("error", cb
+ ).on("close", function xferClose () {
+ xfer(res, files, cb);
+ });
+}
+
+/**
+ * For a given baton containing stats, files, and
+ * a HTTP response, stream them to the HTTP response
+ * and callback when complete.
+ *
+ * While streaming the response, carbon-copy the
+ * file data to the baton's `cache` MochaCache
+ * instance. If a file exists in that cache,
+ * serve it from there instead of going to
+ * the filesystem.
+ *
+ * **When a cache object is specified, this
+ * implementation assumes files being served
+ * will not change during the server's operation.**
+ * If a file changes on disk and it's still in the
+ * cache, problems can occur if it falls out of the
+ * file cache before the "FS mstat" cache, since the
+ * streamed file may not match its length.
+ *
+ * @param {Object} baton The baton, containing files, res, and stat.
+ * @param {Function} cb Callback, 1-arity, the error or null.
+ */
+proto.stream = function stream (baton, cb) {
+ var res = baton.res,
+ files = baton.files.slice(0);
+
+ xfer(res, files, cb);
+};
+
+/**
+ * Given a baton of an HTTP request and response,
+ * file stats and an array of files, handle the HTTP response
+ * by writing the correct headers for the combined files
+ * using `stream` to stream the files themselves.
+ *
+ * Conditional GET and HEAD requests are supported.
+ *
+ * Scripts, stylesheets and text are assumed to be UTF-8 encoded
+ * when sending the Content-Type HTTP header.
+ *
+ * The request is **not** ended by this function. You must call
+ * res.end in your callback function. For why, see streamFiles below.
+ *
+ * @method handle
+ * @private
+ * @param {Object} baton A baton, see streamFiles below.
+ * @param {Function} cb Callback, 1-arity, the error or null.
+ */
+proto.handle = function (status, baton, cb) {
+ var onyx = this;
+ var req = baton.req;
+ var res = baton.res;
+ var stat = baton.stat;
+ var files = baton.files;
+ var headers = {};
+ var mtime = Date.parse(stat.mtime);
+
+ headers.Date = (new Date()).toUTCString();
+ headers["Last-Modified"] = (new Date(stat.mtime)).toUTCString();
+
+ // No inode on ETag, since they vary per-system.
+ headers.Etag = JSON.stringify([stat.size, mtime].join("-"));
+
+ // Conditional GET.
+ if (req.headers["if-none-match"] === headers.Etag &&
+ Date.parse(req.headers["if-modified-since"]) >= mtime) {
+ res.writeHead(304, headers);
+ cb(null);
+ } else if (req.method === "HEAD") {
+ res.writeHead(200, headers);
+ cb(null);
+ } else {
+ headers["Content-Type"] = mime.contentTypes[path.extname(files[0]).slice(1)]
+ || "application/octet-stream";
+
+ switch (headers["Content-Type"]) {
+ case "application/javascript":
+ case "text/html":
+ case "text/css":
+ case "text/plain":
+ headers["Content-Type"] += "; charset=utf-8";
+ break;
+ default:
+ delete baton.prepend;
+ delete baton.postpend;
+ break;
+ }
+
+ var size = stat.size;
+
+ if (baton.prepend) {
+ size += baton.prepend.length;
+ }
+
+ if (baton.postpend) {
+ size += baton.postpend.length;
+ }
+
+ headers["Content-Length"] = size;
+
+ res.writeHead(status, headers);
+
+ if (baton.prepend) {
+ res.write(baton.prepend);
+ }
+
+ // TODO register metrics start
+ // var beforeStream = +new Date();
+ onyx.emit("streamStart");
+ onyx.stream(baton, function (err) {
+ if (!err && baton.postpend) {
+ res.write(baton.postpend);
+ }
+ // TODO register metrics end
+ onyx.emit("streamEnd");
+ cb(err);
+ });
+ }
+}
+
+/**
+ * Given an object with an HTTP request, response and
+ * file array, respond to the request with all of the files
+ * in the same request in the order they appear in the file
+ * array.
+ *
+ * **This implementation assumes files being served
+ * will not change during the server's operation.**
+ * See the stream function above for more details.
+ *
+ * The HTTP request is **not** ended by this function. Call
+ * res.end directly in your callback after checking for an
+ * error. That's because in the event of a stream error,
+ * error handling middleware needs to be able to respond
+ * correctly: your callback should call next(err) if an
+ * error occurred, instead of calling res.end.
+ *
+ * The baton argument should be an object with the properties:
+ *
+ * - req: HTTP request.
+ * - res: HTTP response.
+ * - files: Array of files.
+ *
+ * @method streamFiles
+ * @param {Object} baton A baton.
+ * @param {Function} cb Callback, 1-arity, the error or null.
+ */
+proto.streamFiles = function (baton, cb) {
+ var onyx = this;
+
+ if (baton.files.some(function (file) {
+ return decodeURIComponent(file).indexOf("..") !== -1;
+ })) {
+ // This should be a Bad Request; however, if we got
+ // this far with an unsafe filename we'll go with a 500.
+ return cb(new Error("Relative paths not allowed."));
+ }
+
+ this.mstat(baton.files, function (err, stat) {
+ if (err) {
+ return cb(err);
+ }
+ baton.stat = stat;
+ onyx.handle(200, baton, cb);
+ });
+};
+
+exports.Onyx = Onyx;
Oops, something went wrong.

0 comments on commit 88d3ec8

Please sign in to comment.