Support all encodings from iconv-lite

closes #22

HISTORY.md

@@ -1,3 +1,9 @@
+unreleased
+==========
+
+  * Passing string as `options` interpreted as encoding
+  * Support all encodings from `iconv-lite`
+
 1.1.7 / 2014-06-12
 ==================
 

README.md

@@ -12,12 +12,13 @@ Ideal for parsing request bodies.
 
 ```js
 var getRawBody = require('raw-body')
+var typer      = require('media-typer')
 
 app.use(function (req, res, next) {
   getRawBody(req, {
     length: req.headers['content-length'],
     limit: '1mb',
-    encoding: 'utf8'
+    encoding: typer.parse(req.headers['content-type']).parameters.charset
   }, function (err, string) {
     if (err)
       return next(err)
@@ -35,7 +36,7 @@ app.use(function* (next) {
   var string = yield getRawBody(this.req, {
     length: this.length,
     limit: '1mb',
-    encoding: 'utf8'
+    encoding: this.charset
   })
 })
 ```
@@ -55,8 +56,9 @@ Options:
 - `encoding` - The requested encoding.
   By default, a `Buffer` instance will be returned.
   Most likely, you want `utf8`.
-  You can use any type of encoding supported by [StringDecoder](http://nodejs.org/api/string_decoder.html).
-  You can also pass `true` which sets it to the default `utf8`
+  You can use any type of encoding supported by [iconv-lite](https://www.npmjs.org/package/iconv-lite#readme).
+
+You can also pass a string in place of options to just specify the encoding.
 
 `callback(err, res)`:
 

index.js

@@ -1,20 +1,26 @@
 var bytes = require('bytes')
-
-// NOTE: the trailing slash is not a typo
-var StringDecoder = require('string_decoder/').StringDecoder
+var iconv = require('iconv-lite')
 
 module.exports = function (stream, options, done) {
+  if (options === true || typeof options === 'string') {
+    // short cut for encoding
+    options = {
+      encoding: options
+    }
+  }
+
+  options = options || {}
+
   if (typeof options === 'function') {
     done = options
     options = {}
-  } else if (!options) {
-    options = {}
-  } else if (options === true) {
-    options = {
-      encoding: 'utf8'
-    }
   }
 
+  // get encoding
+  var encoding = options.encoding !== true
+    ? options.encoding
+    : 'utf-8'
+
   // convert the limit to an integer
   var limit = null
   if (typeof options.limit === 'number')
@@ -65,10 +71,7 @@ module.exports = function (stream, options, done) {
   }
 
   var received = 0
-  // note: we delegate any invalid encodings to the constructor
-  var decoder = options.encoding
-    ? new StringDecoder(options.encoding === true ? 'utf8' : options.encoding)
-    : null
+  var decoder = getDecoder(encoding)
   var buffer = decoder
     ? ''
     : []
@@ -117,7 +120,7 @@ module.exports = function (stream, options, done) {
       done(err)
     } else {
       done(null, decoder
-        ? buffer + decoder.end()
+        ? buffer + (decoder.end() || '')
         : Buffer.concat(buffer)
       )
     }
@@ -135,6 +138,19 @@ module.exports = function (stream, options, done) {
   }
 }
 
+function getDecoder(encoding) {
+  if (!encoding) return null
+
+  try {
+    return iconv.getCodec(encoding).decoder()
+  } catch (e) {
+    var err = makeError('specified encoding unsupported', 'encoding.unsupported')
+    err.status = err.statusCode = 415
+    err.encoding = encoding
+    throw err
+  }
+}
+
 // to create serializable errors you must re-set message so
 // that it is enumerable and you must re configure the type
 // property so that is writable and enumerable

package.json

@@ -7,7 +7,7 @@
   "repository": "stream-utils/raw-body",
   "dependencies": {
     "bytes": "1",
-    "string_decoder": "0.10"
+    "iconv-lite": "0.4.2"
   },
   "devDependencies": {
     "istanbul": "0.2.10",

test/index.js

@@ -12,16 +12,6 @@ var file = path.join(__dirname, 'index.js')
 var length = fs.statSync(file).size
 var string = fs.readFileSync(file, 'utf8')
 
-function createStream() {
-  return fs.createReadStream(file)
-}
-
-function checkBuffer(buf) {
-  assert.ok(Buffer.isBuffer(buf))
-  assert.equal(buf.length, length)
-  assert.equal(buf.toString('utf8'), string)
-}
-
 describe('Raw Body', function () {
   it('should work without any options', function (done) {
     getRawBody(createStream(), function (err, buf) {
@@ -228,11 +218,17 @@ describe('Raw Body', function () {
   })
 
   it('should throw when given an invalid encoding', function () {
-    assert.throws(function () {
+    var err
+    try {
       getRawBody(new Readable(), {
         encoding: 'akljsdflkajsdf'
       }, function () {})
-    })
+    } catch (e) {
+      err = e
+    }
+    assert.ok(err)
+    assert.ok(/encoding/.test(err.message))
+    assert.equal(err.status, 415)
   })
 
   describe('when an encoding is set', function () {
@@ -256,10 +252,47 @@ describe('Raw Body', function () {
       })
     })
 
+    it('should handle encoding as options string', function (done) {
+      getRawBody(createStream(), 'utf8', function (err, str) {
+        assert.ifError(err)
+        assert.equal(str, string)
+        done()
+      })
+    })
+
+    it('should decode codepage string', function (done) {
+      var stream = createStream(new Buffer('bf43f36d6f20657374e1733f', 'hex'))
+      var string = '¿Cómo estás?'
+      getRawBody(stream, 'iso-8859-1', function (err, str) {
+        assert.ifError(err)
+        assert.equal(str, string)
+        done()
+      })
+    })
+
+    it('should decode UTF-8 string', function (done) {
+      var stream = createStream(new Buffer('c2bf43c3b36d6f20657374c3a1733f', 'hex'))
+      var string = '¿Cómo estás?'
+      getRawBody(stream, 'utf-8', function (err, str) {
+        assert.ifError(err)
+        assert.equal(str, string)
+        done()
+      })
+    })
+
+    it('should decode UTF-16LE string', function (done) {
+      // UTF-16LE is different from UTF-16 due to BOM behavior
+      var stream = createStream(new Buffer('bf004300f3006d006f002000650073007400e10073003f00', 'hex'))
+      var string = '¿Cómo estás?'
+      getRawBody(stream, 'utf-16le', function (err, str) {
+        assert.ifError(err)
+        assert.equal(str, string)
+        done()
+      })
+    })
+
     it('should correctly calculate the expected length', function (done) {
-      var stream = new Readable()
-      stream.push('{"test":"å"}')
-      stream.push(null)
+      var stream = createStream(new Buffer('{"test":"å"}'))
 
       getRawBody(stream, {
         encoding: 'utf8',
@@ -354,3 +387,21 @@ describe('Raw Body', function () {
     })
   })
 })
+
+function checkBuffer(buf) {
+  assert.ok(Buffer.isBuffer(buf))
+  assert.equal(buf.length, length)
+  assert.equal(buf.toString('utf8'), string)
+}
+
+function createStream(buf) {
+  if (!buf) return fs.createReadStream(file)
+
+  var stream = new Readable()
+  stream._read = function () {
+    stream.push(buf)
+    stream.push(null)
+  }
+
+  return stream
+}