Merge pull request #461 from kppullin/issue-371-try-2

Strip the UTF8 BOM from a UTF encoded response

index.js

@@ -764,6 +764,11 @@ Request.prototype.start = function () {
               response.body = body.toString(self.encoding)
             }
           } else if (buffer.length) {
+            // The UTF8 BOM [0xEF,0xBB,0xBF] is converted to [0xFE,0xFF] in the JS UTC16/UCS2 representation.
+            // Strip this value out when the encoding is set to 'utf8', as upstream consumers won't expect it and it breaks JSON.parse().
+            if (self.encoding === 'utf8' && buffer[0].length > 0 && buffer[0][0] === "\uFEFF") {
+              buffer[0] = buffer[0].substring(1)
+            }
             response.body = buffer.join('')
           }
 

tests/test-body.js

@@ -35,6 +35,11 @@ var tests =
     , encoding: 'hex'
     , expectBody: "efa3bfcea9e29883"
     }
+  , testGetUTF8:
+     { resp: server.createGetResponse(new Buffer([0xEF, 0xBB, 0xBF, 226, 152, 131]))
+     , encoding: "utf8"
+     , expectBody: "☃"
+     }
   , testGetJSON :
      { resp : server.createGetResponse('{"test":true}', 'application/json')
      , json : true