Permalink
Browse files

upgraded node-thrift to support utf8

  • Loading branch information...
1 parent cd7b37e commit 5b58e06977f7a5545724b7f12acc54555b715c9e @Jae Jae committed Jul 15, 2011
@@ -0,0 +1,197 @@
+/**
+ * Support for handling 64-bit int numbers in Javascript (node.js)
+ *
+ * JS Numbers are IEEE-754 binary double-precision floats, which limits the
+ * range of values that can be represented with integer precision to:
+ *
+ * 2^^53 <= N <= 2^53
+ *
+ * Int64 objects wrap a node Buffer that holds the 8-bytes of int64 data. These
+ * objects operate directly on the buffer which means that if they are created
+ * using an existing buffer then setting the value will modify the Buffer, and
+ * vice-versa.
+ *
+ * Internal Representation
+ *
+ * The internal buffer format is Big Endian. I.e. the most-significant byte is
+ * at buffer[0], the least-significant at buffer[7]. For the purposes of
+ * converting to/from JS native numbers, the value is assumed to be a signed
+ * integer stored in 2's complement form.
+ *
+ * For details about IEEE-754 see:
+ * http://en.wikipedia.org/wiki/Double_precision_floating-point_format
+ */
+
+// Useful masks and values for bit twiddling
+var MASK31 = 0x7fffffff, VAL31 = 0x80000000;
+var MASK32 = 0xffffffff, VAL32 = 0x100000000;
+
+// Map for converting hex octets to strings
+var _HEX = [];
+for (var i = 0; i < 256; i++) {
+ _HEX[i] = (i > 0xF ? '' : '0') + i.toString(16);
+}
+
+//
+// Int64
+//
+
+/**
+ * Constructor accepts any of the following argument types:
+ *
+ * new Int64(buffer[, offset=0]) - Existing Buffer with byte offset
+ * new Int64(string) - Hex string (throws if n is outside int64 range)
+ * new Int64(number) - Number (throws if n is outside int64 range)
+ * new Int64(hi, lo) - Raw bits as two 32-bit values
+ */
+var Int64 = module.exports = function(a1, a2) {
+ if (a1 instanceof Buffer) {
+ this.buffer = a1;
+ this.offset = a2 || 0;
+ } else {
+ this.buffer = this.buffer || new Buffer(8);
+ this.offset = 0;
+ this.setValue.apply(this, arguments);
+ }
+};
+
+
+// Max integer value that JS can accurately represent
+Int64.MAX_INT = Math.pow(2, 53);
+
+// Min integer value that JS can accurately represent
+Int64.MIN_INT = -Math.pow(2, 53);
+
+Int64.prototype = {
+ /**
+ * Do in-place 2's compliment. See
+ * http://en.wikipedia.org/wiki/Two's_complement
+ */
+ _2scomp: function() {
+ var b = this.buffer, o = this.offset, carry = 1;
+ for (var i = o + 7; i >= o; i--) {
+ var v = (b[i] ^ 0xff) + carry;
+ b[i] = v & 0xff;
+ carry = v >> 8;
+ }
+ },
+
+ /**
+ * Set the value. Takes any of the following arguments:
+ *
+ * setValue(string) - A hexidecimal string
+ * setValue(number) - Number (throws if n is outside int64 range)
+ * setValue(hi, lo) - Raw bits as two 32-bit values
+ */
+ setValue: function(hi, lo) {
+ var negate = false;
+ if (arguments.length == 1) {
+ if (typeof(hi) == 'number') {
+ // Simplify bitfield retrieval by using abs() value. We restore sign
+ // later
+ negate = hi < 0;
+ hi = Math.abs(hi);
+ lo = hi % VAL32;
+ hi = hi / VAL32;
+ if (hi > VAL32) throw new RangeError(hi + ' is outside Int64 range');
+ hi = hi | 0;
+ } else if (typeof(hi) == 'string') {
+ hi = (hi + '').replace(/^0x/, '');
+ lo = hi.substr(-8);
+ hi = hi.length > 8 ? hi.substr(0, hi.length - 8) : '';
+ hi = parseInt(hi, 16);
+ lo = parseInt(lo, 16);
+ } else {
+ throw new Error(hi + ' must be a Number or String');
+ }
+ }
+
+ // Technically we should throw if hi or lo is outside int32 range here, but
+ // it's not worth the effort. Anything past the 32'nd bit is ignored.
+
+ // Copy bytes to buffer
+ var b = this.buffer, o = this.offset;
+ for (var i = 7; i >= 0; i--) {
+ b[o+i] = lo & 0xff;
+ lo = i == 4 ? hi : lo >>> 8;
+ }
+
+ // Restore sign of passed argument
+ if (negate) this._2scomp();
+ },
+
+ /**
+ * Convert to a native JS number.
+ *
+ * WARNING: Do not expect this value to be accurate to integer precision for
+ * large (positive or negative) numbers!
+ *
+ * @param allowImprecise If true, no check is performed to verify the
+ * returned value is accurate to integer precision. If false, imprecise
+ * numbers (very large positive or negative numbers) will be forced to +/-
+ * Infinity.
+ */
+ toNumber: function(allowImprecise) {
+ var b = this.buffer, o = this.offset;
+
+ // Running sum of octets, doing a 2's complement
+ var negate = b[0] & 0x80, x = 0, carry = 1;
+ for (var i = 7, m = 1; i >= 0; i--, m *= 256) {
+ var v = b[o+i];
+
+ // 2's complement for negative numbers
+ if (negate) {
+ v = (v ^ 0xff) + carry;
+ carry = v >> 8;
+ v = v & 0xff;
+ }
+
+ x += v * m;
+ }
+
+ // Return Infinity if we've lost integer precision
+ if (!allowImprecise && x >= Int64.MAX_INT) {
+ return negate ? -Infinity : Infinity;
+ }
+
+ return negate ? -x : x;
+ },
+
+ /**
+ * Convert to a JS Number. Returns +/-Infinity for values that can't be
+ * represented to integer precision.
+ */
+ valueOf: function() {
+ return this.toNumber(false);
+ },
+
+ /**
+ * Return string value
+ *
+ * @param radix Just like Number#toString()'s radix
+ */
+ toString: function(radix) {
+ return this.valueOf().toString(radix || 10);
+ },
+
+ /**
+ * Return a string showing the buffer octets, with MSB on the left.
+ *
+ * @param sep separator string. default is '' (empty string)
+ */
+ toOctetString: function(sep) {
+ var out = new Array(8);
+ var b = this.buffer, o = this.offset;
+ for (var i = 0; i < 8; i++) {
+ out[i] = _HEX[b[o+i]];
+ }
+ return out.join(sep || '');
+ },
+
+ /**
+ * Pretty output in console.log
+ */
+ inspect: function() {
+ return '[Int64 value:' + this + ' octets:' + this.toOctetString(' ') + ']';
+ }
+};
@@ -0,0 +1,69 @@
+JavaScript Numbers are represented as [IEEE 754 double-precision floats](http://steve.hollasch.net/cgindex/coding/ieeefloat.html). Unfortunately, this means they lose integer precision for values beyond +/- 2^^53. For projects that need to accurately handle 64-bit ints, such as [node-thrift](https://github.com/wadey/node-thrift), a performant, Number-like class is needed. Int64 is that class.
+
+Int64 instances look and feel much like JS-native Numbers. By way of example ...
+
+ // First, let's illustrate the problem ...
+ > (0x123456789).toString(16)
+ '123456789' // <- what we expect.
+ > (0x123456789abcdef0).toString(16)
+ '123456789abcdf00' // <- Ugh! JS doesn't do big ints. :(
+
+ // So let's create a couple Int64s using the above values ...
+
+ // Require, of course
+ > Int64 = require('node-int64')
+
+ // x's value is what we expect (the decimal value of 0x123456789)
+ > x = new Int64(0x123456789)
+ [Int64 value:4886718345 octets:00 00 00 01 23 45 67 89]
+
+ // y's value is Infinity because it's outside the range of integer
+ // precision. But that's okay - it's still useful because it's internal
+ // representation (octets) is what we passed in
+ > y = new Int64('123456789abcdef0')
+ [Int64 value:Infinity octets:12 34 56 78 9a bc de f0]
+
+ // Let's do some math. Int64's behave like Numbers. (Sorry, Int64 isn't
+ // for doing 64-bit integer arithmetic (yet) - it's just for carrying
+ // around int64 values
+ > x + 1
+ 4886718346
+ > y + 1
+ Infinity
+
+ // Int64 string operations ...
+ > 'value: ' + x
+ 'value: 4886718345'
+ > 'value: ' + y
+ 'value: Infinity'
+ > x.toString(2)
+ '100100011010001010110011110001001'
+ > y.toString(2)
+ 'Infinity'
+
+ // Use JS's isFinite() method to see if the Int64 value is in the
+ // integer-precise range of JS values
+ > isFinite(x)
+ true
+ > isFinite(y)
+ false
+
+ // Get an octet string representation. (Yay, y is what we put in!)
+ > x.toOctetString()
+ '0000000123456789'
+ > y.toOctetString()
+ '123456789abcdef0'
+
+ // Finally, some other ways to create Int64s ...
+
+ // Pass hi/lo words
+ > new Int64(0x12345678, 0x9abcdef0)
+ [Int64 value:Infinity octets:12 34 56 78 9a bc de f0]
+
+ // Pass a Buffer
+ > new Int64(new Buffer([0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0]))
+ [Int64 value:Infinity octets:12 34 56 78 9a bc de f0]
+
+ // Pass a Buffer and offset
+ > new Int64(new Buffer([0,0,0,0,0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0]), 4)
+ [Int64 value:Infinity octets:12 34 56 78 9a bc de f0]
@@ -0,0 +1,12 @@
+{
+ "name" : "node-int64",
+ "description" : "Support for representing 64-bit integers in JavaScript",
+ "url" : "http://github.com/broofa/node-int64",
+ "keywords" : ["math", "integer", "int64"],
+ "author" : "Robert Kieffer <robert@broofa.com>",
+ "contributors" : [],
+ "dependencies" : [],
+ "lib" : ".",
+ "main" : "./Int64.js",
+ "version" : "0.3.0"
+}
@@ -0,0 +1,31 @@
+var assert = require('assert');
+var Int64 = require('./Int64');
+
+var args = [
+ [0], '0000000000000000', 0,
+ [1], '0000000000000001', 1,
+ [-1], 'ffffffffffffffff', -1,
+ [1e18], '0de0b6b3a7640000', 1e18,
+ ['0001234500654321'], '0001234500654321', 0x1234500654321,
+ ['0ff1234500654321'], '0ff1234500654321', 0xff1234500654300, // Imprecise!
+ [0xff12345, 0x654321], '0ff1234500654321', 0xff1234500654300, // Imprecise!
+ [0xfffaffff, 0xfffff700],'fffafffffffff700', -0x5000000000900,
+ [0xafffffff, 0xfffff700],'affffffffffff700', -0x5000000000000800, // Imprecise!
+ ['0x0000123450654321'], '0000123450654321', 0x123450654321,
+ ['0xFFFFFFFFFFFFFFFF'], 'ffffffffffffffff', -1
+];
+
+// Test constructor argments
+
+for (var i = 0; i < args.length; i += 3) {
+ var a = args[i], octets = args[i+1], number = args[i+2];
+ console.log('Testing ' + a.join(', '));
+ // Create instance
+ var x = new Int64();
+ Int64.apply(x, a);
+
+ assert.equal(x.toOctetString(), octets,
+ 'Constuctor with ' + args.join(', '));
+
+ assert.equal(x.toNumber(true), number);
+}
Oops, something went wrong.

0 comments on commit 5b58e06

Please sign in to comment.