From 1e6124746cbd28d7353275888ceb0d468b2c9c16 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Thu, 27 Jan 2011 15:10:45 +0100 Subject: [PATCH 01/37] conversion pg text to binary protocol started --- lib/connection.js | 9 +- lib/query.js | 133 ++++++++++++++++-- lib/result.js | 32 +++++ lib/writer.js | 48 ++++--- package.json | 4 +- script/list-db-types.js | 30 +--- test/integration/client/api-tests.js | 18 ++- test/integration/client/array-tests.js | 33 +++++ .../client/result-metadata-tests.js | 25 ++++ .../integration/client/type-coercion-tests.js | 2 +- test/test-helper.js | 22 ++- test/unit/client/prepared-statement-tests.js | 2 +- test/unit/client/query-tests.js | 30 +++- test/unit/client/result-metadata-tests.js | 39 +++++ ...esults.js => typed-query-results-tests.js} | 9 +- test/unit/writer-tests.js | 24 ++++ 16 files changed, 388 insertions(+), 72 deletions(-) create mode 100644 lib/result.js create mode 100644 test/integration/client/array-tests.js create mode 100644 test/integration/client/result-metadata-tests.js create mode 100644 test/unit/client/result-metadata-tests.js rename test/unit/client/{typed-query-results.js => typed-query-results-tests.js} (90%) diff --git a/lib/connection.js b/lib/connection.js index b2adbd879..d232d4f5b 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -148,7 +148,8 @@ p.bind = function(config) { buffer.addString(val); } } - buffer.addInt16(0); //no format codes, use text + buffer.addInt16(1); // format codes to use binary + buffer.addInt16(1); //0x42 = 'B' this.send(0x42, buffer.flush()); }; @@ -365,7 +366,7 @@ p.parseD = function(msg) { var fields = []; for(var i = 0; i < fieldCount; i++) { var length = this.parseInt32(); - fields[i] = (length === -1 ? null : this.readString(length)) + fields[i] = (length === -1 ? null : this.readBytes(length)) }; msg.fieldCount = fieldCount; msg.fields = fields; @@ -434,6 +435,10 @@ p.readString = function(length) { return this.buffer.toString(this.encoding, this.offset, (this.offset += length)); }; +p.readBytes = function(length) { + return this.buffer.slice(this.offset, this.offset += length); +}; + p.parseCString = function() { var start = this.offset; while(this.buffer[this.offset++]) { }; diff --git a/lib/query.js b/lib/query.js index de64c4161..50983b6e9 100644 --- a/lib/query.js +++ b/lib/query.js @@ -37,6 +37,7 @@ p.submit = function(connection) { var names = []; var rows = []; var handleRowDescription = function(msg) { + console.log(JSON.stringify(msg)); for(var i = 0; i < msg.fields.length; i++) { converters[i] = dataTypeParsers[msg.fields[i].dataTypeID] || noParse; names[i] = msg.fields[i].name; @@ -47,6 +48,7 @@ p.submit = function(connection) { for(var i = 0; i < msg.fields.length; i++) { var rawValue = msg.fields[i]; result[names[i]] = rawValue === null ? null : converters[i](rawValue); + console.log(names[i] + ": " + result[names[i]]); } self.emit('row', result); @@ -206,22 +208,135 @@ var dateParser = function(isoDate) { return date; }; +function shl(a,b) { + // Copyright (c) 1996 Henri Torgemane. All Rights Reserved. + // fix for crappy << + for (var i=0;i 0) { + lastValue += precisionBitsCounter; + } + } + + return lastValue; + }; + + var mantissa = parseBits(data, precisionBits, exponentBits + 1, parsePrecisionBits); + + // special cases + if (exponent == (Math.pow(2, exponentBits + 1) - 1)) { + if (mantissa == 0) { + return (sign == 0) ? Infinity : -Infinity; + } + + return NaN; + } + + // normale number + return ((sign == 0) ? 1 : -1) * Math.pow(2, exponent - bias) * mantissa; +}; + +var parseBits = function(data, bits, offset, callback) { + offset = offset || 0; + callback = callback || function(lastValue, newValue, bits) { return (lastValue * Math.pow(2, bits)) + newValue; }; + var offsetBytes = offset >> 3; + + // read first (maybe partial) byte + var mask = 0xff; + var firstBits = 8 - (offset % 8); + if (bits < firstBits) { + mask = (0xff << (8 - bits)) & 0xff; + firstBits = bits; + } + + if (offset) { + mask = mask >> (offset % 8); + } + var result = callback(0, data[offsetBytes] & mask, firstBits); + + // read bytes + var bytes = (bits + offset) >> 3; + for (var i = offsetBytes + 1; i < bytes; i++) { + result = callback(result, data[i], 8); + } + + // bits to read, that are not a complete byte + var lastBits = (bits + offset) % 8; + if (lastBits > 0) { + result = callback(result, data[bytes] >> (8 - lastBits), lastBits); + } + + return result; +} + +var parseBinaryInt64 = function(value) { + return parseBits(value, 64); +} + +var parseBinaryInt32 = function(value) { + return parseBits(value, 32); +} + +var parseBinaryInt16 = function(value) { + return parseBits(value, 16); +} + +var parseBinaryFloat32 = function(value) { + return parseFloat(value, 23, 8); +} + +var parseBinaryFloat64 = function(value) { + return parseFloat(value, 52, 11); +} + // To help we test dateParser Query.dateParser = dateParser; var dataTypeParsers = { - 20: parseInt, - 21: parseInt, - 23: parseInt, - 26: parseInt, - 1700: parseFloat, - 700: parseFloat, - 701: parseFloat, 16: function(dbVal) { //boolean - return dbVal === 't'; + console.log(JSON.stringify(dbVal)); + return value[0] == 1; }, - 1114: dateParser, + + 20: parseBinaryInt64, + 21: parseBinaryInt16, + 23: parseBinaryInt32, + 26: parseBinaryInt64, + 700: parseBinaryFloat32, + 701: parseBinaryFloat64, +// 1009: arrayParser, + 1114: parseBinaryInt64, // TOFIX: dateParser, 1184: dateParser + // 1700: parseFloat, }; diff --git a/lib/result.js b/lib/result.js new file mode 100644 index 000000000..f46ed4185 --- /dev/null +++ b/lib/result.js @@ -0,0 +1,32 @@ +//result object returned from query +//in the 'end' event and also +//passed as second argument to provided callback +var Result = function() { + this.rows = []; +}; + +var p = Result.prototype; + + +var matchRegexp = /([A-Za-z]+) (\d+ )?(\d+)?/ + +//adds a command complete message +p.addCommandComplete = function(msg) { + var match = matchRegexp.exec(msg.text); + if(match) { + this.command = match[1]; + //match 3 will only be existing on insert commands + if(match[3]) { + this.rowCount = parseInt(match[3]); + this.oid = parseInt(match[2]); + } else { + this.rowCount = parseInt(match[2]); + } + } +}; + +p.addRow = function(row) { + this.rows.push(row); +}; + +module.exports = Result; diff --git a/lib/writer.js b/lib/writer.js index 03e35524a..f87ba29f3 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -1,25 +1,18 @@ var Writer = function(size) { this.size = size || 1024; - this.buffer = new Buffer(this.size); - this.offset = 0; + this.buffer = new Buffer(this.size + 5); + this.offset = 5; }; var p = Writer.prototype; -p._remaining = function() { - return this.buffer.length - this.offset; -} - -p._resize = function() { - var oldBuffer = this.buffer; - this.buffer = Buffer(oldBuffer.length + this.size); - oldBuffer.copy(this.buffer); -} - //resizes internal buffer if not enough size left p._ensure = function(size) { - if(this._remaining() < size) { - this._resize() + var remaining = this.buffer.length - this.offset; + if(remaining < size) { + var oldBuffer = this.buffer; + this.buffer = Buffer(oldBuffer.length + size); + oldBuffer.copy(this.buffer); } } @@ -46,7 +39,7 @@ p.addCString = function(string) { this.buffer.write(string, this.offset); this.offset += len; this.buffer[this.offset] = 0; //add null terminator - return this; + return this; } p.addChar = function(char) { @@ -56,10 +49,6 @@ p.addChar = function(char) { return this; } -p.join = function() { - return this.buffer.slice(0, this.offset); -} - p.addString = function(string) { var string = string || ""; var len = Buffer.byteLength(string); @@ -70,7 +59,7 @@ p.addString = function(string) { } p.getByteLength = function() { - return this.offset; + return this.offset - 5; } p.add = function(otherBuffer) { @@ -81,11 +70,24 @@ p.add = function(otherBuffer) { } p.clear = function() { - this.offset=0; + this.offset=5; +} + +p.join = function(code) { + if(code) { + var end = this.offset; + this.offset = 0; + this.buffer[this.offset++] = code; + //write the length which is length of entire packet not including + //message type code byte + this.addInt32(end - 1); + this.offset = end; + } + return this.buffer.slice(code ? 0 : 5, this.offset); } -p.flush = function() { - var result = this.join(); +p.flush = function(code) { + var result = this.join(code); this.clear(); return result; } diff --git a/package.json b/package.json index ed58735b1..ab47aeafb 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { "name": "pg", - "version": "0.2.3", + "version": "0.2.6", "description": "Pure JavaScript PostgreSQL client", "homepage": "http://github.com/brianc/node-postgres", "repository" : { @@ -9,6 +9,6 @@ "author" : "Brian Carlson ", "main" : "./lib/index", "directories" : { "lib" : "./lib" }, - "scripts" : { "test" : "node ./test/run.js" }, + "scripts" : { "test" : "make test" }, "engines" : { "node": ">= 0.2.2" } } diff --git a/script/list-db-types.js b/script/list-db-types.js index d2f02c1cd..71b4ab7e5 100644 --- a/script/list-db-types.js +++ b/script/list-db-types.js @@ -1,24 +1,6 @@ -var net = require('net') -var Connection = require(__dirname+'/../lib/connection'); - -var con = new Connection({stream: new net.Stream()}); -con.connect('5432', 'localhost'); - -con.on('connect', function() { - con.startup({ - user: 'brian', - database: 'postgres' - }); -}); - -con.on('dataRow', function(msg) { - console.log(msg.fields); -}); - -con.on('readyForQuery', function() { - con.query('select oid, typname from pg_type where typtype = \'b\' order by typname'); -}); - -con.on('commandComplete', function() { - con.end(); -}); +var helper = require(__dirname + "/../test/integration/test-helper"); +var pg = helper.pg; +pg.connect(helper.connectionString(), assert.success(function(client) { + var query = client.query('select oid, typname from pg_type where typtype = \'b\' order by oid'); + query.on('row', console.log); +})) diff --git a/test/integration/client/api-tests.js b/test/integration/client/api-tests.js index bdffbee58..428b1b714 100644 --- a/test/integration/client/api-tests.js +++ b/test/integration/client/api-tests.js @@ -6,7 +6,7 @@ var log = function() { //console.log.apply(console, arguments); } -var sink = new helper.Sink(4, 10000, function() { +var sink = new helper.Sink(5, 10000, function() { log("ending connection pool: %s", connectionString); pg.end(connectionString); }); @@ -92,3 +92,19 @@ test("query errors are handled and do not bubble if callback is provded", functi })) })) }) + +test('callback is fired once and only once', function() { + pg.connect(connectionString, assert.calls(function(err, client) { + assert.isNull(err); + client.query("CREATE TEMP TABLE boom(name varchar(10))"); + var callCount = 0; + client.query([ + "INSERT INTO boom(name) VALUES('hai')", + "INSERT INTO boom(name) VALUES('boom')", + "INSERT INTO boom(name) VALUES('zoom')", + ].join(";"), function(err, callback) { + assert.equal(callCount++, 0, "Call count should be 0. More means this callback fired more than once."); + sink.add(); + }) + })) +}) diff --git a/test/integration/client/array-tests.js b/test/integration/client/array-tests.js new file mode 100644 index 000000000..c45882d14 --- /dev/null +++ b/test/integration/client/array-tests.js @@ -0,0 +1,33 @@ +var helper = require(__dirname + "/test-helper"); +var pg = helper.pg; +var conString = helper.connectionString(); + +test('parsing array results', function() { + pg.connect(conString, assert.calls(function(err, client) { + assert.isNull(err); + client.query("CREATE TEMP TABLE why(names text[], numbors integer[])"); + client.query('INSERT INTO why(names, numbors) VALUES(\'{"aaron", "brian","a b c" }\', \'{1, 2, 3}\')').on('error', console.log); + test('numbers', function() { + // client.connection.on('message', console.log) + client.query('SELECT numbors FROM why', assert.success(function(result) { + assert.length(result.rows[0].numbors, 3); + assert.equal(result.rows[0].numbors[0], 1); + assert.equal(result.rows[0].numbors[1], 2); + assert.equal(result.rows[0].numbors[2], 3); + })) + }) + + test('parses string arrays', function() { + client.query('SELECT names FROM why', assert.success(function(result) { + var names = result.rows[0].names; + assert.length(names, 3); + assert.equal(names[0], 'aaron'); + assert.equal(names[1], 'brian'); + assert.equal(names[2], "a b c"); + pg.end(); + })) + }) + })) +}) + + diff --git a/test/integration/client/result-metadata-tests.js b/test/integration/client/result-metadata-tests.js new file mode 100644 index 000000000..2f466f6fb --- /dev/null +++ b/test/integration/client/result-metadata-tests.js @@ -0,0 +1,25 @@ +var helper = require(__dirname + "/test-helper"); +var pg = helper.pg; +var conString = helper.connectionString(); + +pg.connect(conString, assert.calls(function(err, client) { + assert.isNull(err); + client.query("CREATE TEMP TABLE zugzug(name varchar(10))", assert.calls(function(err, result) { + assert.isNull(err); + //let's list this as ignored for now + // process.nextTick(function() { + // test('should identify "CREATE TABLE" message', function() { + // return false; + // assert.equal(result.command, "CREATE TABLE"); + // assert.equal(result.rowCount, 0); + // }) + // }) + assert.equal(result.oid, null); + client.query("INSERT INTO zugzug(name) VALUES('more work?')", assert.calls(function(err, result) { + assert.equal(result.command, "INSERT"); + assert.equal(result.rowCount, 1); + process.nextTick(client.end.bind(client)); + return false; + })) + })) +})) diff --git a/test/integration/client/type-coercion-tests.js b/test/integration/client/type-coercion-tests.js index 8faf51a33..6d99de645 100644 --- a/test/integration/client/type-coercion-tests.js +++ b/test/integration/client/type-coercion-tests.js @@ -107,6 +107,7 @@ test("timestampz round trip", function() { text: 'select * from date_tests where name = $1', values: ['now'] }); + assert.emits(result, 'row', function(row) { var date = row.tstz; assert.equal(date.getYear(),now.getYear()); @@ -118,7 +119,6 @@ test("timestampz round trip", function() { test("milliseconds are equal", function() { assert.equal(date.getMilliseconds(), now.getMilliseconds()); }); - }); client.on('drain', client.end.bind(client)); diff --git a/test/test-helper.js b/test/test-helper.js index fbeaa4d24..7f01fa129 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -64,18 +64,21 @@ assert.UTCDate = function(actual, year, month, day, hours, min, sec, milisecond) assert.equal(actualMili, milisecond, "expected milisecond " + milisecond + " but got " + actualMili); }; +var spit = function(actual, expected) { + console.log(""); + console.log("actual " + sys.inspect(actual)); + console.log("expect " + sys.inspect(expected)); + console.log(""); +} + assert.equalBuffers = function(actual, expected) { if(actual.length != expected.length) { - console.log(""); - console.log("actual " + sys.inspect(actual)); - console.log("expect " + sys.inspect(expected)); - console.log(""); + spit(actual, expected) assert.equal(actual.length, expected.length); } for(var i = 0; i < actual.length; i++) { if(actual[i] != expected[i]) { - console.log(actual); - console.log(expected); + spit(actual, expected) } assert.equal(actual[i],expected[i]); } @@ -85,6 +88,13 @@ assert.empty = function(actual) { assert.length(actual, 0); }; +assert.success = function(callback) { + return assert.calls(function(err, arg) { + assert.isNull(err); + callback(arg); + }) +} + assert.length = function(actual, expectedLength) { assert.equal(actual.length, expectedLength); diff --git a/test/unit/client/prepared-statement-tests.js b/test/unit/client/prepared-statement-tests.js index 209e3464e..2d3b77b95 100644 --- a/test/unit/client/prepared-statement-tests.js +++ b/test/unit/client/prepared-statement-tests.js @@ -23,7 +23,7 @@ con.execute = function(arg) { executeArg = arg; process.nextTick(function() { con.emit('rowData',{ fields: [] }); - con.emit('commandComplete'); + con.emit('commandComplete', { text: "" }); }); }; diff --git a/test/unit/client/query-tests.js b/test/unit/client/query-tests.js index ae5842f4b..a024d189b 100644 --- a/test/unit/client/query-tests.js +++ b/test/unit/client/query-tests.js @@ -2,10 +2,36 @@ var helper = require(__dirname + '/test-helper'); var q = require('query') test("testing dateParser", function() { - assert.equal(q.dateParser("2010-12-11 09:09:04").toUTCString(),new Date("2010-12-11 09:09:04 GMT").toUTCString()); + assert.equal(q.dateParser("2010-12-11 09:09:04").toUTCString(),new Date("2010-12-11 09:09:04 GMT").toUTCString()); +}); + +var testForMs = function(part, expected) { + var dateString = "2010-01-01 01:01:01" + part; + test('testing for correcting parsing of ' + dateString, function() { + var ms = q.dateParser(dateString).getMilliseconds(); + assert.equal(ms, expected) + }) +} + +testForMs('.1', 100); +testForMs('.01', 10); +testForMs('.74', 740); + +test("testing 2dateParser", function() { + var actual = "2010-12-11 09:09:04.1"; + var expected = "\"2010-12-11T09:09:04.100Z\""; + assert.equal(JSON.stringify(q.dateParser(actual)),expected); +}); + +test("testing 2dateParser", function() { + var actual = "2011-01-23 22:15:51.28-06"; + var expected = "\"2011-01-24T04:15:51.280Z\""; + assert.equal(JSON.stringify(q.dateParser(actual)),expected); }); test("testing 2dateParser", function() { - assert.equal(JSON.stringify(q.dateParser("2010-12-11 09:09:04.19")),"\"2010-12-11T09:09:04.190Z\""); + var actual = "2011-01-23 22:15:51.280843-06"; + var expected = "\"2011-01-24T04:15:51.280Z\""; + assert.equal(JSON.stringify(q.dateParser(actual)),expected); }); diff --git a/test/unit/client/result-metadata-tests.js b/test/unit/client/result-metadata-tests.js new file mode 100644 index 000000000..4a04df7fd --- /dev/null +++ b/test/unit/client/result-metadata-tests.js @@ -0,0 +1,39 @@ +var helper = require(__dirname + "/test-helper") + +var testForTag = function(tagText, callback) { + test('includes command tag data for tag ' + tagText, function() { + + var client = helper.client(); + client.connection.emit('readyForQuery') + + var query = client.query("whatever"); + assert.length(client.connection.queries, 1) + + assert.emits(query, 'end', function(result) { + assert.ok(result != null, "should pass something to this event") + callback(result) + }) + + client.connection.emit('commandComplete', { + text: tagText + }); + + client.connection.emit('readyForQuery'); + }) +} + +var check = function(oid, rowCount, command) { + return function(result) { + if(oid != null) { + assert.equal(result.oid, oid); + } + assert.equal(result.rowCount, rowCount); + assert.equal(result.command, command); + } +} + +testForTag("INSERT 0 3", check(0, 3, "INSERT")); +testForTag("INSERT 841 1", check(841, 1, "INSERT")); +testForTag("DELETE 10", check(null, 10, "DELETE")); +testForTag("UPDATE 11", check(null, 11, "UPDATE")); +testForTag("SELECT 20", check(null, 20, "SELECT")); diff --git a/test/unit/client/typed-query-results.js b/test/unit/client/typed-query-results-tests.js similarity index 90% rename from test/unit/client/typed-query-results.js rename to test/unit/client/typed-query-results-tests.js index d6dbb7987..77b3bedaa 100644 --- a/test/unit/client/typed-query-results.js +++ b/test/unit/client/typed-query-results-tests.js @@ -68,9 +68,16 @@ test('typed results', function() { dataTypeID: 1184, actual: '2010-10-31 14:54:13.74-0530', expected: function(val) { - assert.UTCDate(val, 2010, 9, 31, 20, 24, 13, 74); + assert.UTCDate(val, 2010, 9, 31, 20, 24, 13, 740); } },{ + name: 'timestamptz with other milisecond digits dropped', + dataTypeID: 1184, + actual: '2011-01-23 22:05:00.68-06', + expected: function(val) { + assert.UTCDate(val, 2011, 01, 24, 4, 5, 00, 680); + } + }, { name: 'timestampz with huge miliseconds in UTC', dataTypeID: 1184, actual: '2010-10-30 14:11:12.730838Z', diff --git a/test/unit/writer-tests.js b/test/unit/writer-tests.js index 32972de0d..f2052bf45 100644 --- a/test/unit/writer-tests.js +++ b/test/unit/writer-tests.js @@ -149,5 +149,29 @@ test('clearing', function() { }) +test("resizing to much larger", function() { + var subject = new Writer(2); + var string = "!!!!!!!!"; + var result = subject.addCString(string).flush(); + assert.equalBuffers(result, [33, 33, 33, 33, 33, 33, 33, 33, 0]) +}) +test("header", function() { + test('added as a hex code to a full writer', function() { + var subject = new Writer(2); + var result = subject.addCString("!").flush(0x50) + assert.equalBuffers(result, [0x50, 0, 0, 0, 6, 33, 0]); + }) + test('added as a hex code to a non-full writer', function() { + var subject = new Writer(10).addCString("!"); + var joinedResult = subject.join(0x50); + var result = subject.flush(0x50); + assert.equalBuffers(result, [0x50, 0, 0, 0, 6, 33, 0]); + }) + + test('added as a hex code to a buffer which requires resizing', function() { + var result = new Writer(2).addCString("!!!!!!!!").flush(0x50); + assert.equalBuffers(result, [0x50, 0, 0, 0, 0x0D, 33, 33, 33, 33, 33, 33, 33, 33, 0]); + }) +}) From d5538816ef8274d0a4d3180a60961c250b391f6d Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Thu, 27 Jan 2011 21:11:42 +0100 Subject: [PATCH 02/37] added date parsing --- lib/query.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/query.js b/lib/query.js index 50983b6e9..4e8af8c3f 100644 --- a/lib/query.js +++ b/lib/query.js @@ -318,6 +318,14 @@ var parseBinaryFloat64 = function(value) { return parseFloat(value, 52, 11); } +var parseDate = function(value) { + var sign = parseBits(value, 1); + var rawValue = parseBits(value, 63, 1); + + // discard usecs and shift from 2000 to 1970 + return new Date((((sign == 0) ? 1 : -1) * rawValue / 1000) + 946684800000); +} + // To help we test dateParser Query.dateParser = dateParser; @@ -334,8 +342,8 @@ var dataTypeParsers = { 700: parseBinaryFloat32, 701: parseBinaryFloat64, // 1009: arrayParser, - 1114: parseBinaryInt64, // TOFIX: dateParser, - 1184: dateParser + 1114: parseDate, + //1184: parseBinaryInt64 // 1700: parseFloat, }; From 37e958f779c5f2442893412e8aac00ed349eb338 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 28 Jan 2011 18:06:05 +0100 Subject: [PATCH 03/37] added arrayParser --- lib/query.js | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/lib/query.js b/lib/query.js index 4e8af8c3f..4e93680ad 100644 --- a/lib/query.js +++ b/lib/query.js @@ -326,6 +326,72 @@ var parseDate = function(value) { return new Date((((sign == 0) ? 1 : -1) * rawValue / 1000) + 946684800000); } +var arrayParser = function(value) { + var dim = parseBits(value, 32); + + var flags = parseBits(value, 32, 32); + var elementType = parseBits(value, 32, 64); + + var offset = 96; + var dims = new Array(); + for (var i = 0; i < dim; i++) { + // parse dimension + dims[i] = parseBits(value, 32, offset); + offset += 32; + + // ignore lower bounds + offset += 32; + }; + + + var parseElement = function(elementType) { + // parse content length + var length = parseBits(value, 32, offset); + offset += 32; + + // parse null values + if (length == 0xffffffff) { + return null; + } + + if (elementType == 0x17) { + // int + var result = parseBits(value, length * 8, offset); + offset += length * 8; + return result; + } + else if (elementType == 0x19) { + // string + var result = value.toString('utf8', offset >> 3, (offset += (length << 3)) >> 3); + return result; + } + else { + console.log("ERROR: ElementType not implemented: " + elementType); + } + }; + + var parseArray = function(dimension, elementType) { + var array = new Array(); + + if (dimension.length > 1) { + var count = dimension.shift(); + for (var i = 0; i < count; i++) { + array[i] = parseArray(dimension, elementType); + } + dimension.unshift(count); + } + else { + for (var i = 0; i < dimension[0]; i++) { + array[i] = parseElement(elementType); + } + } + + return array; + } + + return parseArray(dims, elementType); +}; + // To help we test dateParser Query.dateParser = dateParser; @@ -341,7 +407,8 @@ var dataTypeParsers = { 26: parseBinaryInt64, 700: parseBinaryFloat32, 701: parseBinaryFloat64, -// 1009: arrayParser, + 1007: arrayParser, + 1009: arrayParser, 1114: parseDate, //1184: parseBinaryInt64 // 1700: parseFloat, From b98994ae39ad6dda4a2504c3390122fa634088e6 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 28 Jan 2011 18:06:37 +0100 Subject: [PATCH 04/37] removed debug output --- lib/query.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/query.js b/lib/query.js index 4e93680ad..12a7108e4 100644 --- a/lib/query.js +++ b/lib/query.js @@ -37,7 +37,6 @@ p.submit = function(connection) { var names = []; var rows = []; var handleRowDescription = function(msg) { - console.log(JSON.stringify(msg)); for(var i = 0; i < msg.fields.length; i++) { converters[i] = dataTypeParsers[msg.fields[i].dataTypeID] || noParse; names[i] = msg.fields[i].name; @@ -48,7 +47,6 @@ p.submit = function(connection) { for(var i = 0; i < msg.fields.length; i++) { var rawValue = msg.fields[i]; result[names[i]] = rawValue === null ? null : converters[i](rawValue); - console.log(names[i] + ": " + result[names[i]]); } self.emit('row', result); From 01e0fb1b922c51206f621f9fd3def48c7b8beafc Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 28 Jan 2011 18:16:12 +0100 Subject: [PATCH 05/37] added microseconds to date --- lib/query.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/query.js b/lib/query.js index 12a7108e4..592c6d446 100644 --- a/lib/query.js +++ b/lib/query.js @@ -321,7 +321,21 @@ var parseDate = function(value) { var rawValue = parseBits(value, 63, 1); // discard usecs and shift from 2000 to 1970 - return new Date((((sign == 0) ? 1 : -1) * rawValue / 1000) + 946684800000); + var result = new Date((((sign == 0) ? 1 : -1) * rawValue / 1000) + 946684800000); + + // add microseconds to the date + result.usec = rawValue % 1000; + result.getMicroSeconds = function() { + return this.usec; + }; + result.setMicroSeconds = function(value) { + this.usec = value; + }; + result.getUTCMicroSeconds = function() { + return this.usec; + }; + + return result; } var arrayParser = function(value) { From fec176e23b39c78ab96723fb1c6aac2992f8aaaf Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sat, 29 Jan 2011 01:32:41 +0100 Subject: [PATCH 06/37] fixed typo --- lib/query.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/query.js b/lib/query.js index 2e4f6dd9c..6f5886146 100644 --- a/lib/query.js +++ b/lib/query.js @@ -51,6 +51,7 @@ var buildDataRowMetadata = function(msg, converters, names) { case 1700: case 700: converters[i] = parseBinaryFloat32; + break; case 701: converters[i] = parseBinaryFloat64; break; @@ -65,7 +66,7 @@ var buildDataRowMetadata = function(msg, converters, names) { break; case 1007: case 1008: - converters[i] = arrayParser, + converters[i] = arrayParser; break; default: converters[i] = dataTypeParsers[dataTypeId] || noParse; From 0d36ce0799cd732015ceb6044e83476d475e785a Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sat, 29 Jan 2011 13:58:18 +0100 Subject: [PATCH 07/37] added numeric parser, modularized parsers --- lib/binaryParser.js | 230 +++++++++++++++++++++++++++++++++++ lib/query.js | 283 ++++---------------------------------------- lib/textParser.js | 83 +++++++++++++ 3 files changed, 338 insertions(+), 258 deletions(-) create mode 100644 lib/binaryParser.js create mode 100644 lib/textParser.js diff --git a/lib/binaryParser.js b/lib/binaryParser.js new file mode 100644 index 000000000..3231597cb --- /dev/null +++ b/lib/binaryParser.js @@ -0,0 +1,230 @@ +var BinaryParser = function(config) { + config = config || {}; + this.encoding = config.encoding || 'utf8'; +}; + +var p = BinaryParser.prototype; + +var parseBits = function(data, bits, offset, callback) { + offset = offset || 0; + callback = callback || function(lastValue, newValue, bits) { return (lastValue * Math.pow(2, bits)) + newValue; }; + var offsetBytes = offset >> 3; + + // read first (maybe partial) byte + var mask = 0xff; + var firstBits = 8 - (offset % 8); + if (bits < firstBits) { + mask = (0xff << (8 - bits)) & 0xff; + firstBits = bits; + } + + if (offset) { + mask = mask >> (offset % 8); + } + + var result = 0; + if ((offset % 8) + bits > 8) { + result = callback(0, data[offsetBytes] & mask, firstBits); + } + + // read bytes + var bytes = (bits + offset) >> 3; + for (var i = offsetBytes + 1; i < bytes; i++) { + result = callback(result, data[i], 8); + } + + // bits to read, that are not a complete byte + var lastBits = (bits + offset) % 8; + if (lastBits > 0) { + result = callback(result, data[bytes] >> (8 - lastBits), lastBits); + } + + return result; +} + +var parseFloat = function(data, precisionBits, exponentBits) { + var bias = Math.pow(2, exponentBits - 1) - 1; + var sign = parseBits(data, 1); + var exponent = parseBits(data, exponentBits, 1); + + if (exponent == 0) + return 0; + + // parse mantissa + var precisionBitsCounter = 1; + var parsePrecisionBits = function(lastValue, newValue, bits) { + if (lastValue == 0) { + lastValue = 1; + } + + for (var i = 1; i <= bits; i++) { + precisionBitsCounter /= 2; + if ((newValue & (0x1 << (bits - i))) > 0) { + lastValue += precisionBitsCounter; + } + } + + return lastValue; + }; + + var mantissa = parseBits(data, precisionBits, exponentBits + 1, parsePrecisionBits); + + // special cases + if (exponent == (Math.pow(2, exponentBits + 1) - 1)) { + if (mantissa == 0) { + return (sign == 0) ? Infinity : -Infinity; + } + + return NaN; + } + + // normale number + return ((sign == 0) ? 1 : -1) * Math.pow(2, exponent - bias) * mantissa; +}; + +p.parseBool = function(value) { + console.log(JSON.stringify(value)); + return (parseBits(value, 16) == 0); +} + +p.parseInt16 = function(value) { + if (parseBits(value, 1) == 1) { + return -1 * (Math.pow(2, 15) - parseBits(value, 15, 1)); + } + + return parseBits(value, 15, 1); +} + +p.parseInt32 = function(value) { + if (parseBits(value, 1) == 1) { + return -1 * (Math.pow(2, 31) - parseBits(value, 31, 1)); + } + + return parseBits(value, 31, 1); +} + +p.parseInt64 = function(value) { + if (parseBits(value, 1) == 1) { + return -1 * (Math.pow(2, 63) - parseBits(value, 63, 1)); + } + + return parseBits(value, 63, 1); +} + +p.parseFloat32 = function(value) { + return parseFloat(value, 23, 8); +} + +p.parseFloat64 = function(value) { + return parseFloat(value, 52, 11); +} + +p.parseNumeric = function(value) { + var sign = parseBits(value, 16, 32); + if (sign == 0xc000) { + return NaN; + } + + var weight = Math.pow(10000, parseBits(value, 16, 16)); + var result = 0; + + var digits = new Array(); + var ndigits = parseBits(value, 16); + for (var i = 0; i < ndigits; i++) { + result += parseBits(value, 16, 64 + (16 * i)) * weight; + weight /= 10000; + } + + var scale = Math.pow(10, parseBits(value, 16, 48)); + return ((sign == 0) ? 1 : -1) * Math.round(result * scale) / scale; +} + +p.parseDate = function(value) { + var sign = parseBits(value, 1); + var rawValue = parseBits(value, 63, 1); + + // discard usecs and shift from 2000 to 1970 + var result = new Date((((sign == 0) ? 1 : -1) * rawValue / 1000) + 946684800000); + + // add microseconds to the date + result.usec = rawValue % 1000; + result.getMicroSeconds = function() { + return this.usec; + }; + result.setMicroSeconds = function(value) { + this.usec = value; + }; + result.getUTCMicroSeconds = function() { + return this.usec; + }; + + return result; +} + +p.parseIntArray = p.parseStringArray = function(value) { + var dim = parseBits(value, 32); + + var flags = parseBits(value, 32, 32); + var elementType = parseBits(value, 32, 64); + + var offset = 96; + var dims = new Array(); + for (var i = 0; i < dim; i++) { + // parse dimension + dims[i] = parseBits(value, 32, offset); + offset += 32; + + // ignore lower bounds + offset += 32; + }; + + + var parseElement = function(elementType) { + // parse content length + var length = parseBits(value, 32, offset); + offset += 32; + + // parse null values + if (length == 0xffffffff) { + return null; + } + + if (elementType == 0x17) { + // int + var result = parseBits(value, length * 8, offset); + offset += length * 8; + return result; + } + else if (elementType == 0x19) { + // string + var result = value.toString('utf8', offset >> 3, (offset += (length << 3)) >> 3); + return result; + } + else { + console.log("ERROR: ElementType not implemented: " + elementType); + } + }; + + var parse = function(dimension, elementType) { + var array = new Array(); + + if (dimension.length > 1) { + var count = dimension.shift(); + for (var i = 0; i < count; i++) { + array[i] = parseArray(dimension, elementType); + } + dimension.unshift(count); + } + else { + for (var i = 0; i < dimension[0]; i++) { + array[i] = parseElement(elementType); + } + } + + return array; + } + + return parse(dims, elementType); +}; + +module.exports = BinaryParser; diff --git a/lib/query.js b/lib/query.js index 6f5886146..9c996bf07 100644 --- a/lib/query.js +++ b/lib/query.js @@ -1,6 +1,8 @@ var EventEmitter = require('events').EventEmitter; var sys = require('sys');var sys = require('sys'); var Result = require(__dirname + "/result"); +var TextParser = require(__dirname + "/textParser"); +var BinaryParser = require(__dirname + "/binaryParser"); var Query = function(config) { this.text = config.text; @@ -30,43 +32,52 @@ var noParse = function(val) { //creates datarow metatdata from the supplied //data row information var buildDataRowMetadata = function(msg, converters, names) { + var parsers = { + text: new TextParser(), + binary: new BinaryParser() + }; + var len = msg.fields.length; for(var i = 0; i < len; i++) { var field = msg.fields[i]; var dataTypeId = field.dataTypeID; + var format = field.format; names[i] = field.name; switch(dataTypeId) { case 20: - converters[i] = parseBinaryInt64; + converters[i] = parsers[format].parseInt64; break; case 21: - converters[i] = parseBinaryInt16; + converters[i] = parsers[format].parseInt16; break; case 23: - converters[i] = parseBinaryInt32; + converters[i] = parsers[format].parseInt32; break; case 26: - converters[i] = parseBinaryInt64; + converters[i] = parsers[format].parseInt64; break; - case 1700: case 700: - converters[i] = parseBinaryFloat32; + converters[i] = parsers[format].parseFloat32; break; case 701: - converters[i] = parseBinaryFloat64; + converters[i] = parsers[format].parseFloat64; + break; + case 1700: + converters[i] = parsers[format].parseNumeric; break; case 16: - converters[i] = function(val) { - return val == 1; - }; + converters[i] = parsers[format].parseBool; break; case 1114: case 1184: - converters[i] = parseDate; + converters[i] = parsers[format].parseDate; break; - case 1007: case 1008: - converters[i] = arrayParser; + case 1009: + converters[i] = parsers[format].parseStringArray; + break; + case 1007: + converters[i] = parsers[format].parseIntArray; break; default: converters[i] = dataTypeParsers[dataTypeId] || noParse; @@ -96,6 +107,7 @@ p.submit = function(connection) { for(var i = 0; i < msg.fields.length; i++) { var rawValue = msg.fields[i]; row[names[i]] = rawValue === null ? null : converters[i](rawValue); + console.log(names[i] + ': ' + JSON.stringify(row[names[i]])); } self.emit('row', row); @@ -202,251 +214,6 @@ p.prepare = function(connection) { connection.on('error', onCommandComplete); }; -var dateParser = function(isoDate) { - //TODO this could do w/ a refactor - - var dateMatcher = /(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})(\.\d{1,})?/; - - var match = dateMatcher.exec(isoDate); - var year = match[1]; - var month = parseInt(match[2],10)-1; - var day = match[3]; - var hour = parseInt(match[4],10); - var min = parseInt(match[5],10); - var seconds = parseInt(match[6], 10); - - var miliString = match[7]; - var mili = 0; - if(miliString) { - mili = 1000 * parseFloat(miliString); - } - - var tZone = /([Z|+\-])(\d{2})?(\d{2})?/.exec(isoDate.split(' ')[1]); - //minutes to adjust for timezone - var tzAdjust = 0; - - if(tZone) { - var type = tZone[1]; - switch(type) { - case 'Z': break; - case '-': - tzAdjust = -(((parseInt(tZone[2],10)*60)+(parseInt(tZone[3]||0,10)))); - break; - case '+': - tzAdjust = (((parseInt(tZone[2],10)*60)+(parseInt(tZone[3]||0,10)))); - break; - default: - throw new Error("Unidentifed tZone part " + type); - } - } - - var utcOffset = Date.UTC(year, month, day, hour, min, seconds, mili); - - var date = new Date(utcOffset - (tzAdjust * 60* 1000)); - return date; -}; - -function shl(a,b) { - // Copyright (c) 1996 Henri Torgemane. All Rights Reserved. - // fix for crappy << - for (var i=0;i 0) { - lastValue += precisionBitsCounter; - } - } - - return lastValue; - }; - - var mantissa = parseBits(data, precisionBits, exponentBits + 1, parsePrecisionBits); - - // special cases - if (exponent == (Math.pow(2, exponentBits + 1) - 1)) { - if (mantissa == 0) { - return (sign == 0) ? Infinity : -Infinity; - } - - return NaN; - } - - // normale number - return ((sign == 0) ? 1 : -1) * Math.pow(2, exponent - bias) * mantissa; -}; - -var parseBits = function(data, bits, offset, callback) { - offset = offset || 0; - callback = callback || function(lastValue, newValue, bits) { return (lastValue * Math.pow(2, bits)) + newValue; }; - var offsetBytes = offset >> 3; - - // read first (maybe partial) byte - var mask = 0xff; - var firstBits = 8 - (offset % 8); - if (bits < firstBits) { - mask = (0xff << (8 - bits)) & 0xff; - firstBits = bits; - } - - if (offset) { - mask = mask >> (offset % 8); - } - var result = callback(0, data[offsetBytes] & mask, firstBits); - - // read bytes - var bytes = (bits + offset) >> 3; - for (var i = offsetBytes + 1; i < bytes; i++) { - result = callback(result, data[i], 8); - } - - // bits to read, that are not a complete byte - var lastBits = (bits + offset) % 8; - if (lastBits > 0) { - result = callback(result, data[bytes] >> (8 - lastBits), lastBits); - } - - return result; -} - -var parseBinaryInt64 = function(value) { - return parseBits(value, 64); -} - -var parseBinaryInt32 = function(value) { - return parseBits(value, 32); -} - -var parseBinaryInt16 = function(value) { - return parseBits(value, 16); -} - -var parseBinaryFloat32 = function(value) { - return parseFloat(value, 23, 8); -} - -var parseBinaryFloat64 = function(value) { - return parseFloat(value, 52, 11); -} - -var parseDate = function(value) { - var sign = parseBits(value, 1); - var rawValue = parseBits(value, 63, 1); - - // discard usecs and shift from 2000 to 1970 - var result = new Date((((sign == 0) ? 1 : -1) * rawValue / 1000) + 946684800000); - - // add microseconds to the date - result.usec = rawValue % 1000; - result.getMicroSeconds = function() { - return this.usec; - }; - result.setMicroSeconds = function(value) { - this.usec = value; - }; - result.getUTCMicroSeconds = function() { - return this.usec; - }; - - return result; -} - -var arrayParser = function(value) { - var dim = parseBits(value, 32); - - var flags = parseBits(value, 32, 32); - var elementType = parseBits(value, 32, 64); - - var offset = 96; - var dims = new Array(); - for (var i = 0; i < dim; i++) { - // parse dimension - dims[i] = parseBits(value, 32, offset); - offset += 32; - - // ignore lower bounds - offset += 32; - }; - - - var parseElement = function(elementType) { - // parse content length - var length = parseBits(value, 32, offset); - offset += 32; - - // parse null values - if (length == 0xffffffff) { - return null; - } - - if (elementType == 0x17) { - // int - var result = parseBits(value, length * 8, offset); - offset += length * 8; - return result; - } - else if (elementType == 0x19) { - // string - var result = value.toString('utf8', offset >> 3, (offset += (length << 3)) >> 3); - return result; - } - else { - console.log("ERROR: ElementType not implemented: " + elementType); - } - }; - - var parseArray = function(dimension, elementType) { - var array = new Array(); - - if (dimension.length > 1) { - var count = dimension.shift(); - for (var i = 0; i < count; i++) { - array[i] = parseArray(dimension, elementType); - } - dimension.unshift(count); - } - else { - for (var i = 0; i < dimension[0]; i++) { - array[i] = parseElement(elementType); - } - } - - return array; - } - - return parseArray(dims, elementType); -}; - -// To help we test dateParser -Query.dateParser = dateParser; - var dataTypeParsers = { }; diff --git a/lib/textParser.js b/lib/textParser.js new file mode 100644 index 000000000..c42aa0914 --- /dev/null +++ b/lib/textParser.js @@ -0,0 +1,83 @@ +var TextParser = function(config) { + config = config || {}; +}; + +var p = TextParser.prototype; + +p.parseBool = function(value) { + return (value === 't'); +} + +p.parseInt64 = p.parseInt32 = p.parseInt16 = function(value) { + return parseInt(value); +} + +p.parseNumeric = p.parseFloat64 = p.parseFloat32 = function(value) { + return parseFloat(value); +} + +p.parseDate = function(value) { + //TODO this could do w/ a refactor + + var dateMatcher = /(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})(\.\d{1,})?/; + + var match = dateMatcher.exec(value); + var year = match[1]; + var month = parseInt(match[2],10)-1; + var day = match[3]; + var hour = parseInt(match[4],10); + var min = parseInt(match[5],10); + var seconds = parseInt(match[6], 10); + + var miliString = match[7]; + var mili = 0; + if(miliString) { + mili = 1000 * this.parseFloat(miliString); + } + + var tZone = /([Z|+\-])(\d{2})?(\d{2})?/.exec(isoDate.split(' ')[1]); + //minutes to adjust for timezone + var tzAdjust = 0; + + if(tZone) { + var type = tZone[1]; + switch(type) { + case 'Z': break; + case '-': + tzAdjust = -(((parseInt(tZone[2],10)*60)+(parseInt(tZone[3]||0,10)))); + break; + case '+': + tzAdjust = (((parseInt(tZone[2],10)*60)+(parseInt(tZone[3]||0,10)))); + break; + default: + throw new Error("Unidentifed tZone part " + type); + } + } + + var utcOffset = Date.UTC(year, month, day, hour, min, seconds, mili); + + var date = new Date(utcOffset - (tzAdjust * 60* 1000)); + return date; +} + +p.parseIntArray = function(value) { + return JSON.parse(val.replace("{","[").replace("}","]")); +}; + +p.parseStringArray = function(value) { + if (!val) return null; + if (val[0] !== '{' || val[val.length-1] !== '}') + throw "Not postgresql array! (" + arrStr + ")"; + + var x = val.substring(1, val.length - 1); + x = x.match(/(NULL|[^,]+|"((?:.|\n|\r)*?)(?!\\)"|\{((?:.|\n|\r)*?(?!\\)\}) (,|$))/mg); + if (x === null) throw "Not postgre array"; + return x.map(function (el) { + if (el === 'NULL') return null; + if (el[0] === '{') return arguments.callee(el); + if (el[0] === '\"') return el.substring(1, el.length - 1).replace('\\\"', '\"'); + return el; + }); +}; + +module.exports = TextParser; From 96caba59d29b7d0b9a58dd94857c8d961f6f01c3 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 1 Feb 2011 02:20:20 +0100 Subject: [PATCH 08/37] fixed off by one --- lib/binaryParser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/binaryParser.js b/lib/binaryParser.js index 3231597cb..ed2de9335 100644 --- a/lib/binaryParser.js +++ b/lib/binaryParser.js @@ -23,7 +23,7 @@ var parseBits = function(data, bits, offset, callback) { } var result = 0; - if ((offset % 8) + bits > 8) { + if ((offset % 8) + bits >= 8) { result = callback(0, data[offsetBytes] & mask, firstBits); } From df326ec97c11acb86e822e3e51340ba2aeeefbbd Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 1 Feb 2011 02:20:29 +0100 Subject: [PATCH 09/37] fixed bool parsing --- lib/binaryParser.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/binaryParser.js b/lib/binaryParser.js index ed2de9335..2a7bcd603 100644 --- a/lib/binaryParser.js +++ b/lib/binaryParser.js @@ -83,8 +83,7 @@ var parseFloat = function(data, precisionBits, exponentBits) { }; p.parseBool = function(value) { - console.log(JSON.stringify(value)); - return (parseBits(value, 16) == 0); + return (parseBits(value, 8) == 1); } p.parseInt16 = function(value) { From 5b3c501d74d0b3d1610dadf776c746a956537925 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 1 Feb 2011 02:21:45 +0100 Subject: [PATCH 10/37] use config --- lib/binaryParser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/binaryParser.js b/lib/binaryParser.js index 2a7bcd603..ec24c94b3 100644 --- a/lib/binaryParser.js +++ b/lib/binaryParser.js @@ -196,7 +196,7 @@ p.parseIntArray = p.parseStringArray = function(value) { } else if (elementType == 0x19) { // string - var result = value.toString('utf8', offset >> 3, (offset += (length << 3)) >> 3); + var result = value.toString(this.encoding, offset >> 3, (offset += (length << 3)) >> 3); return result; } else { From 88724805dfacc4bea57b91e0fa3412cf22ce1f20 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 1 Feb 2011 02:23:21 +0100 Subject: [PATCH 11/37] removed debug --- lib/query.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/query.js b/lib/query.js index 9c996bf07..ecd188589 100644 --- a/lib/query.js +++ b/lib/query.js @@ -107,7 +107,6 @@ p.submit = function(connection) { for(var i = 0; i < msg.fields.length; i++) { var rawValue = msg.fields[i]; row[names[i]] = rawValue === null ? null : converters[i](rawValue); - console.log(names[i] + ': ' + JSON.stringify(row[names[i]])); } self.emit('row', row); From a0be34da39b9e554085a8d41ccc7eeaccf9881e1 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 7 Feb 2011 12:46:08 +0100 Subject: [PATCH 12/37] corrected typo --- lib/textParser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/textParser.js b/lib/textParser.js index c42aa0914..7c4e6a486 100644 --- a/lib/textParser.js +++ b/lib/textParser.js @@ -35,7 +35,7 @@ p.parseDate = function(value) { mili = 1000 * this.parseFloat(miliString); } - var tZone = /([Z|+\-])(\d{2})?(\d{2})?/.exec(isoDate.split(' ')[1]); + var tZone = /([Z|+\-])(\d{2})?(\d{2})?/.exec(value.split(' ')[1]); //minutes to adjust for timezone var tzAdjust = 0; From a9e40a2d7d07cf0486020ec10a825db31c142692 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 14 Feb 2011 16:41:27 +0100 Subject: [PATCH 13/37] fix typo --- lib/binaryParser.js | 2 +- lib/textParser.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/binaryParser.js b/lib/binaryParser.js index ec24c94b3..4b5a14035 100644 --- a/lib/binaryParser.js +++ b/lib/binaryParser.js @@ -210,7 +210,7 @@ p.parseIntArray = p.parseStringArray = function(value) { if (dimension.length > 1) { var count = dimension.shift(); for (var i = 0; i < count; i++) { - array[i] = parseArray(dimension, elementType); + array[i] = parse(dimension, elementType); } dimension.unshift(count); } diff --git a/lib/textParser.js b/lib/textParser.js index 7c4e6a486..f384050b8 100644 --- a/lib/textParser.js +++ b/lib/textParser.js @@ -65,11 +65,11 @@ p.parseIntArray = function(value) { }; p.parseStringArray = function(value) { - if (!val) return null; - if (val[0] !== '{' || val[val.length-1] !== '}') - throw "Not postgresql array! (" + arrStr + ")"; + if (!value) return null; + if (value[0] !== '{' || value[value.length-1] !== '}') + throw "Not postgresql array! (" + value + ")"; - var x = val.substring(1, val.length - 1); + var x = value.substring(1, value.length - 1); x = x.match(/(NULL|[^,]+|"((?:.|\n|\r)*?)(?!\\)"|\{((?:.|\n|\r)*?(?!\\)\}) (,|$))/mg); if (x === null) throw "Not postgre array"; return x.map(function (el) { From 796b8dfadc0bebb781e6722c179cbaee214149b5 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 14 Feb 2011 16:42:04 +0100 Subject: [PATCH 14/37] added option for using the binary format --- lib/connection.js | 11 +++++++++-- lib/query.js | 6 ++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/connection.js b/lib/connection.js index b4198bffb..2b5c01392 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -122,6 +122,7 @@ p.bind = function(config) { config = config || {}; config.portal = config.portal || ''; config.statement = config.statement || ''; + config.binary = config.binary || false; var values = config.values || []; var len = values.length; var buffer = this.writer @@ -139,8 +140,14 @@ p.bind = function(config) { buffer.addString(val); } } - buffer.addInt16(1); // format codes to use binary - buffer.addInt16(1); + + if (config.binary) { + buffer.addInt16(1); // format codes to use binary + buffer.addInt16(1); + } + else { + buffer.addInt16(0); // format codes to use text + } //0x42 = 'B' this._send(0x42, buffer); diff --git a/lib/query.js b/lib/query.js index ecd188589..5747ad32e 100644 --- a/lib/query.js +++ b/lib/query.js @@ -10,6 +10,7 @@ var Query = function(config) { this.rows = config.rows; this.types = config.types; this.name = config.name; + this.binary = config.binary; //for code clarity purposes we'll declare this here though it's not //set or used until a rowDescription message comes in this.rowDescription = null; @@ -21,7 +22,7 @@ sys.inherits(Query, EventEmitter); var p = Query.prototype; p.requiresPreparation = function() { - return (this.values || 0).length > 0 || this.name || this.rows; + return (this.values || 0).length > 0 || this.name || this.rows || this.binary; }; @@ -182,7 +183,8 @@ p.prepare = function(connection) { connection.bind({ portal: self.name, statement: self.name, - values: self.values + values: self.values, + binary: self.binary }); connection.describe({ From c731cd284426f7872091b05733bd578ae6a584e4 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 2 Mar 2011 14:12:53 +0100 Subject: [PATCH 15/37] added support for bigint array type --- lib/binaryParser.js | 4 ++-- lib/query.js | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/binaryParser.js b/lib/binaryParser.js index 4b5a14035..133e7ab42 100644 --- a/lib/binaryParser.js +++ b/lib/binaryParser.js @@ -188,8 +188,8 @@ p.parseIntArray = p.parseStringArray = function(value) { return null; } - if (elementType == 0x17) { - // int + if ((elementType == 0x17) || (elementType == 0x14)) { + // int/bigint var result = parseBits(value, length * 8, offset); offset += length * 8; return result; diff --git a/lib/query.js b/lib/query.js index 5747ad32e..8b82f279b 100644 --- a/lib/query.js +++ b/lib/query.js @@ -78,6 +78,7 @@ var buildDataRowMetadata = function(msg, converters, names) { converters[i] = parsers[format].parseStringArray; break; case 1007: + case 1016: converters[i] = parsers[format].parseIntArray; break; default: From acdd726a29152c209d378e5bb28a47c7f201acaa Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Thu, 3 Mar 2011 08:19:07 +0100 Subject: [PATCH 16/37] fixed merge error --- lib/query.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/query.js b/lib/query.js index 146d3eab6..696137dba 100644 --- a/lib/query.js +++ b/lib/query.js @@ -50,6 +50,7 @@ p.handleRowDescription = function(msg) { for(var i = 0; i < len; i++) { var field = msg.fields[i]; var dataTypeId = field.dataTypeID; + var format = field.format; this._fieldNames[i] = field.name; switch(dataTypeId) { case 20: From ba9b85fe261f97c6fac80b302bd13b92112e248c Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 6 Jun 2011 19:31:57 +0200 Subject: [PATCH 17/37] binaryParser: added function to parse text fields --- lib/binaryParser.js | 8 ++++++++ lib/query.js | 3 +++ lib/textParser.js | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/lib/binaryParser.js b/lib/binaryParser.js index 133e7ab42..e24292b2f 100644 --- a/lib/binaryParser.js +++ b/lib/binaryParser.js @@ -226,4 +226,12 @@ p.parseIntArray = p.parseStringArray = function(value) { return parse(dims, elementType); }; +p.parseText = function(value) { + var convertToChar = function(chr) { + return String.fromCharCode(chr); + }; + + return value.map(convertToChar).join(''); +}; + module.exports = BinaryParser; diff --git a/lib/query.js b/lib/query.js index 696137dba..5561baf69 100644 --- a/lib/query.js +++ b/lib/query.js @@ -62,6 +62,9 @@ p.handleRowDescription = function(msg) { case 23: this._fieldConverters[i] = parsers[format].parseInt32; break; + case 25: + this._fieldConverters[i] = parsers[format].parseText; + break; case 26: this._fieldConverters[i] = parsers[format].parseInt64; break; diff --git a/lib/textParser.js b/lib/textParser.js index f384050b8..6549b8d86 100644 --- a/lib/textParser.js +++ b/lib/textParser.js @@ -80,4 +80,8 @@ p.parseStringArray = function(value) { }); }; +p.parseText = function(value) { + return value; +}; + module.exports = TextParser; From c513780fae7ade2ba0fb3804f176ffabe4ea9f31 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 6 Jun 2011 19:32:20 +0200 Subject: [PATCH 18/37] textParser: fix error --- lib/textParser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/textParser.js b/lib/textParser.js index 6549b8d86..3620407bc 100644 --- a/lib/textParser.js +++ b/lib/textParser.js @@ -32,7 +32,7 @@ p.parseDate = function(value) { var miliString = match[7]; var mili = 0; if(miliString) { - mili = 1000 * this.parseFloat(miliString); + mili = 1000 * parseFloat(miliString); } var tZone = /([Z|+\-])(\d{2})?(\d{2})?/.exec(value.split(' ')[1]); From 671a5c52ff46b4fd134add0bffdd77d082950b23 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 7 Jun 2011 16:12:04 +0200 Subject: [PATCH 19/37] binaryParser: fixed text protocoll --- lib/binaryParser.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/binaryParser.js b/lib/binaryParser.js index e24292b2f..6dd99706f 100644 --- a/lib/binaryParser.js +++ b/lib/binaryParser.js @@ -227,11 +227,7 @@ p.parseIntArray = p.parseStringArray = function(value) { }; p.parseText = function(value) { - var convertToChar = function(chr) { - return String.fromCharCode(chr); - }; - - return value.map(convertToChar).join(''); + return value.toString('utf8'); }; module.exports = BinaryParser; From e891e7f4fb92bcb73ae4b0c7427ecd1aec7efe1b Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Thu, 16 Jun 2011 18:32:20 +0200 Subject: [PATCH 20/37] parser: added bool parsing --- lib/binaryParser.js | 4 ++++ lib/query.js | 3 +++ lib/textParser.js | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/lib/binaryParser.js b/lib/binaryParser.js index 6dd99706f..2c3a84507 100644 --- a/lib/binaryParser.js +++ b/lib/binaryParser.js @@ -230,4 +230,8 @@ p.parseText = function(value) { return value.toString('utf8'); }; +p.parseBool = function(value) { + return (parseBits(value, 8) > 0); +}; + module.exports = BinaryParser; diff --git a/lib/query.js b/lib/query.js index 5561baf69..05c93872f 100644 --- a/lib/query.js +++ b/lib/query.js @@ -53,6 +53,9 @@ p.handleRowDescription = function(msg) { var format = field.format; this._fieldNames[i] = field.name; switch(dataTypeId) { + case 16: + this._fieldConverters[i] = parsers[format].parseBool; + break; case 20: this._fieldConverters[i] = parsers[format].parseInt64; break; diff --git a/lib/textParser.js b/lib/textParser.js index 3620407bc..839f9488b 100644 --- a/lib/textParser.js +++ b/lib/textParser.js @@ -84,4 +84,8 @@ p.parseText = function(value) { return value; }; +p.parseBool = function(value) { + return value == 't'; +}; + module.exports = TextParser; From abaa4a10202087aa3713026586f1708fdf1fe37f Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Thu, 16 Jun 2011 18:35:06 +0200 Subject: [PATCH 21/37] tests: added some tests for binary parser --- test/unit/client/typed-query-results-tests.js | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/test/unit/client/typed-query-results-tests.js b/test/unit/client/typed-query-results-tests.js index 77b3bedaa..69a0b771d 100644 --- a/test/unit/client/typed-query-results-tests.js +++ b/test/unit/client/typed-query-results-tests.js @@ -10,61 +10,73 @@ test('typed results', function() { //TODO refactor to this style var tests = [{ name: 'string/varchar', + format: 'text', dataTypeID: 1043, actual: 'bang', expected: 'bang' },{ name: 'integer/int4', + format: 'text', dataTypeID: 23, actual: '100', expected: 100 },{ name: 'smallint/int2', + format: 'text', dataTypeID: 21, actual: '101', expected: 101 },{ name: 'bigint/int8', + format: 'text', dataTypeID: 20, actual: '102', expected: 102 },{ name: 'oid', + format: 'text', dataTypeID: 26, actual: '103', expected: 103 },{ name: 'numeric', + format: 'text', dataTypeID: 1700, actual: '12.34', expected: 12.34 },{ name: 'real/float4', dataTypeID: 700, + format: 'text', actual: '123.456', expected: 123.456 },{ name: 'double precision / float8', + format: 'text', dataTypeID: 701, actual: '1.2', expected: 1.2 },{ name: 'boolean true', + format: 'text', dataTypeID: 16, actual: 't', expected: true },{ name: 'boolean false', + format: 'text', dataTypeID: 16, actual: 'f', expected: false },{ name: 'boolean null', + format: 'text', dataTypeID: 16, actual: null, expected: null },{ name: 'timestamptz with minutes in timezone', + format: 'text', dataTypeID: 1184, actual: '2010-10-31 14:54:13.74-0530', expected: function(val) { @@ -72,6 +84,7 @@ test('typed results', function() { } },{ name: 'timestamptz with other milisecond digits dropped', + format: 'text', dataTypeID: 1184, actual: '2011-01-23 22:05:00.68-06', expected: function(val) { @@ -79,6 +92,7 @@ test('typed results', function() { } }, { name: 'timestampz with huge miliseconds in UTC', + format: 'text', dataTypeID: 1184, actual: '2010-10-30 14:11:12.730838Z', expected: function(val) { @@ -86,6 +100,7 @@ test('typed results', function() { } },{ name: 'timestampz with no miliseconds', + format: 'text', dataTypeID: 1184, actual: '2010-10-30 13:10:01+05', expected: function(val) { @@ -93,11 +108,87 @@ test('typed results', function() { } },{ name: 'timestamp', + format: 'text', dataTypeID: 1114, actual: '2010-10-31 00:00:00', expected: function(val) { assert.UTCDate(val, 2010, 9, 31, 0, 0, 0, 0); } + }, + + + { + name: 'binary-string/varchar', + format: 'binary', + dataTypeID: 1043, + actual: 'bang', + expected: 'bang' + },{ + name: 'binary-integer/int4', + format: 'binary', + dataTypeID: 23, + actual: [0, 0, 0, 100], + expected: 100 + },{ + name: 'binary-smallint/int2', + format: 'binary', + dataTypeID: 21, + actual: [0, 101], + expected: 101 + },{ + name: 'binary-bigint/int8', + format: 'binary', + dataTypeID: 20, + actual: [0, 0, 0, 0, 0, 0, 0, 102], + expected: 102 + },{ + name: 'binary-bigint/int8-full', + format: 'binary', + dataTypeID: 20, + actual: [1, 0, 0, 0, 0, 0, 0, 102], + expected: 72057594037928030 + },{ + name: 'binary-oid', + format: 'binary', + dataTypeID: 26, + actual: [0, 0, 0, 0, 0, 0, 0, 103], + expected: 103 + },{ + name: 'binary-numeric', + format: 'binary', + dataTypeID: 1700, + actual: [0,2,0,0,0,0,0,0x64,0,12,0xd,0x48,0,0,0,0], + expected: 12.34 + },{ + name: 'binary-real/float4', + dataTypeID: 700, + format: 'binary', + actual: [0x41, 0x48, 0x00, 0x00], + expected: 12.5 + },{ + name: 'binary-double precision / float8', + format: 'binary', + dataTypeID: 701, + actual: [0x3F,0xF3,0x33,0x33,0x33,0x33,0x33,0x33], + expected: 1.2 + },{ + name: 'binary-boolean true', + format: 'binary', + dataTypeID: 16, + actual: [1], + expected: true + },{ + name: 'binary-boolean false', + format: 'binary', + dataTypeID: 16, + actual: [0], + expected: false + },{ + name: 'binary-string', + format: 'binary', + dataTypeID: 25, + actual: new Buffer([0x73, 0x6c, 0x61, 0x64, 0x64, 0x61]), + expected: 'sladda' }]; From a8acf9a656a5704262675b87a7bad46c3c2c485f Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 19 Jun 2011 14:53:26 +0200 Subject: [PATCH 22/37] tests: added test for binary null --- test/unit/client/typed-query-results-tests.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/unit/client/typed-query-results-tests.js b/test/unit/client/typed-query-results-tests.js index 69a0b771d..d204c176c 100644 --- a/test/unit/client/typed-query-results-tests.js +++ b/test/unit/client/typed-query-results-tests.js @@ -183,6 +183,12 @@ test('typed results', function() { dataTypeID: 16, actual: [0], expected: false + },{ + name: 'binary-boolean null', + format: 'binary', + dataTypeID: 16, + actual: null, + expected: null },{ name: 'binary-string', format: 'binary', From 727de59494fc75884d1a49cd0a2db4dabb2e5b2f Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 19 Jun 2011 16:31:48 +0200 Subject: [PATCH 23/37] tests: added test for binary timestamp --- test/unit/client/typed-query-results-tests.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/unit/client/typed-query-results-tests.js b/test/unit/client/typed-query-results-tests.js index d204c176c..5127d97a4 100644 --- a/test/unit/client/typed-query-results-tests.js +++ b/test/unit/client/typed-query-results-tests.js @@ -189,6 +189,14 @@ test('typed results', function() { dataTypeID: 16, actual: null, expected: null + },{ + name: 'binary-timestamp', + format: 'binary', + dataTypeID: 1184, + actual: [0x00, 0x01, 0x36, 0xee, 0x3e, 0x66, 0x9f, 0xe0], + expected: function(val) { + assert.UTCDate(val, 2010, 9, 31, 20, 24, 13, 740); + } },{ name: 'binary-string', format: 'binary', From 36243af0afe7f7879a76fb89e8c9661525cff9f8 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 10 Jul 2011 22:49:52 +0200 Subject: [PATCH 24/37] tests: fixed bug, because of renamed function --- test/unit/client/query-tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/client/query-tests.js b/test/unit/client/query-tests.js index 19bbc21c3..366ac1b54 100644 --- a/test/unit/client/query-tests.js +++ b/test/unit/client/query-tests.js @@ -1,6 +1,6 @@ var helper = require(__dirname + '/test-helper'); var q = {}; -q.dateParser = require(__dirname + "/../../../lib/types").getStringTypeParser(1114); +q.dateParser = require(__dirname + "/../../../lib/types").getTypeParser(1114, 'text'); test("testing dateParser", function() { assert.equal(q.dateParser("2010-12-11 09:09:04").toUTCString(),new Date("2010-12-11 09:09:04 GMT").toUTCString()); From 59c5df6ef4c387dd3d88fa27bf1fe006cad8e6be Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 18 Nov 2011 21:07:00 +0100 Subject: [PATCH 25/37] remove name duplication --- lib/query.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/query.js b/lib/query.js index d5a093d70..3d5dfc08f 100644 --- a/lib/query.js +++ b/lib/query.js @@ -2,7 +2,7 @@ var EventEmitter = require('events').EventEmitter; var util = require('util'); var Result = require(__dirname + "/result"); -var types = require(__dirname + "/types"); +var Types = require(__dirname + "/types"); var Query = function(config) { this.text = config.text; @@ -44,8 +44,7 @@ p.handleRowDescription = function(msg) { var field = msg.fields[i]; var format = field.format; this._fieldNames[i] = field.name; - this._fieldConverters[i] = types.getTypeParser(field.dataTypeID, format); - + this._fieldConverters[i] = Types.getTypeParser(field.dataTypeID, format); }; }; From aff94b00684f0d51ea9df28fa36282a1bde75573 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 20 Nov 2011 20:34:14 +0100 Subject: [PATCH 26/37] removed merge artifact --- lib/types.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/types.js b/lib/types.js index dad762494..c3fd8a488 100644 --- a/lib/types.js +++ b/lib/types.js @@ -28,12 +28,6 @@ binaryParsers.init(function(oid, converter) { typeParsers.binary[oid] = converter; }); -var parseByteA = function(val) { - return new Buffer(val.replace(/\\([0-7]{3})/g, function (full_match, code) { - return String.fromCharCode(parseInt(code, 8)); - }).replace(/\\\\/g, "\\"), "binary"); -} - module.exports = { getTypeParser: getTypeParser, } From e9838cc5bbf689a8042fa1f92b6c12b0fd97a3d3 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 21 Nov 2011 11:23:26 +0100 Subject: [PATCH 27/37] fix textParsers some textParsers requires the input value to be a string, so convert it before calling the textParsers the same problem exists in test/integration/connection/query-test so that there also need to be a String call --- lib/types.js | 6 ++++-- test/integration/connection/query-tests.js | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/types.js b/lib/types.js index c3fd8a488..c2bbc1108 100644 --- a/lib/types.js +++ b/lib/types.js @@ -8,7 +8,7 @@ var typeParsers = { //the empty parse function var noParse = function(val) { - return val; + return String(val); } //returns a function used to convert a specific type (specified by @@ -21,7 +21,9 @@ var getTypeParser = function(oid, format) { }; textParsers.init(function(oid, converter) { - typeParsers.text[oid] = converter; + typeParsers.text[oid] = function(value) { + return converter(String(value)); + }; }); binaryParsers.init(function(oid, converter) { diff --git a/test/integration/connection/query-tests.js b/test/integration/connection/query-tests.js index 79e0e13ad..f308546d7 100644 --- a/test/integration/connection/query-tests.js +++ b/test/integration/connection/query-tests.js @@ -20,6 +20,6 @@ test('simple query', function() { process.on('exit', function() { assert.equal(rows.length, 2); assert.equal(rows[0].length, 1); - assert.strictEqual(rows[0] [0], '1'); - assert.strictEqual(rows[1] [0], '2'); + assert.strictEqual(String(rows[0] [0]), '1'); + assert.strictEqual(String(rows[1] [0]), '2'); }); From 070155a577a4e55425ce0fb369030f3386419b24 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 21 Nov 2011 11:37:45 +0100 Subject: [PATCH 28/37] fix native bindings native bindings need to get the textParsers with the new syntax --- lib/native/query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/native/query.js b/lib/native/query.js index dbfd2b2d1..6187c20a5 100644 --- a/lib/native/query.js +++ b/lib/native/query.js @@ -60,7 +60,7 @@ var mapRowData = function(row) { var result = {}; for(var i = 0, len = row.length; i < len; i++) { var item = row[i]; - result[item.name] = item.value == null ? null : types.getStringTypeParser(item.type)(item.value); + result[item.name] = item.value == null ? null : types.getTypeParser(item.type, 'text')(item.value); } return result; } From f3c9a532e0c97a2e512341a6519ac81e42a13bc1 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 21 Nov 2011 11:40:17 +0100 Subject: [PATCH 29/37] code beautification --- lib/binaryParsers.js | 51 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/lib/binaryParsers.js b/lib/binaryParsers.js index c1b5b6c2f..7d135386e 100644 --- a/lib/binaryParsers.js +++ b/lib/binaryParsers.js @@ -33,20 +33,20 @@ var parseBits = function(data, bits, offset, callback) { } return result; -} +}; -var parseFloat = function(data, precisionBits, exponentBits) { +var parseFloatFromBits = function(data, precisionBits, exponentBits) { var bias = Math.pow(2, exponentBits - 1) - 1; var sign = parseBits(data, 1); var exponent = parseBits(data, exponentBits, 1); - if (exponent == 0) + if (exponent === 0) return 0; // parse mantissa var precisionBitsCounter = 1; var parsePrecisionBits = function(lastValue, newValue, bits) { - if (lastValue == 0) { + if (lastValue === 0) { lastValue = 1; } @@ -64,20 +64,20 @@ var parseFloat = function(data, precisionBits, exponentBits) { // special cases if (exponent == (Math.pow(2, exponentBits + 1) - 1)) { - if (mantissa == 0) { - return (sign == 0) ? Infinity : -Infinity; + if (mantissa === 0) { + return (sign === 0) ? Infinity : -Infinity; } return NaN; } // normale number - return ((sign == 0) ? 1 : -1) * Math.pow(2, exponent - bias) * mantissa; + return ((sign === 0) ? 1 : -1) * Math.pow(2, exponent - bias) * mantissa; }; var parseBool = function(value) { return (parseBits(value, 8) == 1); -} +}; var parseInt16 = function(value) { if (parseBits(value, 1) == 1) { @@ -85,7 +85,7 @@ var parseInt16 = function(value) { } return parseBits(value, 15, 1); -} +}; var parseInt32 = function(value) { if (parseBits(value, 1) == 1) { @@ -93,7 +93,7 @@ var parseInt32 = function(value) { } return parseBits(value, 31, 1); -} +}; var parseInt64 = function(value) { if (parseBits(value, 1) == 1) { @@ -101,15 +101,15 @@ var parseInt64 = function(value) { } return parseBits(value, 63, 1); -} +}; var parseFloat32 = function(value) { - return parseFloat(value, 23, 8); -} + return parseFloatFromBits(value, 23, 8); +}; var parseFloat64 = function(value) { - return parseFloat(value, 52, 11); -} + return parseFloatFromBits(value, 52, 11); +}; var parseNumeric = function(value) { var sign = parseBits(value, 16, 32); @@ -120,7 +120,7 @@ var parseNumeric = function(value) { var weight = Math.pow(10000, parseBits(value, 16, 16)); var result = 0; - var digits = new Array(); + var digits = []; var ndigits = parseBits(value, 16); for (var i = 0; i < ndigits; i++) { result += parseBits(value, 16, 64 + (16 * i)) * weight; @@ -128,15 +128,15 @@ var parseNumeric = function(value) { } var scale = Math.pow(10, parseBits(value, 16, 48)); - return ((sign == 0) ? 1 : -1) * Math.round(result * scale) / scale; -} + return ((sign === 0) ? 1 : -1) * Math.round(result * scale) / scale; +}; var parseDate = function(value) { var sign = parseBits(value, 1); var rawValue = parseBits(value, 63, 1); // discard usecs and shift from 2000 to 1970 - var result = new Date((((sign == 0) ? 1 : -1) * rawValue / 1000) + 946684800000); + var result = new Date((((sign === 0) ? 1 : -1) * rawValue / 1000) + 946684800000); // add microseconds to the date result.usec = rawValue % 1000; @@ -151,7 +151,7 @@ var parseDate = function(value) { }; return result; -} +}; var parseArray = function(value) { var dim = parseBits(value, 32); @@ -160,7 +160,7 @@ var parseArray = function(value) { var elementType = parseBits(value, 32, 64); var offset = 96; - var dims = new Array(); + var dims = []; for (var i = 0; i < dim; i++) { // parse dimension dims[i] = parseBits(value, 32, offset); @@ -168,8 +168,7 @@ var parseArray = function(value) { // ignore lower bounds offset += 32; - }; - + } var parseElement = function(elementType) { // parse content length @@ -198,7 +197,7 @@ var parseArray = function(value) { }; var parse = function(dimension, elementType) { - var array = new Array(); + var array = []; if (dimension.length > 1) { var count = dimension.shift(); @@ -214,7 +213,7 @@ var parseArray = function(value) { } return array; - } + }; return parse(dims, elementType); }; @@ -246,5 +245,5 @@ var init = function(register) { }; module.exports = { - init: init, + init: init }; From 09ee46da22f4b1a3a2a9c9b44e36524183f002aa Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 22 Nov 2011 04:45:57 +0100 Subject: [PATCH 30/37] fix binaryParsers: oid is 32bit --- lib/binaryParsers.js | 2 +- test/unit/client/typed-query-results-tests.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/binaryParsers.js b/lib/binaryParsers.js index 7d135386e..dcec67dcb 100644 --- a/lib/binaryParsers.js +++ b/lib/binaryParsers.js @@ -230,7 +230,7 @@ var init = function(register) { register(20, parseInt64); register(21, parseInt16); register(23, parseInt32); - register(26, parseInt64); + register(26, parseInt32); register(1700, parseNumeric); register(700, parseFloat32); register(701, parseFloat64); diff --git a/test/unit/client/typed-query-results-tests.js b/test/unit/client/typed-query-results-tests.js index 938ce9cab..baa86bd4c 100644 --- a/test/unit/client/typed-query-results-tests.js +++ b/test/unit/client/typed-query-results-tests.js @@ -191,7 +191,7 @@ test('typed results', function() { name: 'binary-oid', format: 'binary', dataTypeID: 26, - actual: [0, 0, 0, 0, 0, 0, 0, 103], + actual: [0, 0, 0, 103], expected: 103 },{ name: 'binary-numeric', From 5d8c8bbcdcf741644b968ba900a1418a76ba368d Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 21 Nov 2011 11:42:23 +0100 Subject: [PATCH 31/37] fix recognition of query format for empty queries in dictionary format queries like {text: ""} did not get recognized correctly before and get converted to {text: {text: ""}} --- lib/client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/client.js b/lib/client.js index 891e18120..975eac66b 100644 --- a/lib/client.js +++ b/lib/client.js @@ -172,7 +172,7 @@ p._pulseQueryQueue = function() { p.query = function(config, values, callback) { //can take in strings or config objects - config = (config.text || config.name) ? config : { text: config }; + config = (typeof(config) == 'string') ? { text: config } : config; if(values) { if(typeof values === 'function') { From f8962fd03690defb649d7b4e9000f7193384c424 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 21 Nov 2011 11:45:55 +0100 Subject: [PATCH 32/37] connection can be binary by default if connection is created with config.binary = true, all queries get executed with binary result unless explicit disabled with binary = false --- lib/client.js | 4 ++++ lib/defaults.js | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/client.js b/lib/client.js index 975eac66b..5d1af39aa 100644 --- a/lib/client.js +++ b/lib/client.js @@ -20,6 +20,7 @@ var Client = function(config) { this.connection = config.connection || new Connection({stream: config.stream}); this.queryQueue = []; this.password = config.password || defaults.password; + this.binary = config.binary || defaults.binary; this.encoding = 'utf8'; this.processID = null; this.secretKey = null; @@ -173,6 +174,9 @@ p._pulseQueryQueue = function() { p.query = function(config, values, callback) { //can take in strings or config objects config = (typeof(config) == 'string') ? { text: config } : config; + if (this.binary && !('binary' in config)) { + config.binary = true; + } if(values) { if(typeof values === 'function') { diff --git a/lib/defaults.js b/lib/defaults.js index e60c74379..8df164cab 100644 --- a/lib/defaults.js +++ b/lib/defaults.js @@ -14,5 +14,7 @@ module.exports = { //0 will disable connection pooling poolSize: 10, //duration of node-pool timeout - poolIdleTimeout: 30000 + poolIdleTimeout: 30000, + // binary result mode + binary: false } From 239d8bd0c21e9ef5045956fc58ef4f9b3a3381c1 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 22 Nov 2011 04:36:48 +0100 Subject: [PATCH 33/37] fixed binaryParsers for small negativ values WARNING: bigint support is not correctly working for really big values. If the value of a integer gets big the number gets fuzzy in javascript. This is not a limitation of this library. If you want to handle bigint with the exact value, get it as string and do not calculate things with it! --- lib/binaryParsers.js | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/lib/binaryParsers.js b/lib/binaryParsers.js index dcec67dcb..9333a972e 100644 --- a/lib/binaryParsers.js +++ b/lib/binaryParsers.js @@ -1,8 +1,17 @@ -var parseBits = function(data, bits, offset, callback) { +var parseBits = function(data, bits, offset, invert, callback) { offset = offset || 0; + invert = invert || false; callback = callback || function(lastValue, newValue, bits) { return (lastValue * Math.pow(2, bits)) + newValue; }; var offsetBytes = offset >> 3; + var inv = function(value) { + if (invert) { + return ~value & 0xff; + } + + return value; + }; + // read first (maybe partial) byte var mask = 0xff; var firstBits = 8 - (offset % 8); @@ -17,19 +26,19 @@ var parseBits = function(data, bits, offset, callback) { var result = 0; if ((offset % 8) + bits >= 8) { - result = callback(0, data[offsetBytes] & mask, firstBits); + result = callback(0, inv(data[offsetBytes]) & mask, firstBits); } // read bytes var bytes = (bits + offset) >> 3; for (var i = offsetBytes + 1; i < bytes; i++) { - result = callback(result, data[i], 8); + result = callback(result, inv(data[i]), 8); } // bits to read, that are not a complete byte var lastBits = (bits + offset) % 8; if (lastBits > 0) { - result = callback(result, data[bytes] >> (8 - lastBits), lastBits); + result = callback(result, inv(data[bytes]) >> (8 - lastBits), lastBits); } return result; @@ -60,7 +69,7 @@ var parseFloatFromBits = function(data, precisionBits, exponentBits) { return lastValue; }; - var mantissa = parseBits(data, precisionBits, exponentBits + 1, parsePrecisionBits); + var mantissa = parseBits(data, precisionBits, exponentBits + 1, false, parsePrecisionBits); // special cases if (exponent == (Math.pow(2, exponentBits + 1) - 1)) { @@ -81,7 +90,7 @@ var parseBool = function(value) { var parseInt16 = function(value) { if (parseBits(value, 1) == 1) { - return -1 * (Math.pow(2, 15) - parseBits(value, 15, 1)); + return -1 * (parseBits(value, 15, 1, true) + 1); } return parseBits(value, 15, 1); @@ -89,7 +98,7 @@ var parseInt16 = function(value) { var parseInt32 = function(value) { if (parseBits(value, 1) == 1) { - return -1 * (Math.pow(2, 31) - parseBits(value, 31, 1)); + return -1 * (parseBits(value, 31, 1, true) + 1); } return parseBits(value, 31, 1); @@ -97,7 +106,7 @@ var parseInt32 = function(value) { var parseInt64 = function(value) { if (parseBits(value, 1) == 1) { - return -1 * (Math.pow(2, 63) - parseBits(value, 63, 1)); + return -1 * (parseBits(value, 63, 1, true) + 1); } return parseBits(value, 63, 1); From f698ed44595ff4f36ab629c979fa9c223f4b2890 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 22 Nov 2011 04:42:43 +0100 Subject: [PATCH 34/37] use config dict in all test instead of the connection string use the config dict in all tests to be able to specify things like binary mode --- script/test-connection.js | 6 ++-- test/integration/client/api-tests.js | 22 ++++++------ test/integration/client/array-tests.js | 3 +- test/integration/client/cancel-query-tests.js | 6 ++-- test/integration/client/drain-tests.js | 4 +-- .../client/result-metadata-tests.js | 3 +- test/integration/client/test-helper.js | 25 ++++---------- test/integration/client/transaction-tests.js | 7 ++-- .../integration/client/type-coercion-tests.js | 8 ++--- .../connection-pool/ending-pool-tests.js | 8 ++--- .../connection-pool/error-tests.js | 12 +++---- .../connection-pool/idle-timeout-tests.js | 2 +- .../connection-pool/test-helper.js | 17 +++------- .../connection-pool/unique-name-tests.js | 34 +++++++------------ test/native/callback-api-tests.js | 3 +- test/native/connection-tests.js | 2 +- test/native/error-tests.js | 7 ++-- test/native/evented-api-tests.js | 5 ++- test/native/stress-tests.js | 6 ++-- test/test-helper.js | 4 +-- 20 files changed, 68 insertions(+), 116 deletions(-) diff --git a/script/test-connection.js b/script/test-connection.js index 3099a4dc3..811286104 100644 --- a/script/test-connection.js +++ b/script/test-connection.js @@ -1,9 +1,9 @@ var helper = require(__dirname + '/../test/test-helper'); -var connectionString = helper.connectionString(); + console.log(); -console.log("testing ability to connect to '%s'", connectionString); +console.log("testing ability to connect to '%j'", helper.config); var pg = require(__dirname + '/../lib'); -pg.connect(connectionString, function(err, client) { +pg.connect(helper.config, function(err, client) { if(err !== null) { console.error("Recieved connection error when attempting to contact PostgreSQL:"); console.error(err); diff --git a/test/integration/client/api-tests.js b/test/integration/client/api-tests.js index 023cc297b..0fd941089 100644 --- a/test/integration/client/api-tests.js +++ b/test/integration/client/api-tests.js @@ -5,20 +5,18 @@ if(helper.args.native) { pg = require(__dirname + '/../../../lib').native; } -var connectionString = helper.connectionString(__filename); - var log = function() { //console.log.apply(console, arguments); } var sink = new helper.Sink(5, 10000, function() { - log("ending connection pool: %s", connectionString); - pg.end(connectionString); + log("ending connection pool: %j", helper.config); + pg.end(helper.config); }); test('api', function() { - log("connecting to %s", connectionString) - pg.connect(connectionString, assert.calls(function(err, client) { + log("connecting to %j", helper.config) + pg.connect(helper.config, assert.calls(function(err, client) { assert.equal(err, null, "Failed to connect: " + helper.sys.inspect(err)); client.query('CREATE TEMP TABLE band(name varchar(100))'); @@ -60,7 +58,7 @@ test('api', function() { }) test('executing nested queries', function() { - pg.connect(connectionString, assert.calls(function(err, client) { + pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); log("connected for nested queriese") client.query('select now as now from NOW()', assert.calls(function(err, result) { @@ -87,7 +85,7 @@ test('raises error if cannot connect', function() { }) test("query errors are handled and do not bubble if callback is provded", function() { - pg.connect(connectionString, assert.calls(function(err, client) { + pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err) log("checking for query error") client.query("SELECT OISDJF FROM LEIWLISEJLSE", assert.calls(function(err, result) { @@ -99,7 +97,7 @@ test("query errors are handled and do not bubble if callback is provded", functi }) test('callback is fired once and only once', function() { - pg.connect(connectionString, assert.calls(function(err, client) { + pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); client.query("CREATE TEMP TABLE boom(name varchar(10))"); var callCount = 0; @@ -115,7 +113,7 @@ test('callback is fired once and only once', function() { }) test('can provide callback and config object', function() { - pg.connect(connectionString, assert.calls(function(err, client) { + pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); client.query({ name: 'boom', @@ -128,7 +126,7 @@ test('can provide callback and config object', function() { }) test('can provide callback and config and parameters', function() { - pg.connect(connectionString, assert.calls(function(err, client) { + pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); var config = { text: 'select $1::text as val' @@ -142,7 +140,7 @@ test('can provide callback and config and parameters', function() { }) test('null and undefined are both inserted as NULL', function() { - pg.connect(connectionString, assert.calls(function(err, client) { + pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); client.query("CREATE TEMP TABLE my_nulls(a varchar(1), b varchar(1), c integer, d integer, e date, f date)"); client.query("INSERT INTO my_nulls(a,b,c,d,e,f) VALUES ($1,$2,$3,$4,$5,$6)", [ null, undefined, null, undefined, null, undefined ]); diff --git a/test/integration/client/array-tests.js b/test/integration/client/array-tests.js index 669100c83..548b37680 100644 --- a/test/integration/client/array-tests.js +++ b/test/integration/client/array-tests.js @@ -1,9 +1,8 @@ var helper = require(__dirname + "/test-helper"); var pg = helper.pg; -var conString = helper.connectionString(); test('parsing array results', function() { - pg.connect(conString, assert.calls(function(err, client) { + pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); client.query("CREATE TEMP TABLE why(names text[], numbors integer[])"); client.query('INSERT INTO why(names, numbors) VALUES(\'{"aaron", "brian","a b c" }\', \'{1, 2, 3}\')').on('error', console.log); diff --git a/test/integration/client/cancel-query-tests.js b/test/integration/client/cancel-query-tests.js index 36d5710b6..842b471ae 100644 --- a/test/integration/client/cancel-query-tests.js +++ b/test/integration/client/cancel-query-tests.js @@ -28,9 +28,9 @@ test("cancellation of a query", function() { rows4++; }); - helper.pg.cancel(helper.connectionString, client, query1); - helper.pg.cancel(helper.connectionString, client, query2); - helper.pg.cancel(helper.connectionString, client, query4); + helper.pg.cancel(helper.config, client, query1); + helper.pg.cancel(helper.config, client, query2); + helper.pg.cancel(helper.config, client, query4); setTimeout(function() { assert.equal(rows1, 0); diff --git a/test/integration/client/drain-tests.js b/test/integration/client/drain-tests.js index 0aff28eb4..b6a2434d4 100644 --- a/test/integration/client/drain-tests.js +++ b/test/integration/client/drain-tests.js @@ -6,7 +6,7 @@ if(helper.args.native) { } var testDrainOfClientWithPendingQueries = function() { - pg.connect(helper.connectionString(), assert.success(function(client) { + pg.connect(helper.config, assert.success(function(client) { test('when there are pending queries and client is resumed', function() { var drainCount = 0; client.on('drain', function() { @@ -28,7 +28,7 @@ var testDrainOfClientWithPendingQueries = function() { })); }; -pg.connect(helper.connectionString(), assert.success(function(client) { +pg.connect(helper.config, assert.success(function(client) { var drainCount = 0; client.on('drain', function() { drainCount++; diff --git a/test/integration/client/result-metadata-tests.js b/test/integration/client/result-metadata-tests.js index b69028f60..1c4f94df1 100644 --- a/test/integration/client/result-metadata-tests.js +++ b/test/integration/client/result-metadata-tests.js @@ -1,10 +1,9 @@ var helper = require(__dirname + "/test-helper"); var pg = helper.pg; -var conString = helper.connectionString(); test('should return insert metadata', function() { return false; - pg.connect(conString, assert.calls(function(err, client) { + pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); client.query("CREATE TEMP TABLE zugzug(name varchar(10))", assert.calls(function(err, result) { assert.isNull(err); diff --git a/test/integration/client/test-helper.js b/test/integration/client/test-helper.js index b01235c97..d8ae3d854 100644 --- a/test/integration/client/test-helper.js +++ b/test/integration/client/test-helper.js @@ -1,21 +1,10 @@ var helper = require(__dirname+'/../test-helper'); -module.exports = { - //creates a client from cli parameters - client: function() { - var client = new Client({ - database: helper.args.database, - user: helper.args.user, - password: helper.args.password, - host: helper.args.host, - port: helper.args.port - }); - - client.connect(); - return client; - }, - connectionString: helper.connectionString, - Sink: helper.Sink, - pg: helper.pg, - args: helper.args +//creates a client from cli parameters +helper.client = function() { + var client = new Client(helper.config); + client.connect(); + return client; }; + +module.exports = helper; diff --git a/test/integration/client/transaction-tests.js b/test/integration/client/transaction-tests.js index 8e3f8ac98..4fbfd18b9 100644 --- a/test/integration/client/transaction-tests.js +++ b/test/integration/client/transaction-tests.js @@ -5,9 +5,7 @@ var sink = new helper.Sink(2, function() { }); test('a single connection transaction', function() { - var connectionString = helper.connectionString(); - - helper.pg.connect(connectionString, assert.calls(function(err, client) { + helper.pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); client.query('begin'); @@ -48,8 +46,7 @@ test('a single connection transaction', function() { }) test('gh#36', function() { - var connectionString = helper.connectionString(); - helper.pg.connect(connectionString, function(err, client) { + helper.pg.connect(helper.config, function(err, client) { if(err) throw err; client.query("BEGIN"); client.query({ diff --git a/test/integration/client/type-coercion-tests.js b/test/integration/client/type-coercion-tests.js index 124f82ef1..c8f39991a 100644 --- a/test/integration/client/type-coercion-tests.js +++ b/test/integration/client/type-coercion-tests.js @@ -1,9 +1,9 @@ var helper = require(__dirname + '/test-helper'); var sink; -var connectionString = helper.connectionString(); + var testForTypeCoercion = function(type){ - helper.pg.connect(connectionString, function(err, client) { - assert.isNull(err) + helper.pg.connect(helper.config, function(err, client) { + assert.isNull(err); client.query("create temp table test_type(col " + type.name + ")", assert.calls(function(err, result) { assert.isNull(err); test("Coerces " + type.name, function() { @@ -126,7 +126,7 @@ test("timestampz round trip", function() { client.on('drain', client.end.bind(client)); }); -helper.pg.connect(helper.connectionString(), assert.calls(function(err, client) { +helper.pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); client.query('select null as res;', assert.calls(function(err, res) { assert.isNull(err); diff --git a/test/integration/connection-pool/ending-pool-tests.js b/test/integration/connection-pool/ending-pool-tests.js index c26904211..e46c0fc1b 100644 --- a/test/integration/connection-pool/ending-pool-tests.js +++ b/test/integration/connection-pool/ending-pool-tests.js @@ -1,8 +1,4 @@ var helper = require(__dirname + '/test-helper') -var conString1 = helper.connectionString(); -var conString2 = helper.connectionString(); -var conString3 = helper.connectionString(); -var conString4 = helper.connectionString(); var called = false; test('disconnects', function() { @@ -11,8 +7,8 @@ test('disconnects', function() { //this should exit the process, killing each connection pool helper.pg.end(); }); - [conString1, conString2, conString3, conString4].forEach(function() { - helper.pg.connect(conString1, function(err, client) { + [helper.config, helper.config, helper.config, helper.config].forEach(function(config) { + helper.pg.connect(config, function(err, client) { assert.isNull(err); client.query("SELECT * FROM NOW()", function(err, result) { process.nextTick(function() { diff --git a/test/integration/connection-pool/error-tests.js b/test/integration/connection-pool/error-tests.js index c4653ef7e..11badf04a 100644 --- a/test/integration/connection-pool/error-tests.js +++ b/test/integration/connection-pool/error-tests.js @@ -2,24 +2,22 @@ var helper = require(__dirname + "/../test-helper"); var pg = require(__dirname + "/../../../lib"); helper.pg = pg; -var conString = helper.connectionString(); - //first make pool hold 2 clients -pg.defaults.poolSize = 2; +helper.pg.defaults.poolSize = 2; var killIdleQuery = 'SELECT procpid, (SELECT pg_terminate_backend(procpid)) AS killed FROM pg_stat_activity WHERE current_query LIKE \'\''; //get first client -pg.connect(conString, assert.success(function(client) { +helper.pg.connect(helper.config, assert.success(function(client) { client.id = 1; - pg.connect(conString, assert.success(function(client2) { + helper.pg.connect(helper.config, assert.success(function(client2) { client2.id = 2; //subscribe to the pg error event - assert.emits(pg, 'error', function(error, brokenClient) { + assert.emits(helper.pg, 'error', function(error, brokenClient) { assert.ok(error); assert.ok(brokenClient); assert.equal(client.id, brokenClient.id); - pg.end(); + helper.pg.end(); }); //kill the connection from client client2.query(killIdleQuery, assert.success(function(res) { diff --git a/test/integration/connection-pool/idle-timeout-tests.js b/test/integration/connection-pool/idle-timeout-tests.js index 342442e28..c6cbbd9f6 100644 --- a/test/integration/connection-pool/idle-timeout-tests.js +++ b/test/integration/connection-pool/idle-timeout-tests.js @@ -3,7 +3,7 @@ var helper = require(__dirname + '/test-helper'); helper.pg.defaults.poolIdleTimeout = 200; test('idle timeout', function() { - helper.pg.connect(helper.connectionString(), assert.calls(function(err, client) { + helper.pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); client.query('SELECT NOW()'); //just let this one time out diff --git a/test/integration/connection-pool/test-helper.js b/test/integration/connection-pool/test-helper.js index 8345ea1c0..cc86677d7 100644 --- a/test/integration/connection-pool/test-helper.js +++ b/test/integration/connection-pool/test-helper.js @@ -1,18 +1,15 @@ var helper = require(__dirname + "/../test-helper"); -var pg = require(__dirname + "/../../../lib"); -helper.pg = pg; -var testPoolSize = function(max) { - var conString = helper.connectionString(); +helper.testPoolSize = function(max) { var sink = new helper.Sink(max, function() { - helper.pg.end(conString); + helper.pg.end(); }); test("can pool " + max + " times", function() { for(var i = 0; i < max; i++) { helper.pg.poolSize = 10; test("connection #" + i + " executes", function() { - helper.pg.connect(conString, function(err, client) { + helper.pg.connect(helper.config, function(err, client) { assert.isNull(err); client.query("select * from person", function(err, result) { assert.lengthIs(result.rows, 26) @@ -30,11 +27,5 @@ var testPoolSize = function(max) { }) } -module.exports = { - args: helper.args, - pg: helper.pg, - connectionString: helper.connectionString, - Sink: helper.Sink, - testPoolSize: testPoolSize -} +module.exports = helper; diff --git a/test/integration/connection-pool/unique-name-tests.js b/test/integration/connection-pool/unique-name-tests.js index 84dda439d..6893fbd7a 100644 --- a/test/integration/connection-pool/unique-name-tests.js +++ b/test/integration/connection-pool/unique-name-tests.js @@ -7,35 +7,25 @@ helper.pg.defaults.database = helper.args.database; helper.pg.defaults.port = helper.args.port; helper.pg.defaults.host = helper.args.host; helper.pg.defaults.poolIdleTimeout = 100; -var args = { - user: helper.args.user, - password: helper.args.password, - database: helper.args.database, - port: helper.args.port, - host: helper.args.host -} -var moreArgs = { - database: helper.args.database, - password: helper.args.password, - port: helper.args.port, - user: helper.args.user, - host: helper.args.host, - zomg: true +var moreArgs = {}; +for (c in helper.config) { + moreArgs[c] = helper.config[c]; } +moreArgs.zomg = true; -var badArgs = { - user: helper.args.user + 'laksdjfl', - host: helper.args.host, - password: helper.args.password + 'asldkfjlas', - database: helper.args.database, - port: helper.args.port, - zomg: true +var badArgs = {}; +for (c in helper.config) { + badArgs[c] = helper.config[c]; } +badArgs.user = badArgs.user + 'laksdjfl'; +badArgs.password = badArgs.password + 'asldkfjlas'; +badArgs.zomg = true; + test('connecting with complete config', function() { - helper.pg.connect(args, assert.calls(function(err, client) { + helper.pg.connect(helper.config, assert.calls(function(err, client) { assert.isNull(err); client.iGotAccessed = true; client.query("SELECT NOW()") diff --git a/test/native/callback-api-tests.js b/test/native/callback-api-tests.js index 56b232697..450066822 100644 --- a/test/native/callback-api-tests.js +++ b/test/native/callback-api-tests.js @@ -1,9 +1,8 @@ var helper = require(__dirname + "/../test-helper"); var Client = require(__dirname + "/../../lib/native"); -var conString = helper.connectionString(); test('fires callback with results', function() { - var client = new Client(conString); + var client = new Client(helper.config); client.connect(); client.query('SELECT 1 as num', assert.calls(function(err, result) { assert.isNull(err); diff --git a/test/native/connection-tests.js b/test/native/connection-tests.js index e1641a4ff..1cb0ed88e 100644 --- a/test/native/connection-tests.js +++ b/test/native/connection-tests.js @@ -12,7 +12,7 @@ test('connecting with wrong parameters', function() { }); test('connects', function() { - var con = new Client(helper.connectionString()); + var con = new Client(helper.config); con.connect(); assert.emits(con, 'connect', function() { test('disconnects', function() { diff --git a/test/native/error-tests.js b/test/native/error-tests.js index 9d196e6f9..3184df57b 100644 --- a/test/native/error-tests.js +++ b/test/native/error-tests.js @@ -1,9 +1,8 @@ var helper = require(__dirname + "/../test-helper"); var Client = require(__dirname + "/../../lib/native"); -var conString = helper.connectionString(); test('query with non-text as first parameter throws error', function() { - var client = new Client(conString); + var client = new Client(helper.config); client.connect(); assert.emits(client, 'connect', function() { assert.throws(function() { @@ -14,7 +13,7 @@ test('query with non-text as first parameter throws error', function() { }) test('parameterized query with non-text as first parameter throws error', function() { - var client = new Client(conString); + var client = new Client(helper.config); client.connect(); assert.emits(client, 'connect', function() { assert.throws(function() { @@ -28,7 +27,7 @@ test('parameterized query with non-text as first parameter throws error', functi }) var connect = function(callback) { - var client = new Client(conString); + var client = new Client(helper.config); client.connect(); assert.emits(client, 'connect', function() { callback(client); diff --git a/test/native/evented-api-tests.js b/test/native/evented-api-tests.js index d70c824fe..db93f5bff 100644 --- a/test/native/evented-api-tests.js +++ b/test/native/evented-api-tests.js @@ -1,9 +1,8 @@ var helper = require(__dirname + "/../test-helper"); var Client = require(__dirname + "/../../lib/native"); -var conString = helper.connectionString(); var setupClient = function() { - var client = new Client(conString); + var client = new Client(helper.config); client.connect(); client.query("CREATE TEMP TABLE boom(name varchar(10), age integer)"); client.query("INSERT INTO boom(name, age) VALUES('Aaron', 26)"); @@ -12,7 +11,7 @@ var setupClient = function() { } test('connects', function() { - var client = new Client(conString); + var client = new Client(helper.config); client.connect(); test('good query', function() { var query = client.query("SELECT 1 as num, 'HELLO' as str"); diff --git a/test/native/stress-tests.js b/test/native/stress-tests.js index 539e667d3..cac03d037 100644 --- a/test/native/stress-tests.js +++ b/test/native/stress-tests.js @@ -2,7 +2,7 @@ var helper = require(__dirname + "/../test-helper"); var Client = require(__dirname + "/../../lib/native"); test('many rows', function() { - var client = new Client(helper.connectionString()); + var client = new Client(helper.config); client.connect(); var q = client.query("SELECT * FROM person"); var rows = []; @@ -16,7 +16,7 @@ test('many rows', function() { }); test('many queries', function() { - var client = new Client(helper.connectionString()); + var client = new Client(helper.config); client.connect(); var count = 0; var expected = 100; @@ -35,7 +35,7 @@ test('many queries', function() { test('many clients', function() { var clients = []; for(var i = 0; i < 10; i++) { - clients.push(new Client(helper.connectionString())); + clients.push(new Client(helper.config)); } clients.forEach(function(client) { client.connect(); diff --git a/test/test-helper.js b/test/test-helper.js index dbc28b5b8..55a0a5c50 100644 --- a/test/test-helper.js +++ b/test/test-helper.js @@ -225,9 +225,7 @@ module.exports = { args: args, Sink: Sink, pg: require(__dirname + '/../lib/'), - connectionString: function() { - return "pg"+(count++)+"://"+args.user+":"+args.password+"@"+args.host+":"+args.port+"/"+args.database; - }, + config: args, sys: sys, Client: Client }; From 2b7c57710ce13e39d4db374f405e14e8129a7d7d Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 22 Nov 2011 04:51:07 +0100 Subject: [PATCH 35/37] add binary cli argument for tests --- test/cli.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/cli.js b/test/cli.js index b781f5b6d..04f583215 100644 --- a/test/cli.js +++ b/test/cli.js @@ -40,6 +40,8 @@ for(var i = 0; i < args.length; i++) { config.test = args[++i]; case '--native': config.native = (args[++i] == "true"); + case '--binary': + config.binary = (args[++i] == "true"); default: break; } From 6b032c466b40da203d42db6a357eacced4628df2 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 22 Nov 2011 04:59:54 +0100 Subject: [PATCH 36/37] added test-binary target integration tests could now be started in binary mode some tests are executed in text mode anyway, they are currently not compatible with binary mode or prepared statements at all (f.e. multiple statements in one query) --- Makefile | 9 ++++++++- test/integration/client/empty-query-tests.js | 4 ++-- test/integration/client/error-handling-tests.js | 2 +- test/integration/client/simple-query-tests.js | 8 ++++---- test/integration/client/type-coercion-tests.js | 7 +++++++ test/integration/connection-pool/unique-name-tests.js | 1 + 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index b4885d14c..0a3bd679a 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ node-command := xargs -n 1 -I file node file $(params) .PHONY : test test-connection test-integration bench test-native build/default/binding.node test: test-unit -test-all: test-unit test-integration test-native +test-all: test-unit test-integration test-native test-binary bench: @find benchmark -name "*-bench.js" | $(node-command) @@ -28,6 +28,9 @@ test-unit: test-connection: @node script/test-connection.js $(params) +test-connection-binary: + @node script/test-connection.js $(params) --binary true + test-native: build/default/binding.node @echo "***Testing native bindings***" @find test/native -name "*-tests.js" | $(node-command) @@ -36,3 +39,7 @@ test-native: build/default/binding.node test-integration: test-connection @echo "***Testing Pure Javascript***" @find test/integration -name "*-tests.js" | $(node-command) + +test-binary: test-connection-binary + @echo "***Testing Pure Javascript (binary)***" + @find test/integration -name "*-tests.js" | $(node-command) --binary true diff --git a/test/integration/client/empty-query-tests.js b/test/integration/client/empty-query-tests.js index 475acf793..3eb207c4a 100644 --- a/test/integration/client/empty-query-tests.js +++ b/test/integration/client/empty-query-tests.js @@ -5,11 +5,11 @@ test("empty query message handling", function() { assert.emits(client, 'drain', function() { client.end(); }); - client.query(""); + client.query({text: "", binary: false}); }); test('callback supported', assert.calls(function() { - client.query("", function(err, result) { + client.query({text: "", binary: false}, function(err, result) { assert.isNull(err); assert.empty(result.rows); }) diff --git a/test/integration/client/error-handling-tests.js b/test/integration/client/error-handling-tests.js index 4f1cb0cfe..0a855238f 100644 --- a/test/integration/client/error-handling-tests.js +++ b/test/integration/client/error-handling-tests.js @@ -30,7 +30,7 @@ test('error handling', function(){ var client = createErorrClient(); - var q = client.query("CREATE TEMP TABLE boom(age integer); INSERT INTO boom (age) VALUES (28);"); + var q = client.query({text: "CREATE TEMP TABLE boom(age integer); INSERT INTO boom (age) VALUES (28);", binary: false}); test("when query is parsing", function() { diff --git a/test/integration/client/simple-query-tests.js b/test/integration/client/simple-query-tests.js index da573d43e..2edabd73f 100644 --- a/test/integration/client/simple-query-tests.js +++ b/test/integration/client/simple-query-tests.js @@ -37,7 +37,7 @@ test("simple query interface", function() { test("multiple simple queries", function() { var client = helper.client(); - client.query("create temp table bang(id serial, name varchar(5));insert into bang(name) VALUES('boom');") + client.query({ text: "create temp table bang(id serial, name varchar(5));insert into bang(name) VALUES('boom');", binary: false }) client.query("insert into bang(name) VALUES ('yes');"); var query = client.query("select name from bang"); assert.emits(query, 'row', function(row) { @@ -51,9 +51,9 @@ test("multiple simple queries", function() { test("multiple select statements", function() { var client = helper.client(); - client.query("create temp table boom(age integer); insert into boom(age) values(1); insert into boom(age) values(2); insert into boom(age) values(3)"); - client.query("create temp table bang(name varchar(5)); insert into bang(name) values('zoom');"); - var result = client.query("select age from boom where age < 2; select name from bang"); + client.query({text: "create temp table boom(age integer); insert into boom(age) values(1); insert into boom(age) values(2); insert into boom(age) values(3)", binary: false}); + client.query({text: "create temp table bang(name varchar(5)); insert into bang(name) values('zoom');", binary: false}); + var result = client.query({text: "select age from boom where age < 2; select name from bang", binary: false}); assert.emits(result, 'row', function(row) { assert.strictEqual(row['age'], 1); assert.emits(result, 'row', function(row) { diff --git a/test/integration/client/type-coercion-tests.js b/test/integration/client/type-coercion-tests.js index c8f39991a..fab12c6d1 100644 --- a/test/integration/client/type-coercion-tests.js +++ b/test/integration/client/type-coercion-tests.js @@ -79,6 +79,13 @@ var types = [{ values: ['13:12:12.321', null] }]; +// ignore some tests in binary mode +if (helper.config.binary) { + types = types.filter(function(type) { + return !(type.name in {'real':1, 'timetz':1, 'time':1}); + }); +} + var valueCount = 0; types.forEach(function(type) { valueCount += type.values.length; diff --git a/test/integration/connection-pool/unique-name-tests.js b/test/integration/connection-pool/unique-name-tests.js index 6893fbd7a..a92a00414 100644 --- a/test/integration/connection-pool/unique-name-tests.js +++ b/test/integration/connection-pool/unique-name-tests.js @@ -6,6 +6,7 @@ helper.pg.defaults.password = helper.args.password; helper.pg.defaults.database = helper.args.database; helper.pg.defaults.port = helper.args.port; helper.pg.defaults.host = helper.args.host; +helper.pg.defaults.binary = helper.args.binary; helper.pg.defaults.poolIdleTimeout = 100; var moreArgs = {}; From b2a2d029ab744cf8a5cf072480cf3b01a9509420 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 22 Nov 2011 05:00:16 +0100 Subject: [PATCH 37/37] fixed test, column should be accessed with name --- test/integration/client/type-coercion-tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/client/type-coercion-tests.js b/test/integration/client/type-coercion-tests.js index fab12c6d1..2c23f130c 100644 --- a/test/integration/client/type-coercion-tests.js +++ b/test/integration/client/type-coercion-tests.js @@ -23,7 +23,7 @@ var testForTypeCoercion = function(type){ }); assert.emits(query, 'row', function(row) { - assert.strictEqual(row.col, val, "expected " + type.name + " of " + val + " but got " + row[0]); + assert.strictEqual(row.col, val, "expected " + type.name + " of " + val + " but got " + row.col); }, "row should have been called for " + type.name + " of " + val); client.query('delete from test_type');