Permalink
Browse files

Somehow updated to 0.1.93, HTTP module still throwing errors sometimes

  • Loading branch information...
kurokikaze committed May 6, 2010
1 parent a41a8f4 commit 3a55f1f62a4e5ce68a19541ab3505f847adff893
Showing with 301 additions and 18 deletions.
  1. +23 −11 site/app.js
  2. +188 −0 site/bits.js
  3. +87 −6 site/limestone.js
  4. +3 −1 site/views/partials/result.html.haml
View
@@ -27,6 +27,7 @@ get('/search', function(){
this.render('results.html.haml', {'locals': {'header': 'Search results for "' + query + '"', 'query': query, 'results': 'Connection error'}});
}
//sys.puts('Connected, sending query');
+ debugger;
limestone.query({'query': query, maxmatches: 20}, function(err, answer) {
limestone.disconnect();
@@ -42,17 +43,28 @@ get('/search', function(){
});
})(function(elements){
var pages = [];
- for (element in elements) {
- pages.push({'title': elements[element].title, 'url': 'http://' + elements[element].url});
- }
-
- self.render('results.html.haml', {
- 'locals': {
- 'header': 'Search results for "' + query + '"',
- 'query': query,
- 'number_of_matches': 'Found ' + answer.match_count + ' matches',
- 'results': pages
- }
+
+ var texts = elements.map(function(doc){return doc.text});
+
+ limestone.connect(9312, function(err) {
+
+ limestone.buildExcerpts(texts, 'searchengine', answer.words, {}, function(err, excerpts) {
+
+ for (element in elements) {
+ pages.push({'title': elements[element].title, 'url': 'http://' + elements[element].url, 'excerpt':'excerpt for ' + elements[element]._id + '!'});
+ }
+
+ self.render('results.html.haml', {
+ 'locals': {
+ 'header': 'Search results for "' + query + '"',
+ 'query': query,
+ 'number_of_matches': 'Found ' + answer.match_count + ' matches, words are ' + JSON.stringify(answer.words),
+ 'results': pages
+ }
+ });
+
+ });
+
});
});
});
View
@@ -0,0 +1,188 @@
+/*jslint eqeqeq: true, immed: true, newcap: true, nomen: true, onevar: true, plusplus: true, regexp: true, undef: true, white: true, indent: 2 */
+
+// Converter between javascript values and "raw"
+// streams which are encoded as javascript strings
+// that only use the first 8 bits of each character.
+
+// exports.Encoder - A binary string builder object.
+(function () {
+
+ var chr, proto;
+
+ exports.Encoder = function (ServerCommand, ClientCommand) {
+ // if (ServerCommand & ClientCommand) {
+ this.header = encode_int16(ServerCommand) + encode_int16(ClientCommand);
+ // } else {
+ // this.header = "";
+ // }
+ this.data = "";
+ };
+
+ chr = String.fromCharCode;
+ proto = exports.Encoder.prototype;
+
+ // Factor out the encode so it can be shared by add_header and push_int32
+ function encode_int32(number) {
+ var a, b, c, d, unsigned;
+ unsigned = (number < 0) ? (number + 0x100000000) : number;
+ a = Math.floor(unsigned / 0xffffff);
+ unsigned &= 0xffffff;
+ b = Math.floor(unsigned / 0xffff);
+ unsigned &= 0xffff;
+ c = Math.floor(unsigned / 0xff);
+ unsigned &= 0xff;
+ d = Math.floor(unsigned);
+ return chr(a) + chr(b) + chr(c) + chr(d);
+ }
+
+ function encode_int16(number) {
+ var a, b, unsigned;
+ unsigned = (number < 0) ? (number + 0x10000) : number;
+ a = Math.floor(unsigned / 0xff);
+ unsigned &= 0xff;
+ b = Math.floor(unsigned);
+ return chr(a) + chr(b);
+ }
+
+ // Add a postgres header to the binary string and return it.
+ proto.toString = function (noCount) {
+ noCount = noCount | false;
+
+ var out_string = '';
+ if (noCount) {
+ out_string = this.header + encode_int32(this.data.length) + this.data;
+ } else {
+ out_string = this.header + encode_int32(this.data.length + 4) + encode_int32(1) + this.data;
+ }
+
+ return out_string;
+ };
+
+ // Return data without headers (e.g. for negotiating with server about protocol version)
+ proto.toRawString = function () {
+ return this.data;
+ };
+
+ // Encode number as 32 bit 2s compliment
+ proto.push_int32 = function (number) {
+ this.data += encode_int32(number);
+ return this;
+ };
+
+ // Encode number as 16 bit 2s compliment
+ proto.push_int16 = function (number) {
+ var a, b, unsigned;
+ unsigned = (number < 0) ? (number + 0x10000) : number;
+ a = Math.floor(unsigned / 0xff);
+ unsigned &= 0xff;
+ b = Math.floor(unsigned);
+ this.data += chr(a) + chr(b);
+ return this;
+ };
+
+ // Encode string without null terminator
+ proto.push_raw_string = function (text) {
+ this.data += text;
+ return this;
+ };
+
+ // Encode string and precede it with 32-bit int length
+ proto.push_lstring = function (text) {
+ this.data += encode_int32(text.length) + text;
+ return this;
+ };
+
+ // Encode text as null terminated string
+ proto.push_cstring = function (text) {
+ this.data += text + "\0";
+ return this;
+ };
+
+ // Encode as a null terminated array of cstrings
+ proto.push_multi_cstring = function (fields) {
+ this.data += fields.join("\0") + "\0\0";
+ return this;
+ };
+
+ proto.push_hash = function (hash) {
+ for (var key in hash) {
+ if (hash.hasOwnProperty(key)) {
+ this.data += key + "\0" + hash[key] + "\0";
+ }
+ }
+ this.data += "\0";
+ return this;
+ };
+
+}());
+
+// exports.Decoder - A binary string consumer object.
+// TODO: Convert to use a moving pointer instead of creating a new substring
+// each iteration. This will help performance a bit on parsing.
+(function () {
+
+ var proto;
+
+ exports.Decoder = function (data) {
+ this.data = data;
+ };
+
+ proto = exports.Decoder.prototype;
+
+ proto.shift_code = function () {
+ var code = this.data[0];
+ this.data = this.data.substr(1);
+ return code;
+ };
+
+ // Convert 4 characters to signed 32 bit integer
+ proto.shift_int32 = function () {
+ var unsigned = this.data.charCodeAt(0) * 0x1000000 + this.data.charCodeAt(1) * 0x10000 + this.data.charCodeAt(2) * 0x100 + this.data.charCodeAt(3);
+ this.data = this.data.substr(4);
+ return (unsigned & 0x80000000) ? (unsigned - 0x100000000) : unsigned;
+ };
+
+ // Convert 2 bytes to signed 16 bit integer
+ proto.shift_int16 = function () {
+ var unsigned = this.data.charCodeAt(0) * 0x100 + this.data.charCodeAt(1);
+ this.data = this.data.substr(2);
+ return (unsigned & 0x8000) ? (unsigned - 0x10000) : unsigned;
+ };
+
+ // Grab number of bytes as a string
+ proto.shift_raw_string = function (len) {
+ var string = this.data.substr(0, len);
+ this.data = this.data.substr(len);
+ return string;
+ };
+
+ // Grab number of bytes as a string
+ proto.shift_lstring = function () {
+ var length = this.shift_int32();
+ var string = this.data.substr(0, length);
+ this.data = this.data.substr(length);
+ return string;
+ };
+
+ // Grab a null terminated string
+ proto.shift_cstring = function () {
+ var pos, string;
+ pos = this.data.indexOf("\0");
+ string = this.data.substr(0, pos);
+ this.data = this.data.substr(pos + 1);
+ return string;
+ };
+
+ // Grab a null terminated array of null terminated strings
+ proto.shift_multi_cstring = function () {
+ var pos, string;
+ pos = this.data.indexOf("\0\0");
+ string = this.data.substr(0, pos).split("\0");
+ this.data = this.data.substr(pos + 1);
+ return string;
+ };
+
+
+}());
+
+
View
@@ -217,7 +217,7 @@ var Sphinx = {
} */
- var request = (new bits.Encoder(0, Sphinx.clientCommand.SEARCH)).push_int32(0).push_int32(20).push_int32(Sphinx.searchMode.ALL).push_int32(Sphinx.rankingMode.BM25).push_int32(Sphinx.sortMode.RELEVANCE);
+ var request = (new bits.Encoder(Sphinx.command.SEARCH, Sphinx.clientCommand.SEARCH)).push_int32(0).push_int32(20).push_int32(Sphinx.searchMode.ALL).push_int32(Sphinx.rankingMode.BM25).push_int32(Sphinx.sortMode.RELEVANCE);
request.push_int32(0); // "sort by" is not supported yet
@@ -263,7 +263,7 @@ var Sphinx = {
request.push_lstring(query_parameters.selectlist); // Select-list
- server_conn.write(request.toString(), 'binary');
+ server_conn.write(request.toString(), 'binary');
server_conn.addListener('data', function(data) {
// Got response!
@@ -375,7 +375,7 @@ var Sphinx = {
if (attribute.type == Sphinx.attribute.BIGINT) {
attr_value = response.shift_int32();
attr_value = response.shift_int32();
- match.attrs[output.attributes[attribute].name.name] = attr_value;
+ match.attrs[output.attributes[attribute].name] = attr_value;
continue;
}
@@ -402,17 +402,98 @@ var Sphinx = {
output.msecs = response.shift_int32();
output.words_count = response.shift_int32();
output.words = [];
- for (i = 0; i < output.words; i++) {
+ for (i = 0; i <=output.words; i++) {
output.words.push(response.shift_lstring());
}
-
// sys.puts('Unused data:' + response.length + ' bytes');
// @todo: implement words
return output;
}
+ Sphinx.buildExcerpts = function (matches, index, words, options, callback) {
+
+ if (typeof words == 'Array') {
+ words = words.join(',');
+ }
+ index = index || "*";
+
+ options = {
+ beforeMatch : options.beforeMatch || '<b>',
+ afterMatch : options.afterMatch || '</b>',
+ chunkSeparator : options.chunkSeparator || '...',
+ limit : options.limit || 256,
+ around : options.around || 5,
+ exactPhrase : options.exactPhrase || false,
+ singlePassage : options.singlePassage || false,
+ useBoundaries : options.useBoundaries || false,
+ weightOrder : options.weightOrder || false
+ }
+
+ var request = (new bits.Encoder(Sphinx.command.EXCERPT, Sphinx.clientCommand.EXCERPT));
+
+ // Start of actual request
+
+ request.push_int32(0); // mode
+
+ var flag = 1;
+
+ if (options.exactPhrase) flag = flag | 2;
+ if (options.singlePassage) flag = flag | 4;
+ if (options.useBoundaries) flag = flag | 8;
+ if (options.weightOrder) flag = flag | 16;
+
+ request.push_int32(parseInt(flag));
+
+ request.push_lstring(index);
+ request.push_lstring(words);
+
+ // options
+
+ request.push_lstring(options.beforeMatch);
+ request.push_lstring(options.afterMatch);
+ request.push_lstring(options.chunkSeparator);
+ request.push_int32(options.limit);
+ request.push_int32(options.around);
+
+ // Documents
+
+ request.push_int32(matches.count);
+
+ for (doc in matches) {
+ request.push_lstring(matches[doc].toString());
+ }
+
+ server_conn.write(request.toString(true), 'binary');
+
+ server_conn.addListener('data', function(data) {
+ // Got response!
+ // Command must match the one used in query
+ var response = getResponse(data, Sphinx.clientCommand.EXCERPT);
+
+ // var answer = parseSearchResponse(response);
+ response = new bits.Decoder(response);
+
+ var answer = [];
+
+ for (doc in matches) {
+ response.shift_int32(); // leading 0
+
+ var excerpt = response.shift_lstring();
+
+ if (excerpt.length > 0) {
+ answer[matches[doc]] = excerpt; // Excerpt string
+ }
+ }
+
+ callback(null, answer);
+
+ });
+
+ }
+
})();
-exports = Sphinx;
+// process.mixin(exports, Sphinx);
+for (var i in Sphinx) { exports[i] = Sphinx[i] };
@@ -1 +1,3 @@
-%li%a{href:result.url}= result.title
+%li
+ %a{href:result.url}= result.title
+ %p= result.excerpt

0 comments on commit 3a55f1f

Please sign in to comment.