Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

better node.js integration;; fixed encoding problem -> java side

  • Loading branch information...
commit 6264fa050ea071cf007433e79b6ab340da683889 1 parent 29ed53f
@karussell authored
View
16 README
@@ -19,4 +19,18 @@ At the moment feeding is possible only via API (engine.feedDocs) see Server.js
start server via 'node Server.js'
-query via http://127.0.0.1:8124/select?q=blup
+query via http://127.0.0.1:8124/select?q=blup
+
+
+Features
+========
+
+* 100% JavaScript
+* 100% in-memory
+* 100% open source, Apache 2 License
+* Queryable via HTTP
+* Solr compatible JSON
+* Boostable query
+* pagable, filterable and sortable results
+* Real time BUT: jsii is not transaction safe, because there is no 'commit'
+* Unit tests
View
26 web/js/todo.txt → TODO.txt
@@ -1,3 +1,5 @@
+support fq param
+
jetwick:
de.jetwick.solr.SolrTweetSearch.readDoc(SolrTweetSearch.java:196)
java.lang.String cannot be cast to java.util.Date
@@ -9,25 +11,25 @@ FATAL ERROR: CALL_AND_RETRY_2 Allocation failed - process out of memory
=> 10 KB for 1 doc !?
call gc explicitely! increase available RAM!?
-when feeding docs do not add but overwrite existing! use idField
-
-parse date when feeding!
- /SECOND /MINUTE !?
-
-sort against id (date)
-
+parse date when feeding! /SECOND /MINUTE !?
measure RAM usage + qtime when load testing
regular refresh via F5 -> only first result differs
range query date:[NOW-8HOUR TO *]
create a filter cache. keys als "date:[mySpecialFilter]" values ala DocBitSet (docs that fullfill the query)
search in AND-results via doc.field.indexOf("x y z") when querying "x y z" (query with "" !)
-npm install jsii
+
js twitter clients
http://github.com/masylum/twitter-js
http://github.com/jdub/node-twitter
-support sort=field asc/desc (int, date->long)
+major TODOs
+* when feeding: overwrite existing! use idField
+* feeding via HTTP
+* range queries
+* better scoring (with tf = min(4,tf))
+* is there really no common way to improve the speed of iterating over an hash ala Java:
+ for(var key, value in hash.entries) { }
+* npm install jsii
-TODO
- jetwick integration is difficult because
- solrj does not have a JSONResponseParser yet: https://issues.apache.org/jira/browse/SOLR-402
+SolrJ
+* solrj does not have a JSONResponseParser yet: https://issues.apache.org/jira/browse/SOLR-402
View
21 web/js/spec/DocSpec.js
@@ -1,21 +0,0 @@
-//@deprecated
-describe("Doc", function() {
-
- it("should clone itself", function() {
- var doc = new Doc({
- test: "bla blu",
- id : 1
- });
-
- var cDoc = doc.clone();
- cDoc["test"] = "blip";
- cDoc.score = 1;
- expect(doc["test"]).toBe("bla blu");
- expect(doc.test).toBe("bla blu");
- expect(doc.id).toBe(1);
- expect(doc.score).toBe(0);
- expect(cDoc.score).toBe(1);
- expect(cDoc["test"]).toBe("blip");
- expect(cDoc["id"]).toBe(1);
- });
-});
View
5 web/js/src/BitSet.js
@@ -14,10 +14,13 @@
* a common operation is 'x >>> 5' which converts the bitSetIndex into
* the index necessary to retrieve one of the 32 bit chunk.
*/
-BitSet = function () {
+
+BitSet = function() {
this.arr = [];
}
+if (typeof module !== "undefined") module.exports = BitSet
+
BitSet.prototype.and = function(set) {
// a & a is just a
if (this === set)
View
21 web/js/src/Doc.js
@@ -1,21 +0,0 @@
-// @deprecated
-function Doc(hash) {
- for(var prop in hash)
- this[prop] = hash[prop];
-
- // TODO make sure no property is named 'score''
- this.score = 0;
-}
-
-// Doc extends Array
-//Doc.prototype = new Array();
-// but call Doc constructor ...
-//Doc.prototype.constructor = Doc
-// call ancestor methods using Function.call()
-// Doc.prototype.mymethod=function(){
-// Array.prototype.mymethod.call(this);
-// }
-
-Doc.prototype.clone = function() {
- return clone(this);
-}
View
8 web/js/src/JSii.js
@@ -7,7 +7,7 @@
*
* fast (?), in-memory, simple
*/
-JSii = function () {
+JSii = function() {
// inverted index ala {fieldX: {"test": bitSet1}, {"pest": bitSet2},
// fieldY: {"test": bitSet3}, ...
this.iindex = {};
@@ -22,7 +22,8 @@ JSii = function () {
id : 'string',
text : 'text',
tw : 'text',
- user : 'string'
+ user : 'string',
+ lang : 'string'
};
this.defaultSearchField = 'text';
@@ -36,6 +37,9 @@ JSii = function () {
}
}
+if (typeof module !== "undefined") module.exports = JSii
+
+
JSii.prototype.trim = function (str) {
return str.replace(/^\s*/, "").replace(/\s*$/, "");
}
View
38 web/js/src/Server.js
@@ -4,18 +4,23 @@
* Using node.js v0.2.4
*/
-// require statements are not hotupdateable!?
+
+require('./json2');
+
+// require statements are not hotupdateable
+// process is a global variable
+
var http = require('http');
//var querystring = require('querystring');
var url = require('url');
var fs = require('fs');
var util = require('util');
-// process is a global variable
-require('./JSii');
-require('./BitSet');
-require('./json2');
-require("./Solr");
-require("./XmlHandler");
+
+// require is necessary for JSii
+var BitSet = require('./BitSet');
+var JSii = require('./JSii');
+var SolrClient = require("./SolrClient");
+var XmlHandler = require("./XmlHandler");
// feed docs from solr into our index
var engine = new JSii();
@@ -27,7 +32,7 @@ var querySolr = function(webapp, login, pw) {
var queryStr = "";
var feedingInProcess = false;
- var feedDocsCallBack = function (err, response) {
+ var feedDocsCallBack = function (err, response) {
var responseObj = JSON.parse(response);
if(responseObj == null) {
console.log("Something goes wrong. response was null");
@@ -44,8 +49,9 @@ var querySolr = function(webapp, login, pw) {
var options = {};
options.start = 0;
- options.rows = 1000;
- //options.fq = "lang:en";
+ options.rows = 100;
+ // prefer english/german lang
+// options.fq = "lang:de";
setInterval(function() {
if(feedingInProcess)
return;
@@ -53,7 +59,7 @@ var querySolr = function(webapp, login, pw) {
feedingInProcess = true;
client.query(queryStr, options, feedDocsCallBack);
options.start += options.rows;
- }, 60 * 1000);
+ }, 10000);
}
fs.open("src/pw.txt", "r", 0666, function(err, fd){
@@ -74,10 +80,16 @@ fs.open("src/pw.txt", "r", 0666, function(err, fd){
// static mini example feeding
//engine.feedDocs([{
// id:1,
-// text : "blasenfrei blup"
+// tw : "unschlüssig blup",
+// user: "peter",
+// repl_i : 0,
+// retw_i : 0
//}, {
// id:2,
-// text : "blap blup"
+// tw : "blap blup",
+// user: "test",
+// repl_i : 0,
+// retw_i : 0
//}]);
// accept clients
View
4 web/js/src/Solr.js → web/js/src/SolrClient.js
@@ -10,7 +10,7 @@ var http = require("http");
var querystring = require("querystring");
var base64 = require('./base64');
-SolrClient = function (host, port, webapp, login, pw) {
+SolrClient = function(host, port, webapp, login, pw) {
this.host = host || "127.0.0.1";
this.port = port || "8983";
this.fullHost = this.host + ":" + this.port;
@@ -21,6 +21,8 @@ SolrClient = function (host, port, webapp, login, pw) {
this.pw = pw;
};
+if (typeof module !== "undefined") module.exports = SolrClient
+
SolrClient.prototype.query = function (query, options, callback) {
var queryParams = options || {};
queryParams.q = query;
View
11 web/js/src/XmlHandler.js
@@ -4,8 +4,11 @@ XmlHandler = function() {
this.stack = [];
}
+if (typeof module !== "undefined") module.exports = XmlHandler
+
XmlHandler.prototype.header = function() {
- this.str = '<?xml version="1.0" encoding="UTF-8"?>';
+ this.str = '<?xml version="1.0" encoding="utf8"?>';
+// this.str = '';
return this;
}
@@ -28,7 +31,7 @@ XmlHandler.prototype.start = function(el, attrs) {
}
XmlHandler.prototype.text = function(str) {
- if(typeof str === "string")
+ if(typeof str === "string") {
for(var i = 0; i < str.length; i++) {
var tmp = str.charAt(i);
switch(tmp) {
@@ -50,8 +53,8 @@ XmlHandler.prototype.text = function(str) {
default:
this.str += tmp;
}
- }
- else
+ }
+ } else
this.str += '' + str;
return this;
}
View
306 web/js/src/json2.js
@@ -30,12 +30,12 @@ if (!this.JSON) {
Date.prototype.toJSON = function (key) {
return isFinite(this.valueOf()) ?
- this.getUTCFullYear() + '-' +
- f(this.getUTCMonth() + 1) + '-' +
- f(this.getUTCDate()) + 'T' +
- f(this.getUTCHours()) + ':' +
- f(this.getUTCMinutes()) + ':' +
- f(this.getUTCSeconds()) + 'Z' : null;
+ this.getUTCFullYear() + '-' +
+ f(this.getUTCMonth() + 1) + '-' +
+ f(this.getUTCDate()) + 'T' +
+ f(this.getUTCHours()) + ':' +
+ f(this.getUTCMinutes()) + ':' +
+ f(this.getUTCSeconds()) + 'Z' : null;
};
String.prototype.toJSON =
@@ -46,226 +46,228 @@ if (!this.JSON) {
}
var cx = /[\u0000\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g,
- escapable = /[\\\"\x00-\x1f\x7f-\x9f\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g,
- gap,
- indent,
- meta = { // table of character substitutions
- '\b': '\\b',
- '\t': '\\t',
- '\n': '\\n',
- '\f': '\\f',
- '\r': '\\r',
- '"' : '\\"',
- '\\': '\\\\'
- },
- rep;
+ escapable = /[\\\"\x00-\x1f\x7f-\x9f\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g,
+ gap,
+ indent,
+ meta = { // table of character substitutions
+ '\b': '\\b',
+ '\t': '\\t',
+ '\n': '\\n',
+ '\f': '\\f',
+ '\r': '\\r',
+ '"' : '\\"',
+ '\\': '\\\\'
+ },
+ rep;
function quote(string) {
-// If the string contains no control characters, no quote characters, and no
-// backslash characters, then we can safely slap some quotes around it.
-// Otherwise we must also replace the offending characters with safe escape
-// sequences.
+ // If the string contains no control characters, no quote characters, and no
+ // backslash characters, then we can safely slap some quotes around it.
+ // Otherwise we must also replace the offending characters with safe escape
+ // sequences.
escapable.lastIndex = 0;
return escapable.test(string) ?
- '"' + string.replace(escapable, function (a) {
- var c = meta[a];
- return typeof c === 'string' ? c :
- '\\u' + ('0000' + a.charCodeAt(0).toString(16)).slice(-4);
- }) + '"' :
- '"' + string + '"';
+ '"' + string.replace(escapable, function (a) {
+ var c = meta[a];
+ return typeof c === 'string' ? c :
+ '\\u' + ('0000' + a.charCodeAt(0).toString(16)).slice(-4);
+ }) + '"' :
+ '"' + string + '"';
}
function str(key, holder) {
-// Produce a string from holder[key].
+ // Produce a string from holder[key].
var i, // The loop counter.
- k, // The member key.
- v, // The member value.
- length,
- mind = gap,
- partial,
- value = holder[key];
+ k, // The member key.
+ v, // The member value.
+ length,
+ mind = gap,
+ partial,
+ value = holder[key];
-// If the value has a toJSON method, call it to obtain a replacement value.
+ // If the value has a toJSON method, call it to obtain a replacement value.
if (value && typeof value === 'object' &&
- typeof value.toJSON === 'function') {
+ typeof value.toJSON === 'function') {
value = value.toJSON(key);
}
-// If we were called with a replacer function, then call the replacer to
-// obtain a replacement value.
+ // If we were called with a replacer function, then call the replacer to
+ // obtain a replacement value.
if (typeof rep === 'function') {
value = rep.call(holder, key, value);
}
-// What happens next depends on the value's type.
+ // What happens next depends on the value's type.
switch (typeof value) {
- case 'string':
- return quote(value);
+ case 'string':
+ return quote(value);
- case 'number':
+ case 'number':
-// JSON numbers must be finite. Encode non-finite numbers as null.
+ // JSON numbers must be finite. Encode non-finite numbers as null.
- return isFinite(value) ? String(value) : 'null';
+ return isFinite(value) ? String(value) : 'null';
- case 'boolean':
- case 'null':
+ case 'boolean':
+ case 'null':
-// If the value is a boolean or null, convert it to a string. Note:
-// typeof null does not produce 'null'. The case is included here in
-// the remote chance that this gets fixed someday.
+ // If the value is a boolean or null, convert it to a string. Note:
+ // typeof null does not produce 'null'. The case is included here in
+ // the remote chance that this gets fixed someday.
- return String(value);
+ return String(value);
-// If the type is 'object', we might be dealing with an object or an array or
-// null.
+ // If the type is 'object', we might be dealing with an object or an array or
+ // null.
- case 'object':
+ case 'object':
-// Due to a specification blunder in ECMAScript, typeof null is 'object',
-// so watch out for that case.
+ // Due to a specification blunder in ECMAScript, typeof null is 'object',
+ // so watch out for that case.
- if (!value) {
- return 'null';
- }
+ if (!value) {
+ return 'null';
+ }
-// Make an array to hold the partial results of stringifying this object value.
+ // Make an array to hold the partial results of stringifying this object value.
- gap += indent;
- partial = [];
+ gap += indent;
+ partial = [];
-// Is the value an array?
+ // Is the value an array?
- if (Object.prototype.toString.apply(value) === '[object Array]') {
+ if (Object.prototype.toString.apply(value) === '[object Array]') {
-// The value is an array. Stringify every element. Use null as a placeholder
-// for non-JSON values.
+ // The value is an array. Stringify every element. Use null as a placeholder
+ // for non-JSON values.
- length = value.length;
- for (i = 0; i < length; i += 1) {
- partial[i] = str(i, value) || 'null';
- }
+ length = value.length;
+ for (i = 0; i < length; i += 1) {
+ partial[i] = str(i, value) || 'null';
+ }
-// Join all of the elements together, separated with commas, and wrap them in
-// brackets.
+ // Join all of the elements together, separated with commas, and wrap them in
+ // brackets.
- v = partial.length === 0 ? '[]' :
+ v = partial.length === 0 ? '[]' :
gap ? '[\n' + gap +
- partial.join(',\n' + gap) + '\n' +
- mind + ']' :
- '[' + partial.join(',') + ']';
- gap = mind;
- return v;
- }
+ partial.join(',\n' + gap) + '\n' +
+ mind + ']' :
+ '[' + partial.join(',') + ']';
+ gap = mind;
+ return v;
+ }
-// If the replacer is an array, use it to select the members to be stringified.
+ // If the replacer is an array, use it to select the members to be stringified.
- if (rep && typeof rep === 'object') {
- length = rep.length;
- for (i = 0; i < length; i += 1) {
- k = rep[i];
- if (typeof k === 'string') {
- v = str(k, value);
- if (v) {
- partial.push(quote(k) + (gap ? ': ' : ':') + v);
+ if (rep && typeof rep === 'object') {
+ length = rep.length;
+ for (i = 0; i < length; i += 1) {
+ k = rep[i];
+ if (typeof k === 'string') {
+ v = str(k, value);
+ if (v) {
+ partial.push(quote(k) + (gap ? ': ' : ':') + v);
+ }
}
}
- }
- } else {
+ } else {
-// Otherwise, iterate through all of the keys in the object.
+ // Otherwise, iterate through all of the keys in the object.
- for (k in value) {
- if (Object.hasOwnProperty.call(value, k)) {
- v = str(k, value);
- if (v) {
- partial.push(quote(k) + (gap ? ': ' : ':') + v);
+ for (k in value) {
+ if (Object.hasOwnProperty.call(value, k)) {
+ v = str(k, value);
+ if (v) {
+ partial.push(quote(k) + (gap ? ': ' : ':') + v);
+ }
}
}
}
- }
-// Join all of the member texts together, separated with commas,
-// and wrap them in braces.
+ // Join all of the member texts together, separated with commas,
+ // and wrap them in braces.
- v = partial.length === 0 ? '{}' :
+ v = partial.length === 0 ? '{}' :
gap ? '{\n' + gap + partial.join(',\n' + gap) + '\n' +
- mind + '}' : '{' + partial.join(',') + '}';
- gap = mind;
- return v;
+ mind + '}' : '{' + partial.join(',') + '}';
+ gap = mind;
+ return v;
}
}
-// If the JSON object does not yet have a stringify method, give it one.
+ // If the JSON object does not yet have a stringify method, give it one.
if (typeof JSON.stringify !== 'function') {
JSON.stringify = function (value, replacer, space) {
-// The stringify method takes a value and an optional replacer, and an optional
-// space parameter, and returns a JSON text. The replacer can be a function
-// that can replace values, or an array of strings that will select the keys.
-// A default replacer method can be provided. Use of the space parameter can
-// produce text that is more easily readable.
+ // The stringify method takes a value and an optional replacer, and an optional
+ // space parameter, and returns a JSON text. The replacer can be a function
+ // that can replace values, or an array of strings that will select the keys.
+ // A default replacer method can be provided. Use of the space parameter can
+ // produce text that is more easily readable.
var i;
gap = '';
indent = '';
-// If the space parameter is a number, make an indent string containing that
-// many spaces.
+ // If the space parameter is a number, make an indent string containing that
+ // many spaces.
if (typeof space === 'number') {
for (i = 0; i < space; i += 1) {
indent += ' ';
}
-// If the space parameter is a string, it will be used as the indent string.
+ // If the space parameter is a string, it will be used as the indent string.
} else if (typeof space === 'string') {
indent = space;
}
-// If there is a replacer, it must be a function or an array.
-// Otherwise, throw an error.
+ // If there is a replacer, it must be a function or an array.
+ // Otherwise, throw an error.
rep = replacer;
if (replacer && typeof replacer !== 'function' &&
- (typeof replacer !== 'object' ||
- typeof replacer.length !== 'number')) {
+ (typeof replacer !== 'object' ||
+ typeof replacer.length !== 'number')) {
throw new Error('JSON.stringify');
}
-// Make a fake root object containing our value under the key of ''.
-// Return the result of stringifying the value.
+ // Make a fake root object containing our value under the key of ''.
+ // Return the result of stringifying the value.
- return str('', {'': value});
+ return str('', {
+ '': value
+ });
};
}
-// If the JSON object does not yet have a parse method, give it one.
+ // If the JSON object does not yet have a parse method, give it one.
if (typeof JSON.parse !== 'function') {
JSON.parse = function (text, reviver) {
-// The parse method takes a text and an optional reviver function, and returns
-// a JavaScript value if the text is a valid JSON text.
+ // The parse method takes a text and an optional reviver function, and returns
+ // a JavaScript value if the text is a valid JSON text.
var j;
function walk(holder, key) {
-// The walk method is used to recursively walk the resulting structure so
-// that modifications can be made.
+ // The walk method is used to recursively walk the resulting structure so
+ // that modifications can be made.
var k, v, value = holder[key];
if (value && typeof value === 'object') {
@@ -284,52 +286,54 @@ if (!this.JSON) {
}
-// Parsing happens in four stages. In the first stage, we replace certain
-// Unicode characters with escape sequences. JavaScript handles many characters
-// incorrectly, either silently deleting them, or treating them as line endings.
+ // Parsing happens in four stages. In the first stage, we replace certain
+ // Unicode characters with escape sequences. JavaScript handles many characters
+ // incorrectly, either silently deleting them, or treating them as line endings.
text = String(text);
cx.lastIndex = 0;
if (cx.test(text)) {
text = text.replace(cx, function (a) {
return '\\u' +
- ('0000' + a.charCodeAt(0).toString(16)).slice(-4);
+ ('0000' + a.charCodeAt(0).toString(16)).slice(-4);
});
}
-// In the second stage, we run the text against regular expressions that look
-// for non-JSON patterns. We are especially concerned with '()' and 'new'
-// because they can cause invocation, and '=' because it can cause mutation.
-// But just to be safe, we want to reject all unexpected forms.
+ // In the second stage, we run the text against regular expressions that look
+ // for non-JSON patterns. We are especially concerned with '()' and 'new'
+ // because they can cause invocation, and '=' because it can cause mutation.
+ // But just to be safe, we want to reject all unexpected forms.
-// We split the second stage into 4 regexp operations in order to work around
-// crippling inefficiencies in IE's and Safari's regexp engines. First we
-// replace the JSON backslash pairs with '@' (a non-JSON character). Second, we
-// replace all simple value tokens with ']' characters. Third, we delete all
-// open brackets that follow a colon or comma or that begin the text. Finally,
-// we look to see that the remaining characters are only whitespace or ']' or
-// ',' or ':' or '{' or '}'. If that is so, then the text is safe for eval.
+ // We split the second stage into 4 regexp operations in order to work around
+ // crippling inefficiencies in IE's and Safari's regexp engines. First we
+ // replace the JSON backslash pairs with '@' (a non-JSON character). Second, we
+ // replace all simple value tokens with ']' characters. Third, we delete all
+ // open brackets that follow a colon or comma or that begin the text. Finally,
+ // we look to see that the remaining characters are only whitespace or ']' or
+ // ',' or ':' or '{' or '}'. If that is so, then the text is safe for eval.
if (/^[\],:{}\s]*$/
-.test(text.replace(/\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g, '@')
-.replace(/"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g, ']')
-.replace(/(?:^|:|,)(?:\s*\[)+/g, ''))) {
+ .test(text.replace(/\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g, '@')
+ .replace(/"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g, ']')
+ .replace(/(?:^|:|,)(?:\s*\[)+/g, ''))) {
-// In the third stage we use the eval function to compile the text into a
-// JavaScript structure. The '{' operator is subject to a syntactic ambiguity
-// in JavaScript: it can begin a block or an object literal. We wrap the text
-// in parens to eliminate the ambiguity.
+ // In the third stage we use the eval function to compile the text into a
+ // JavaScript structure. The '{' operator is subject to a syntactic ambiguity
+ // in JavaScript: it can begin a block or an object literal. We wrap the text
+ // in parens to eliminate the ambiguity.
j = eval('(' + text + ')');
-// In the optional fourth stage, we recursively walk the new structure, passing
-// each name/value pair to a reviver function for possible transformation.
+ // In the optional fourth stage, we recursively walk the new structure, passing
+ // each name/value pair to a reviver function for possible transformation.
return typeof reviver === 'function' ?
- walk({'': j}, '') : j;
+ walk({
+ '': j
+ }, '') : j;
}
-// If the text is not JSON parseable, then a SyntaxError is thrown.
+ // If the text is not JSON parseable, then a SyntaxError is thrown.
throw new SyntaxError('JSON.parse');
};
Please sign in to comment.
Something went wrong with that request. Please try again.