moved from ksana-document

Author: yapcheahshen

configs.js

@@ -0,0 +1,56 @@
+var tokenizers=require('./tokenizers');
+var normalizeTbl=null;
+var setNormalizeTable=function(tbl,obj) {
+	if (!obj) {
+		obj={};
+		for (var i=0;i<tbl.length;i++) {
+			var arr=tbl[i].split("=");
+			obj[arr[0]]=arr[1];
+		}
+	}
+	normalizeTbl=obj;
+	return obj;
+}
+var normalize1=function(token) {
+	if (!token) return "";
+	token=token.replace(/[ \n\.,，。！．「」：；、]/g,'').trim();
+	if (!normalizeTbl) return token;
+	if (token.length==1) {
+		return normalizeTbl[token] || token;
+	} else {
+		for (var i=0;i<token.length;i++) {
+			token[i]=normalizeTbl[token[i]] || token[i];
+		}
+		return token;
+	}
+}
+var isSkip1=function(token) {
+	var t=token.trim();
+	return (t=="" || t=="　" || t=="※" || t=="\n");
+}
+var normalize_tibetan=function(token) {
+	return token.replace(/[།་ ]/g,'').trim();
+}
+
+var isSkip_tibetan=function(token) {
+	var t=token.trim();
+	return (t=="" || t=="　" ||  t=="\n");	
+}
+var simple1={
+	func:{
+		tokenize:tokenizers.simple
+		,setNormalizeTable:setNormalizeTable
+		,normalize: normalize1
+		,isSkip:	isSkip1
+	}
+	
+}
+var tibetan1={
+	func:{
+		tokenize:tokenizers.tibetan
+		,setNormalizeTable:setNormalizeTable
+		,normalize:normalize_tibetan
+		,isSkip:isSkip_tibetan
+	}
+}
+module.exports={"simple1":simple1,"tibetan1":tibetan1}

index.js

@@ -0,0 +1,37 @@
+/* 
+  custom func for building and searching ydb
+
+  keep all version
+  
+  getAPI(version); //return hash of functions , if ver is omit , return lastest
+	
+  postings2Tree      // if version is not supply, get lastest
+  tokenize(text,api) // convert a string into tokens(depends on other api)
+  normalizeToken     // stemming and etc
+  isSpaceChar        // not a searchable token
+  isSkipChar         // 0 vpos
+
+  for client and server side
+  
+*/
+var configs=require("./configs");
+var config_simple="simple1";
+var optimize=function(json,config) {
+	config=config||config_simple;
+	return json;
+}
+
+var getAPI=function(config) {
+	config=config||config_simple;
+	var func=configs[config].func;
+	func.optimize=optimize;
+	if (config=="simple1") {
+		//add common custom function here
+	} else if (config=="tibetan1") {
+
+	} else throw "config "+config +"not supported";
+
+	return func;
+}
+
+module.exports={getAPI:getAPI};

package.json

@@ -0,0 +1,23 @@
+{
+  "name": "ksana-analyzer",
+  "version": "1.0.0",
+  "description": "convert text stream to tokens",
+  "main": "index.js",
+  "scripts": {
+    "test": "mocha"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://yapcheahshen@github.com/ksanaforge/ksana-analyzer"
+  },
+  "keywords": [
+    "tokenizing",
+    "segmentation"
+  ],
+  "author": "yapcheahshen@gmail.com",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/ksanaforge/ksana-analyzer/issues"
+  },
+  "homepage": "https://github.com/ksanaforge/ksana-analyzer"
+}

tokenizers.js

@@ -0,0 +1,153 @@
+var tibetan =function(s) {
+	//continuous tsheg grouped into same token
+	//shad and space grouped into same token
+	var offset=0;
+	var tokens=[],offsets=[];
+	s=s.replace(/\r\n/g,'\n').replace(/\r/g,'\n');
+	var arr=s.split('\n');
+
+	for (var i=0;i<arr.length;i++) {
+		var last=0;
+		var str=arr[i];
+		str.replace(/[།་ ]+/g,function(m,m1){
+			tokens.push(str.substring(last,m1)+m);
+			offsets.push(offset+last);
+			last=m1+m.length;
+		});
+		if (last<str.length) {
+			tokens.push(str.substring(last));
+			offsets.push(last);
+		}
+		if (i===arr.length-1) break;
+		tokens.push('\n');
+		offsets.push(offset+last);
+		offset+=str.length+1;
+	}
+
+	return {tokens:tokens,offsets:offsets};
+};
+var isSpace=function(c) {
+	return (c==" ") ;//|| (c==",") || (c==".");
+}
+var isCJK =function(c) {return ((c>=0x3000 && c<=0x9FFF) 
+|| (c>=0xD800 && c<0xDC00) || (c>=0xFF00) ) ;}
+var simple1=function(s) {
+	var offset=0;
+	var tokens=[],offsets=[];
+	s=s.replace(/\r\n/g,'\n').replace(/\r/g,'\n');
+	arr=s.split('\n');
+
+	var pushtoken=function(t,off) {
+		var i=0;
+		if (t.charCodeAt(0)>255) {
+			while (i<t.length) {
+				var c=t.charCodeAt(i);
+				offsets.push(off+i);
+				tokens.push(t[i]);
+				if (c>=0xD800 && c<=0xDFFF) {
+					tokens[tokens.length-1]+=t[i]; //extension B,C,D
+				}
+				i++;
+			}
+		} else {
+			tokens.push(t);
+			offsets.push(off);	
+		}
+	}
+	for (var i=0;i<arr.length;i++) {
+		var last=0,sp="";
+		str=arr[i];
+		str.replace(/[_0-9A-Za-z]+/g,function(m,m1){
+			while (isSpace(sp=str[last]) && last<str.length) {
+				tokens[tokens.length-1]+=sp;
+				last++;
+			}
+			pushtoken(str.substring(last,m1)+m , offset+last);
+			offsets.push(offset+last);
+			last=m1+m.length;
+		});
+
+		if (last<str.length) {
+			while (isSpace(sp=str[last]) && last<str.length) {
+				tokens[tokens.length-1]+=sp;
+				last++;
+			}
+			pushtoken(str.substring(last), offset+last);
+			
+		}		
+		offsets.push(offset+last);
+		offset+=str.length+1;
+		if (i===arr.length-1) break;
+		tokens.push('\n');
+	}
+
+	return {tokens:tokens,offsets:offsets};
+
+};
+
+var simple=function(s) {
+	var token='';
+	var tokens=[], offsets=[] ;
+	var i=0; 
+	var lastspace=false;
+	var addtoken=function() {
+		if (!token) return;
+		tokens.push(token);
+		offsets.push(i);
+		token='';
+	}
+	while (i<s.length) {
+		var c=s.charAt(i);
+		var code=s.charCodeAt(i);
+		if (isCJK(code)) {
+			addtoken();
+			token=c;
+			if (code>=0xD800 && code<0xDC00) { //high sorragate
+				token+=s.charAt(i+1);i++;
+			}
+			addtoken();
+		} else {
+			if (c=='&' || c=='<' || c=='?' || c=="," || c=="."
+			|| c=='|' || c=='~' || c=='`' || c==';' 
+			|| c=='>' || c==':' 
+			|| c=='=' || c=='@'  || c=="-" 
+			|| c==']' || c=='}'  || c==")" 
+			//|| c=='{' || c=='}'|| c=='[' || c==']' || c=='(' || c==')'
+			|| code==0xf0b || code==0xf0d // tibetan space
+			|| (code>=0x2000 && code<=0x206f)) {
+				addtoken();
+				if (c=='&' || c=='<'){ // || c=='{'|| c=='('|| c=='[') {
+					var endchar='>';
+					if (c=='&') endchar=';'
+					//else if (c=='{') endchar='}';
+					//else if (c=='[') endchar=']';
+					//else if (c=='(') endchar=')';
+
+					while (i<s.length && s.charAt(i)!=endchar) {
+						token+=s.charAt(i);
+						i++;
+					}
+					token+=endchar;
+					addtoken();
+				} else {
+					token=c;
+					addtoken();
+				}
+				token='';
+			} else {
+				if (c==" ") {
+					token+=c;
+					lastspace=true;
+				} else {
+					if (lastspace) addtoken();
+					lastspace=false;
+					token+=c;
+				}
+			}
+		}
+		i++;
+	}
+	addtoken();
+	return {tokens:tokens,offsets:offsets};
+}
+module.exports={simple:simple,tibetan:tibetan};